{ "cells": [ { "cell_type": "markdown", "id": "c931e650", "metadata": { "collapsed": true, "customInput": null, "originalKey": "ac61b043-8ebf-43b9-9fa5-ed9a42a184ce", "papermill": { "duration": 0.006766, "end_time": "2024-05-13T05:18:53.582007", "exception": false, "start_time": "2024-05-13T05:18:53.575241", "status": "completed" }, "showInput": false, "tags": [] }, "source": [ "# Tune a CNN on MNIST\n", "\n", "This tutorial walks through using Ax to tune two hyperparameters (learning rate and momentum) for a PyTorch CNN on the MNIST dataset trained using SGD with momentum." ] }, { "cell_type": "code", "execution_count": 1, "id": "471b1a98", "metadata": { "customInput": null, "customOutput": null, "execution": { "iopub.execute_input": "2024-05-13T05:18:53.595708Z", "iopub.status.busy": "2024-05-13T05:18:53.595213Z", "iopub.status.idle": "2024-05-13T05:18:57.186492Z", "shell.execute_reply": "2024-05-13T05:18:57.185667Z" }, "executionStartTime": 1690415246079, "executionStopTime": 1690415266324, "originalKey": "c2b37f0f-3644-4367-912f-f775082f6676", "papermill": { "duration": 3.612065, "end_time": "2024-05-13T05:18:57.200185", "exception": false, "start_time": "2024-05-13T05:18:53.588120", "status": "completed" }, "requestMsgId": "0b481630-f0f4-436a-a205-a25aa163a364", "showInput": true, "tags": [] }, "outputs": [ { "data": { "text/html": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:18:57] ax.utils.notebook.plotting: Injecting Plotly library into cell. Do not overwrite or delete cell.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:18:57] ax.utils.notebook.plotting: Please see\n", " (https://ax.dev/tutorials/visualizations.html#Fix-for-plots-that-are-not-rendering)\n", " if visualizations are not rendering.\n" ] }, { "data": { "text/html": [ " \n", " " ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "import torch\n", "\n", "import torch.nn as nn\n", "import torch.nn.functional as F\n", "\n", "from ax.service.ax_client import AxClient, ObjectiveProperties\n", "from ax.service.utils.report_utils import exp_to_df\n", "from ax.utils.notebook.plotting import init_notebook_plotting, render\n", "from ax.utils.tutorials.cnn_utils import evaluate, load_mnist, train\n", "from torch._tensor import Tensor\n", "from torch.utils.data import DataLoader\n", "\n", "init_notebook_plotting()" ] }, { "cell_type": "code", "execution_count": 2, "id": "f2a4eb05", "metadata": { "customInput": null, "customOutput": null, "execution": { "iopub.execute_input": "2024-05-13T05:18:57.273339Z", "iopub.status.busy": "2024-05-13T05:18:57.273058Z", "iopub.status.idle": "2024-05-13T05:18:57.277943Z", "shell.execute_reply": "2024-05-13T05:18:57.277428Z" }, "executionStartTime": 1690415266521, "executionStopTime": 1690415266529, "originalKey": "4d0a27c4-a6ce-4b7d-97eb-1c229aabb375", "papermill": { "duration": 0.042899, "end_time": "2024-05-13T05:18:57.279183", "exception": false, "start_time": "2024-05-13T05:18:57.236284", "status": "completed" }, "requestMsgId": "fd975d25-a185-4b09-a50f-7b2bcd89f93f", "showInput": true, "tags": [] }, "outputs": [], "source": [ "torch.manual_seed(42)\n", "dtype = torch.float\n", "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")" ] }, { "cell_type": "markdown", "id": "d5f57409", "metadata": { "customInput": null, "originalKey": "10384e51-444c-4265-b56d-ad078d05d2a1", "papermill": { "duration": 0.03555, "end_time": "2024-05-13T05:18:57.350493", "exception": false, "start_time": "2024-05-13T05:18:57.314943", "status": "completed" }, "showInput": false, "tags": [] }, "source": [ "## 1. Load MNIST data\n", "First, we need to load the MNIST data and partition it into training, validation, and test sets.\n", "\n", "Note: this will download the dataset if necessary." ] }, { "cell_type": "code", "execution_count": 3, "id": "0dc6f4e8", "metadata": { "customInput": null, "customOutput": null, "execution": { "iopub.execute_input": "2024-05-13T05:18:57.424675Z", "iopub.status.busy": "2024-05-13T05:18:57.424073Z", "iopub.status.idle": "2024-05-13T05:19:01.270896Z", "shell.execute_reply": "2024-05-13T05:19:01.270106Z" }, "executionStartTime": 1690415266733, "executionStopTime": 1690415266902, "originalKey": "6f0949e2-1064-44b8-99c0-f6ce23df7c63", "papermill": { "duration": 3.885159, "end_time": "2024-05-13T05:19:01.272362", "exception": false, "start_time": "2024-05-13T05:18:57.387203", "status": "completed" }, "requestMsgId": "8ce7dd21-9afb-4379-ad11-4112b4d27f8a", "showInput": true, "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Failed to download (trying next):\n", "HTTP Error 403: Forbidden\n", "\n", "Downloading https://ossci-datasets.s3.amazonaws.com/mnist/train-images-idx3-ubyte.gz\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Downloading https://ossci-datasets.s3.amazonaws.com/mnist/train-images-idx3-ubyte.gz to ./data/MNIST/raw/train-images-idx3-ubyte.gz\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", " 0%| | 0/9912422 [00:00= p2\" or \"p1 + p2 <= some_bound\".\n", " # outcome_constraints: Optional, a list of strings of form \"constrained_metric <= some_bound\".\n", ")" ] }, { "cell_type": "markdown", "id": "4af08b5d", "metadata": { "customInput": null, "originalKey": "af441a83-50fd-4385-a380-d8ebc570c0e5", "papermill": { "duration": 0.036957, "end_time": "2024-05-13T05:19:01.660047", "exception": false, "start_time": "2024-05-13T05:19:01.623090", "status": "completed" }, "showInput": false, "tags": [] }, "source": [ "## 4. Define how to evaluate trials\n" ] }, { "cell_type": "markdown", "id": "e4056ff7", "metadata": { "customInput": null, "originalKey": "c7630dfd-548b-408a-badf-b6abf79275e2", "papermill": { "duration": 0.037172, "end_time": "2024-05-13T05:19:01.734135", "exception": false, "start_time": "2024-05-13T05:19:01.696963", "status": "completed" }, "showInput": false, "tags": [] }, "source": [ "First we define a simple CNN class to classify the MNIST images" ] }, { "cell_type": "code", "execution_count": 6, "id": "3a239db6", "metadata": { "customInput": null, "customOutput": null, "execution": { "iopub.execute_input": "2024-05-13T05:19:01.810557Z", "iopub.status.busy": "2024-05-13T05:19:01.810017Z", "iopub.status.idle": "2024-05-13T05:19:01.815449Z", "shell.execute_reply": "2024-05-13T05:19:01.814788Z" }, "executionStartTime": 1690415267282, "executionStopTime": 1690415267286, "originalKey": "e41fea0a-ae71-4e6f-8c0a-6eb6ae143fb0", "papermill": { "duration": 0.044872, "end_time": "2024-05-13T05:19:01.816811", "exception": false, "start_time": "2024-05-13T05:19:01.771939", "status": "completed" }, "requestMsgId": "60f14ec9-eb1b-4e88-95c5-15c91f999c90", "showInput": true, "tags": [] }, "outputs": [], "source": [ "class CNN(nn.Module):\n", " \n", " def __init__(self) -> None:\n", " super().__init__()\n", " self.conv1 = nn.Conv2d(1, 20, kernel_size=5, stride=1)\n", " self.fc1 = nn.Linear(8 * 8 * 20, 64)\n", " self.fc2 = nn.Linear(64, 10)\n", "\n", " def forward(self, x: Tensor) -> Tensor:\n", " x = F.relu(self.conv1(x))\n", " x = F.max_pool2d(x, 3, 3)\n", " x = x.view(-1, 8 * 8 * 20)\n", " x = F.relu(self.fc1(x))\n", " x = self.fc2(x)\n", " return F.log_softmax(x, dim=-1)" ] }, { "cell_type": "markdown", "id": "99f51af6", "metadata": { "customInput": null, "originalKey": "8ef6bcb9-c492-4874-b8c7-a07f7e6291ad", "papermill": { "duration": 0.036681, "end_time": "2024-05-13T05:19:01.890660", "exception": false, "start_time": "2024-05-13T05:19:01.853979", "status": "completed" }, "showInput": false, "tags": [] }, "source": [ "In this tutorial, we want to optimize classification accuracy on the validation set as a function of the learning rate and momentum. The `train_evaluate` function takes in a parameterization (set of parameter values), computes the classification accuracy, and returns that metric. " ] }, { "cell_type": "code", "execution_count": 7, "id": "b9ce5a06", "metadata": { "customInput": null, "customOutput": null, "execution": { "iopub.execute_input": "2024-05-13T05:19:01.966535Z", "iopub.status.busy": "2024-05-13T05:19:01.966008Z", "iopub.status.idle": "2024-05-13T05:19:01.970185Z", "shell.execute_reply": "2024-05-13T05:19:01.969552Z" }, "executionStartTime": 1690415267388, "executionStopTime": 1690415267395, "originalKey": "a7e4bcc4-7494-429b-bb93-7ad84d0985af", "papermill": { "duration": 0.043923, "end_time": "2024-05-13T05:19:01.971539", "exception": false, "start_time": "2024-05-13T05:19:01.927616", "status": "completed" }, "requestMsgId": "5d486dbf-60cb-453d-8f24-8605f974b0a7", "showInput": true, "tags": [] }, "outputs": [], "source": [ "def train_evaluate(parameterization):\n", " \"\"\"\n", " Train the model and then compute an evaluation metric.\n", "\n", " In this tutorial, the CNN utils package is doing a lot of work\n", " under the hood:\n", " - `train` initializes the network, defines the loss function\n", " and optimizer, performs the training loop, and returns the\n", " trained model.\n", " - `evaluate` computes the accuracy of the model on the\n", " evaluation dataset and returns the metric.\n", "\n", " For your use case, you can define training and evaluation functions\n", " of your choosing.\n", "\n", " \"\"\"\n", " net = CNN()\n", " net = train(\n", " net=net,\n", " train_loader=train_loader,\n", " parameters=parameterization,\n", " dtype=dtype,\n", " device=device,\n", " )\n", "\n", " return evaluate(\n", " net=net, \n", " data_loader=valid_loader, \n", " dtype=dtype, \n", " device=device,\n", " )\n" ] }, { "cell_type": "markdown", "id": "36ac195f", "metadata": { "customInput": null, "originalKey": "9ab127a8-021f-4ec8-9f4e-f4256a2e322a", "papermill": { "duration": 0.037273, "end_time": "2024-05-13T05:19:02.045963", "exception": false, "start_time": "2024-05-13T05:19:02.008690", "status": "completed" }, "showInput": false, "tags": [] }, "source": [ "## 5. Run optimization loop\n" ] }, { "cell_type": "markdown", "id": "3975644e", "metadata": { "customInput": null, "originalKey": "411a2fb4-e8a3-4414-bc17-09f0b5ba3e74", "papermill": { "duration": 0.037241, "end_time": "2024-05-13T05:19:02.120455", "exception": false, "start_time": "2024-05-13T05:19:02.083214", "status": "completed" }, "showInput": false, "tags": [] }, "source": [ "First we use `attach_trial` to attach a custom trial with manually-chosen parameters. This step is optional, but we include it here to demonstrate adding manual trials and to serve as a baseline model with decent performance. " ] }, { "cell_type": "code", "execution_count": 8, "id": "7ad897b8", "metadata": { "customInput": null, "execution": { "iopub.execute_input": "2024-05-13T05:19:02.196173Z", "iopub.status.busy": "2024-05-13T05:19:02.195892Z", "iopub.status.idle": "2024-05-13T05:19:09.001699Z", "shell.execute_reply": "2024-05-13T05:19:09.001062Z" }, "executionStartTime": 1690415267533, "executionStopTime": 1690415287786, "originalKey": "1388ef55-5642-46ab-b297-c76a73a48aca", "papermill": { "duration": 6.845294, "end_time": "2024-05-13T05:19:09.003062", "exception": false, "start_time": "2024-05-13T05:19:02.157768", "status": "completed" }, "requestMsgId": "b32a4981-ad59-46e1-b701-fa5a5f118d8b", "showInput": true, "tags": [] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:19:02] ax.core.experiment: Attached custom parameterizations [{'lr': 2.6e-05, 'momentum': 0.58}] as trial 0.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:19:08] ax.service.ax_client: Completed trial 0 with data: {'accuracy': (0.841833, None)}.\n" ] } ], "source": [ "# Attach the trial\n", "ax_client.attach_trial(\n", " parameters={\"lr\": 0.000026, \"momentum\": 0.58}\n", ")\n", "\n", "# Get the parameters and run the trial \n", "baseline_parameters = ax_client.get_trial_parameters(trial_index=0)\n", "ax_client.complete_trial(trial_index=0, raw_data=train_evaluate(baseline_parameters))" ] }, { "cell_type": "markdown", "id": "7081f3af", "metadata": { "customInput": null, "originalKey": "f0f886a1-c5c8-44bb-b2fd-9fa3f140357a", "papermill": { "duration": 0.037163, "end_time": "2024-05-13T05:19:09.077585", "exception": false, "start_time": "2024-05-13T05:19:09.040422", "status": "completed" }, "showInput": false, "tags": [] }, "source": [ "Now we start the optimization loop.\n", "\n", "At each step, the user queries the client for a new trial then submits the evaluation of that trial back to the client.\n", "\n", "Note that Ax auto-selects an appropriate optimization algorithm based on the search space. For more advanced use cases that require a specific optimization algorithm, pass a `generation_strategy` argument into the `AxClient` constructor. Note that when Bayesian Optimization is used, generating new trials may take a few minutes." ] }, { "cell_type": "code", "execution_count": 9, "id": "8561f3a0", "metadata": { "customInput": null, "customOutput": null, "execution": { "iopub.execute_input": "2024-05-13T05:19:09.153556Z", "iopub.status.busy": "2024-05-13T05:19:09.152960Z", "iopub.status.idle": "2024-05-13T05:22:11.110448Z", "shell.execute_reply": "2024-05-13T05:22:11.109846Z" }, "executionStartTime": 1690415287908, "executionStopTime": 1690415945107, "originalKey": "bff5d714-1ab3-43d3-b9b3-8c3a53c81dcb", "papermill": { "duration": 181.99706, "end_time": "2024-05-13T05:22:11.111776", "exception": false, "start_time": "2024-05-13T05:19:09.114716", "status": "completed" }, "requestMsgId": "a203534f-85dd-4dfa-9fa6-6aa46a0200a3", "showInput": true, "tags": [] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:19:09] ax.service.ax_client: Generated new trial 1 with parameters {'lr': 0.009955, 'momentum': 0.633423} using model Sobol.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:19:15] ax.service.ax_client: Completed trial 1 with data: {'accuracy': (0.100333, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:19:15] ax.service.ax_client: Generated new trial 2 with parameters {'lr': 5e-06, 'momentum': 0.022851} using model Sobol.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:19:22] ax.service.ax_client: Completed trial 2 with data: {'accuracy': (0.318667, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:19:22] ax.service.ax_client: Generated new trial 3 with parameters {'lr': 7e-06, 'momentum': 0.176948} using model Sobol.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:19:29] ax.service.ax_client: Completed trial 3 with data: {'accuracy': (0.4585, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:19:29] ax.service.ax_client: Generated new trial 4 with parameters {'lr': 8.2e-05, 'momentum': 0.90883} using model Sobol.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:19:36] ax.service.ax_client: Completed trial 4 with data: {'accuracy': (0.926, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:19:36] ax.service.ax_client: Generated new trial 5 with parameters {'lr': 0.000302, 'momentum': 0.341904} using model Sobol.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:19:43] ax.service.ax_client: Completed trial 5 with data: {'accuracy': (0.929, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:19:43] ax.service.ax_client: Generated new trial 6 with parameters {'lr': 0.000137, 'momentum': 0.590917} using model BoTorch.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:19:50] ax.service.ax_client: Completed trial 6 with data: {'accuracy': (0.92, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:19:50] ax.service.ax_client: Generated new trial 7 with parameters {'lr': 1e-05, 'momentum': 1.0} using model BoTorch.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:19:57] ax.service.ax_client: Completed trial 7 with data: {'accuracy': (0.860167, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:19:57] ax.service.ax_client: Generated new trial 8 with parameters {'lr': 0.000246, 'momentum': 0.0} using model BoTorch.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:20:04] ax.service.ax_client: Completed trial 8 with data: {'accuracy': (0.888833, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:20:05] ax.service.ax_client: Generated new trial 9 with parameters {'lr': 0.000149, 'momentum': 0.286357} using model BoTorch.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:20:11] ax.service.ax_client: Completed trial 9 with data: {'accuracy': (0.901667, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:20:12] ax.service.ax_client: Generated new trial 10 with parameters {'lr': 1e-06, 'momentum': 1.0} using model BoTorch.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:20:19] ax.service.ax_client: Completed trial 10 with data: {'accuracy': (0.560333, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:20:19] ax.service.ax_client: Generated new trial 11 with parameters {'lr': 3.7e-05, 'momentum': 1.0} using model BoTorch.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:20:26] ax.service.ax_client: Completed trial 11 with data: {'accuracy': (0.759167, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:20:27] ax.service.ax_client: Generated new trial 12 with parameters {'lr': 0.000261, 'momentum': 0.91355} using model BoTorch.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:20:34] ax.service.ax_client: Completed trial 12 with data: {'accuracy': (0.945333, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:20:35] ax.service.ax_client: Generated new trial 13 with parameters {'lr': 0.00017, 'momentum': 0.799113} using model BoTorch.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:20:41] ax.service.ax_client: Completed trial 13 with data: {'accuracy': (0.942833, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:20:42] ax.service.ax_client: Generated new trial 14 with parameters {'lr': 0.000159, 'momentum': 1.0} using model BoTorch.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:20:49] ax.service.ax_client: Completed trial 14 with data: {'accuracy': (0.842833, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:20:49] ax.service.ax_client: Generated new trial 15 with parameters {'lr': 0.000311, 'momentum': 0.665002} using model BoTorch.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:20:56] ax.service.ax_client: Completed trial 15 with data: {'accuracy': (0.945833, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:20:57] ax.service.ax_client: Generated new trial 16 with parameters {'lr': 0.000837, 'momentum': 0.0} using model BoTorch.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:21:04] ax.service.ax_client: Completed trial 16 with data: {'accuracy': (0.870333, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:21:05] ax.service.ax_client: Generated new trial 17 with parameters {'lr': 0.000573, 'momentum': 0.790425} using model BoTorch.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:21:11] ax.service.ax_client: Completed trial 17 with data: {'accuracy': (0.943833, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:21:12] ax.service.ax_client: Generated new trial 18 with parameters {'lr': 0.000564, 'momentum': 0.479924} using model BoTorch.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:21:19] ax.service.ax_client: Completed trial 18 with data: {'accuracy': (0.912333, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:21:20] ax.modelbridge.base: Untransformed parameter 0.40000000000000013 greater than upper bound 0.4, clamping\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:21:20] ax.service.ax_client: Generated new trial 19 with parameters {'lr': 0.4, 'momentum': 0.0} using model BoTorch.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:21:26] ax.service.ax_client: Completed trial 19 with data: {'accuracy': (0.101, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:21:27] ax.service.ax_client: Generated new trial 20 with parameters {'lr': 0.000315, 'momentum': 0.777594} using model BoTorch.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:21:34] ax.service.ax_client: Completed trial 20 with data: {'accuracy': (0.203667, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:21:34] ax.service.ax_client: Generated new trial 21 with parameters {'lr': 3.5e-05, 'momentum': 0.76736} using model BoTorch.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:21:41] ax.service.ax_client: Completed trial 21 with data: {'accuracy': (0.881833, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:21:42] ax.service.ax_client: Generated new trial 22 with parameters {'lr': 0.0001, 'momentum': 0.0} using model BoTorch.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:21:49] ax.service.ax_client: Completed trial 22 with data: {'accuracy': (0.873167, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:21:49] ax.service.ax_client: Generated new trial 23 with parameters {'lr': 7.6e-05, 'momentum': 0.471701} using model BoTorch.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:21:56] ax.service.ax_client: Completed trial 23 with data: {'accuracy': (0.9015, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:21:57] ax.service.ax_client: Generated new trial 24 with parameters {'lr': 0.000353, 'momentum': 0.183083} using model BoTorch.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:22:03] ax.service.ax_client: Completed trial 24 with data: {'accuracy': (0.913333, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:22:04] ax.service.ax_client: Generated new trial 25 with parameters {'lr': 0.001072, 'momentum': 1.0} using model BoTorch.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:22:11] ax.service.ax_client: Completed trial 25 with data: {'accuracy': (0.102667, None)}.\n" ] } ], "source": [ "for i in range(25):\n", " parameters, trial_index = ax_client.get_next_trial()\n", " # Local evaluation here can be replaced with deployment to external system.\n", " ax_client.complete_trial(trial_index=trial_index, raw_data=train_evaluate(parameters))" ] }, { "cell_type": "markdown", "id": "5b471342", "metadata": { "customInput": null, "originalKey": "ccd16059-db9f-475b-b527-75afb320e0f4", "papermill": { "duration": 0.042324, "end_time": "2024-05-13T05:22:11.193730", "exception": false, "start_time": "2024-05-13T05:22:11.151406", "status": "completed" }, "showInput": false, "tags": [] }, "source": [ "### How many trials can run in parallel?\n", "By default, Ax restricts number of trials that can run in parallel for some optimization stages, in order to improve the optimization performance and reduce the number of trials that the optimization will require. To check the maximum parallelism for each optimization stage:" ] }, { "cell_type": "code", "execution_count": 10, "id": "a978f914", "metadata": { "customInput": null, "customOutput": null, "execution": { "iopub.execute_input": "2024-05-13T05:22:11.274543Z", "iopub.status.busy": "2024-05-13T05:22:11.273945Z", "iopub.status.idle": "2024-05-13T05:22:11.280828Z", "shell.execute_reply": "2024-05-13T05:22:11.280254Z" }, "executionStartTime": 1690415945269, "executionStopTime": 1690415945336, "originalKey": "7182d2f9-912c-464c-b5ad-f65ce6f00017", "papermill": { "duration": 0.048398, "end_time": "2024-05-13T05:22:11.282131", "exception": false, "start_time": "2024-05-13T05:22:11.233733", "status": "completed" }, "requestMsgId": "4cb4ff79-e45b-4c7d-86a1-7f8007eb2c81", "showInput": true, "tags": [] }, "outputs": [ { "data": { "text/plain": [ "[(5, 5), (-1, 3)]" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "ax_client.get_max_parallelism()" ] }, { "cell_type": "markdown", "id": "841b43b5", "metadata": { "customInput": null, "originalKey": "e2f429e6-2ec8-4af2-906b-52a36a53d329", "papermill": { "duration": 0.03919, "end_time": "2024-05-13T05:22:11.360776", "exception": false, "start_time": "2024-05-13T05:22:11.321586", "status": "completed" }, "showInput": false, "tags": [] }, "source": [ "The output of this function is a list of tuples of form (number of trials, max parallelism), so the example above means \"the max parallelism is 5 for the first 5 trials and 3 for all subsequent trials.\" This is because the first 5 trials are produced quasi-randomly and can all be evaluated at once, and subsequent trials are produced via Bayesian optimization, which converges on optimal point in fewer trials when parallelism is limited. MaxParallelismReachedException indicates that the parallelism limit has been reached –– refer to the 'Service API Exceptions Meaning and Handling' section at the end of the tutorial for handling.\n", "\n" ] }, { "cell_type": "markdown", "id": "ce0a0382", "metadata": { "customInput": null, "originalKey": "86c7aef9-993a-411e-add5-05839b00d3cf", "papermill": { "duration": 0.038988, "end_time": "2024-05-13T05:22:11.438979", "exception": false, "start_time": "2024-05-13T05:22:11.399991", "status": "completed" }, "showInput": false, "tags": [] }, "source": [ "### How to view all existing trials during optimization?" ] }, { "cell_type": "code", "execution_count": 11, "id": "9c0fd562", "metadata": { "customInput": null, "execution": { "iopub.execute_input": "2024-05-13T05:22:11.518768Z", "iopub.status.busy": "2024-05-13T05:22:11.518476Z", "iopub.status.idle": "2024-05-13T05:22:11.548574Z", "shell.execute_reply": "2024-05-13T05:22:11.548006Z" }, "executionStartTime": 1690415945532, "executionStopTime": 1690415946199, "originalKey": "3fbad5dc-863a-494e-b04f-d7dc1e47936c", "papermill": { "duration": 0.071769, "end_time": "2024-05-13T05:22:11.549773", "exception": false, "start_time": "2024-05-13T05:22:11.478004", "status": "completed" }, "requestMsgId": "905ea8b6-add0-473e-8516-5be6ad7d7658", "showInput": true, "tags": [] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "[WARNING 05-13 05:22:11] ax.service.utils.report_utils: Column reason missing for all trials. Not appending column.\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
trial_indexarm_nametrial_statusgeneration_methodaccuracylrmomentum
000_0COMPLETEDManual0.8418330.0000260.580000
111_0COMPLETEDSobol0.1003330.0099550.633423
222_0COMPLETEDSobol0.3186670.0000050.022851
333_0COMPLETEDSobol0.4585000.0000070.176948
444_0COMPLETEDSobol0.9260000.0000820.908830
555_0COMPLETEDSobol0.9290000.0003020.341904
666_0COMPLETEDBoTorch0.9200000.0001370.590917
777_0COMPLETEDBoTorch0.8601670.0000101.000000
888_0COMPLETEDBoTorch0.8888330.0002460.000000
999_0COMPLETEDBoTorch0.9016670.0001490.286357
101010_0COMPLETEDBoTorch0.5603330.0000011.000000
111111_0COMPLETEDBoTorch0.7591670.0000371.000000
121212_0COMPLETEDBoTorch0.9453330.0002610.913550
131313_0COMPLETEDBoTorch0.9428330.0001700.799113
141414_0COMPLETEDBoTorch0.8428330.0001591.000000
151515_0COMPLETEDBoTorch0.9458330.0003110.665002
161616_0COMPLETEDBoTorch0.8703330.0008370.000000
171717_0COMPLETEDBoTorch0.9438330.0005730.790425
181818_0COMPLETEDBoTorch0.9123330.0005640.479924
191919_0COMPLETEDBoTorch0.1010000.4000000.000000
202020_0COMPLETEDBoTorch0.2036670.0003150.777594
212121_0COMPLETEDBoTorch0.8818330.0000350.767360
222222_0COMPLETEDBoTorch0.8731670.0001000.000000
232323_0COMPLETEDBoTorch0.9015000.0000760.471701
242424_0COMPLETEDBoTorch0.9133330.0003530.183083
252525_0COMPLETEDBoTorch0.1026670.0010721.000000
\n", "
" ], "text/plain": [ " trial_index arm_name trial_status generation_method accuracy lr \\\n", "0 0 0_0 COMPLETED Manual 0.841833 0.000026 \n", "1 1 1_0 COMPLETED Sobol 0.100333 0.009955 \n", "2 2 2_0 COMPLETED Sobol 0.318667 0.000005 \n", "3 3 3_0 COMPLETED Sobol 0.458500 0.000007 \n", "4 4 4_0 COMPLETED Sobol 0.926000 0.000082 \n", "5 5 5_0 COMPLETED Sobol 0.929000 0.000302 \n", "6 6 6_0 COMPLETED BoTorch 0.920000 0.000137 \n", "7 7 7_0 COMPLETED BoTorch 0.860167 0.000010 \n", "8 8 8_0 COMPLETED BoTorch 0.888833 0.000246 \n", "9 9 9_0 COMPLETED BoTorch 0.901667 0.000149 \n", "10 10 10_0 COMPLETED BoTorch 0.560333 0.000001 \n", "11 11 11_0 COMPLETED BoTorch 0.759167 0.000037 \n", "12 12 12_0 COMPLETED BoTorch 0.945333 0.000261 \n", "13 13 13_0 COMPLETED BoTorch 0.942833 0.000170 \n", "14 14 14_0 COMPLETED BoTorch 0.842833 0.000159 \n", "15 15 15_0 COMPLETED BoTorch 0.945833 0.000311 \n", "16 16 16_0 COMPLETED BoTorch 0.870333 0.000837 \n", "17 17 17_0 COMPLETED BoTorch 0.943833 0.000573 \n", "18 18 18_0 COMPLETED BoTorch 0.912333 0.000564 \n", "19 19 19_0 COMPLETED BoTorch 0.101000 0.400000 \n", "20 20 20_0 COMPLETED BoTorch 0.203667 0.000315 \n", "21 21 21_0 COMPLETED BoTorch 0.881833 0.000035 \n", "22 22 22_0 COMPLETED BoTorch 0.873167 0.000100 \n", "23 23 23_0 COMPLETED BoTorch 0.901500 0.000076 \n", "24 24 24_0 COMPLETED BoTorch 0.913333 0.000353 \n", "25 25 25_0 COMPLETED BoTorch 0.102667 0.001072 \n", "\n", " momentum \n", "0 0.580000 \n", "1 0.633423 \n", "2 0.022851 \n", "3 0.176948 \n", "4 0.908830 \n", "5 0.341904 \n", "6 0.590917 \n", "7 1.000000 \n", "8 0.000000 \n", "9 0.286357 \n", "10 1.000000 \n", "11 1.000000 \n", "12 0.913550 \n", "13 0.799113 \n", "14 1.000000 \n", "15 0.665002 \n", "16 0.000000 \n", "17 0.790425 \n", "18 0.479924 \n", "19 0.000000 \n", "20 0.777594 \n", "21 0.767360 \n", "22 0.000000 \n", "23 0.471701 \n", "24 0.183083 \n", "25 1.000000 " ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "ax_client.get_trials_data_frame()" ] }, { "cell_type": "markdown", "id": "0a3e3a09", "metadata": { "customInput": null, "originalKey": "9f1ebc55-e6f2-498f-9185-569227c2f3d5", "papermill": { "duration": 0.039477, "end_time": "2024-05-13T05:22:11.629040", "exception": false, "start_time": "2024-05-13T05:22:11.589563", "status": "completed" }, "showInput": false, "tags": [] }, "source": [ "## 6. Retrieve best parameters\n", "\n", "Once it's complete, we can access the best parameters found, as well as the corresponding metric values. Note that these parameters may not necessarily be the set that yielded the highest _observed_ accuracy because Ax uses the highest model _predicted_ accuracy to choose the best parameters (see [here](https://ax.dev/api/service.html#module-ax.service.utils.best_point_mixin) for more details). Due to randomness in the data or the algorithm itself, using observed accuracy may result in choosing an outlier for the best set of parameters. Using the model predicted best will use the model to regularize the observations and reduce the likelihood of picking some outlier in the data." ] }, { "cell_type": "code", "execution_count": 12, "id": "61b0ed92", "metadata": { "customInput": null, "customOutput": null, "execution": { "iopub.execute_input": "2024-05-13T05:22:11.709872Z", "iopub.status.busy": "2024-05-13T05:22:11.709239Z", "iopub.status.idle": "2024-05-13T05:22:12.005549Z", "shell.execute_reply": "2024-05-13T05:22:12.004855Z" }, "executionStartTime": 1690415946312, "executionStopTime": 1690415949198, "originalKey": "8fdf0023-2bf5-4cdd-93ea-a8a708dc6845", "papermill": { "duration": 0.33851, "end_time": "2024-05-13T05:22:12.007170", "exception": false, "start_time": "2024-05-13T05:22:11.668660", "status": "completed" }, "requestMsgId": "c0b8c25d-c6ae-476e-be23-f1b963df296b", "showInput": true, "tags": [] }, "outputs": [ { "data": { "text/plain": [ "{'lr': 0.0001494385653033887, 'momentum': 0.2863569208604894}" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "best_parameters, values = ax_client.get_best_parameters()\n", "best_parameters" ] }, { "cell_type": "code", "execution_count": 13, "id": "03c51faa", "metadata": { "customInput": null, "customOutput": null, "execution": { "iopub.execute_input": "2024-05-13T05:22:12.091082Z", "iopub.status.busy": "2024-05-13T05:22:12.090573Z", "iopub.status.idle": "2024-05-13T05:22:12.095122Z", "shell.execute_reply": "2024-05-13T05:22:12.094481Z" }, "executionStartTime": 1690415949308, "executionStopTime": 1690415949313, "originalKey": "f3eb18fc-be99-494a-aeac-e9b05a3bc182", "papermill": { "duration": 0.048424, "end_time": "2024-05-13T05:22:12.096512", "exception": false, "start_time": "2024-05-13T05:22:12.048088", "status": "completed" }, "requestMsgId": "ac214ea0-ea8c-46f2-a988-b42893ef6d6d", "showInput": true, "tags": [] }, "outputs": [ { "data": { "text/plain": [ "{'accuracy': 0.9180803909368207}" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "mean, covariance = values\n", "mean" ] }, { "cell_type": "markdown", "id": "5724f013", "metadata": { "customInput": null, "originalKey": "6be3b006-d090-4c73-a64a-12901d1af817", "papermill": { "duration": 0.039848, "end_time": "2024-05-13T05:22:12.176461", "exception": false, "start_time": "2024-05-13T05:22:12.136613", "status": "completed" }, "showInput": false, "tags": [] }, "source": [ "## 7. Plot the response surface and optimization trace\n", "\n", "Contour plot showing classification accuracy as a function of the two hyperparameters.\n", "\n", "The black squares show points that we have actually run; notice how they are clustered in the optimal region." ] }, { "cell_type": "code", "execution_count": 14, "id": "581f0eef", "metadata": { "customInput": null, "customOutput": null, "execution": { "iopub.execute_input": "2024-05-13T05:22:12.259151Z", "iopub.status.busy": "2024-05-13T05:22:12.258499Z", "iopub.status.idle": "2024-05-13T05:22:12.910288Z", "shell.execute_reply": "2024-05-13T05:22:12.909547Z" }, "executionStartTime": 1690415949431, "executionStopTime": 1690415953540, "originalKey": "1beca759-2fa5-48d1-bfed-c9b13a054733", "papermill": { "duration": 0.697459, "end_time": "2024-05-13T05:22:12.914863", "exception": false, "start_time": "2024-05-13T05:22:12.217404", "status": "completed" }, "requestMsgId": "fa48963e-b43c-4079-81a4-079d347fe9ba", "showInput": true, "tags": [] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:22:12] ax.service.ax_client: Retrieving contour plot with parameter 'lr' on X-axis and 'momentum' on Y-axis, for metric 'accuracy'. Remaining parameters are affixed to the middle of their range.\n" ] }, { "data": { "application/vnd.plotly.v1+json": { "config": { "linkText": "Export to plot.ly", "plotlyServerURL": "https://plot.ly", "showLink": false }, "data": [ { "autocolorscale": false, "autocontour": true, "colorbar": { "tickfont": { "size": 8 }, "ticksuffix": "", "x": 0.45, "y": 0.5 }, "colorscale": [ [ 0.0, "rgb(247,252,253)" ], [ 0.125, "rgb(229,245,249)" ], [ 0.25, "rgb(204,236,230)" ], [ 0.375, "rgb(153,216,201)" ], [ 0.5, "rgb(102,194,164)" ], [ 0.625, "rgb(65,174,118)" ], [ 0.75, "rgb(35,139,69)" ], [ 0.875, "rgb(0,109,44)" ], [ 1.0, "rgb(0,68,27)" ] ], "contours": { "coloring": "heatmap" }, "hoverinfo": "x+y+z", "ncontours": 25, "type": "contour", "x": [ 1e-06, 1.3011511650442548e-06, 1.692994354296022e-06, 2.2028415765056147e-06, 2.866229883678204e-06, 3.729398352432554e-06, 4.852511011181743e-06, 6.3138503555892e-06, 8.215273746089953e-06, 1.0689313005882424e-05, 1.390841207112662e-05, 1.809694657026198e-05, 2.354686311364001e-05, 3.063802837345029e-05, 3.986470631277378e-05, 5.1870009063012666e-05, 6.749072272319499e-05, 8.781563250096393e-05, 0.00011426141253772724, 0.00014867137004306603, 0.00019344392634026088, 0.0002516997901283655, 0.0003274994751669172, 0.0004261263236648159, 0.0005544547624925005, 0.0007214294601814526, 0.000938688782612345, 0.0012213760031100258, 0.0015891948094037057, 0.002067782677737912, 0.0026904978401970136, 0.0035007443993213955, 0.004554997653699184, 0.005926740503884541, 0.007711585311544345, 0.010033938212454078, 0.013055670395116691, 0.01698740074503987, 0.02210317627048227, 0.028759573555516536, 0.03742055263793628, 0.04868979566145066, 0.06335278435066323, 0.0824315491666629, 0.10725590623460621, 0.13955614735503497, 0.18158364372009145, 0.23626776957937787, 0.3074200836506151, 0.4 ], "xaxis": "x", "y": [ 0.0, 0.02040816326530612, 0.04081632653061224, 0.061224489795918366, 0.08163265306122448, 0.1020408163265306, 0.12244897959183673, 0.14285714285714285, 0.16326530612244897, 0.18367346938775508, 0.2040816326530612, 0.22448979591836732, 0.24489795918367346, 0.26530612244897955, 0.2857142857142857, 0.3061224489795918, 0.32653061224489793, 0.3469387755102041, 0.36734693877551017, 0.3877551020408163, 0.4081632653061224, 0.42857142857142855, 0.44897959183673464, 0.4693877551020408, 0.4897959183673469, 0.5102040816326531, 0.5306122448979591, 0.5510204081632653, 0.5714285714285714, 0.5918367346938775, 0.6122448979591836, 0.6326530612244897, 0.6530612244897959, 0.673469387755102, 0.6938775510204082, 0.7142857142857142, 0.7346938775510203, 0.7551020408163265, 0.7755102040816326, 0.7959183673469387, 0.8163265306122448, 0.836734693877551, 0.8571428571428571, 0.8775510204081632, 0.8979591836734693, 0.9183673469387754, 0.9387755102040816, 0.9591836734693877, 0.9795918367346939, 1.0 ], "yaxis": "y", "z": [ [ 0.5144380793302575, 0.5134766674536717, 0.5146158962870273, 0.5182886634026721, 0.524931272131169, 0.5349445356211757, 0.5486336109236334, 0.5661232091109023, 0.5872716359081969, 0.6116583820237507, 0.6386404879778549, 0.6674289152672334, 0.6971591623645534, 0.7269449257414689, 0.7559154813232397, 0.7832420208764234, 0.8081591372969132, 0.8299878993963251, 0.8481674657634005, 0.8622947883639012, 0.87212843631701, 0.8775600281348641, 0.8786028804496476, 0.8753905460081159, 0.8681642635497259, 0.8572596132119799, 0.8430999129902287, 0.8261967078732225, 0.8071307698810198, 0.7865096192007762, 0.7649315428177559, 0.7429592267479259, 0.7211006333895301, 0.6997955301082854, 0.6794068083769014, 0.660216307556446, 0.6424251638578709, 0.6261586752943656, 0.6114753465157992, 0.5983793061641549, 0.5868348738288528, 0.5767818416222699, 0.5681500597334963, 0.5608721032371362, 0.5548930280126341, 0.5501763855415729, 0.5467056860733415, 0.5444803403537961, 0.5435047559786371, 0.54376870356341 ], [ 0.5141636821958955, 0.5133204203813087, 0.5145986881369045, 0.5184334724458104, 0.5252644456245796, 0.5354974569761447, 0.5494441092416035, 0.5672346923507358, 0.5887262436148721, 0.6134870148657559, 0.6408559076003765, 0.6700244119545709, 0.7001108067694174, 0.7302150302654686, 0.7594561524333288, 0.7869983980065444, 0.8120721595006981, 0.8339964984487971, 0.8522100436476401, 0.8663091248876125, 0.8760510142454605, 0.8813260918552467, 0.8821470511721231, 0.8786485480018396, 0.8710762718881228, 0.8597737798795362, 0.845175037752541, 0.827803585279239, 0.8082528352539208, 0.7871436303472938, 0.7650877853282799, 0.7426611752706088, 0.7203840190349635, 0.6987067623354329, 0.6780007438957296, 0.6585534537228458, 0.640568525641792, 0.6241705561349704, 0.609414447079506, 0.5962984114612369, 0.5847792980511105, 0.5747886574110614, 0.5662480197047015, 0.5590820911753777, 0.5532288562144867, 0.548645769679695, 0.5453112623858423, 0.5432206279754962, 0.5423750013599199, 0.5427615718104073 ], [ 0.5140764414621527, 0.5133749665627354, 0.514815661711099, 0.5188342406590991, 0.5258719748064276, 0.536337616728005, 0.5505469172429448, 0.5686338845771077, 0.5904547581423369, 0.6155675895420102, 0.6432939545760985, 0.6728070648682717, 0.7032094213436515, 0.7335886714913091, 0.7630551254687236, 0.7907675325148911, 0.8159537370510924, 0.8379327302001738, 0.8561447287228046, 0.8701863353761956, 0.8798131445820639, 0.8849139711358296, 0.8855004644555755, 0.8817075482922266, 0.8737847714034709, 0.8620835172281898, 0.8470485100273748, 0.8292148370698412, 0.8091874512261596, 0.7875996608748292, 0.7650762976499147, 0.7422062013455862, 0.7195218425474478, 0.697484483052232, 0.6764741017695957, 0.6567840002637211, 0.6386203695620263, 0.622107028504113, 0.6072950711435369, 0.5941764998972067, 0.5827003601088234, 0.5727896358150306, 0.5643572454481398, 0.5573197701336372, 0.5516078832227235, 0.5471726856980126, 0.5439872083096698, 0.5420421892367404, 0.5413348729394961, 0.5418490279367352 ], [ 0.5141790034279364, 0.5136437599518833, 0.5152712315116686, 0.5194964693254231, 0.5267604987807779, 0.5374726824796562, 0.5519503667884101, 0.5703292989398162, 0.5924652194165031, 0.6179069825077633, 0.6459598567407608, 0.6757802600153829, 0.7064565604676452, 0.7370656880490909, 0.7667106881409653, 0.794546348534476, 0.81979962789642, 0.8417913552924039, 0.8599651899962448, 0.8739189592892103, 0.8834065628348677, 0.8883149781808684, 0.8886543366880755, 0.8845590934995394, 0.8762820591250848, 0.8641821870514717, 0.8487147861227005, 0.8304259140692711, 0.8099310792821092, 0.7878751739208082, 0.764895494033881, 0.7415935903325095, 0.7185141668427938, 0.6961294399692989, 0.6748282314017414, 0.6549098258550844, 0.6365830356419742, 0.619970820833195, 0.6051202499957224, 0.592016805416274, 0.5806013856560899, 0.5707880800087747, 0.5624809081600712, 0.5555880805009297, 0.5500327334949697, 0.545759371004923, 0.542735315717609, 0.5409463239054455, 0.5403851534582416, 0.5410314331059309 ], [ 0.514472836719007, 0.5141288610068656, 0.5159681827714536, 0.5204237842216987, 0.5279345431207864, 0.538908007957528, 0.5536603755072793, 0.5723271727479502, 0.5947636146340174, 0.6205101957525783, 0.6488571646239158, 0.6789459591483861, 0.7098526533972773, 0.7406451231304049, 0.7704206830219031, 0.798331687561938, 0.8236058616185354, 0.8455676833532403, 0.8636658339301406, 0.8775003743155911, 0.8868238616887957, 0.8915212737644386, 0.8916007529589979, 0.8871956413426924, 0.878561421971542, 0.8660642094953634, 0.8501694100459828, 0.8314333460498454, 0.8104812088054725, 0.7879686072142061, 0.7645447155381011, 0.7408235104385958, 0.7173618943218888, 0.6946431750386517, 0.6730652279454123, 0.6529335023509415, 0.634459499297254, 0.6177652321241471, 0.6028935114895857, 0.5898229736424924, 0.5784860155628562, 0.568787500619987, 0.5606222682020192, 0.5538899237067698, 0.5485058548291271, 0.5444077390827965, 0.5415568967127187, 0.5399336939131683, 0.5395258556000768, 0.5403083330184892 ], [ 0.5149581848214544, 0.5148308709646462, 0.5169075824583959, 0.5216178194318091, 0.5293963804844126, 0.5406465044883095, 0.5556804085381577, 0.5746313668902749, 0.5973536787697556, 0.6233801229564916, 0.6519875358090252, 0.6823045195141672, 0.7133968551217416, 0.7443250953412393, 0.7741823844064053, 0.8021201723623896, 0.8273685619864158, 0.8492573433931347, 0.8672415799944649, 0.8809246306452979, 0.8900583648887972, 0.8945257644741864, 0.8943325869168166, 0.889610499667845, 0.8806170673038468, 0.867724965388982, 0.8514089185621582, 0.8322346862998804, 0.8108363394808775, 0.7878793778642532, 0.7640242480733912, 0.7398970388290195, 0.7160667977622945, 0.6930280578207159, 0.6711879664449945, 0.6508583284330786, 0.6322534035144013, 0.6154941618352342, 0.6006189069698801, 0.5875990854831349, 0.5763582263500072, 0.5667916338204709, 0.5587846924184033, 0.5522281811668058, 0.5470295424262442, 0.5431194139355116, 0.5404528371027324, 0.5390044109215524, 0.5387563632049391, 0.539678623105506 ], [ 0.5156340353918532, 0.5157488877427863, 0.5180887179181471, 0.5230781354982769, 0.531145930387852, 0.5426885414836427, 0.5580113884971194, 0.5772432507084133, 0.6002366952957602, 0.6265173189665736, 0.6553505322700391, 0.6858545283517703, 0.717086907990527, 0.7481026737980301, 0.7779923720287473, 0.8059080618924829, 0.8310837663143285, 0.8528560829253773, 0.8706876633165551, 0.884186278755482, 0.8931039954800872, 0.8973220038650519, 0.8968434171008247, 0.8917977556310294, 0.8824440476750097, 0.8691607121660333, 0.8524307609141972, 0.8328284495836102, 0.8109959509071986, 0.7876078758312518, 0.7633353315259856, 0.7388161808521585, 0.7146315459061395, 0.6912873147437982, 0.6692001327591925, 0.6486883606992229, 0.6299690889526142, 0.6131621380528556, 0.5983010367875506, 0.5853496796637996, 0.5742223500666488, 0.564804459672374, 0.5569716729481596, 0.5506057365039975, 0.5456059686695336, 0.5418957733910621, 0.5394236620339635, 0.5381581359643931, 0.5380755675032012, 0.5391406919084683 ], [ 0.5164981073725727, 0.5168804848857934, 0.5195090648912541, 0.5248021730539902, 0.5331806940686079, 0.5450318518518796, 0.5606515407042578, 0.5801615243736791, 0.6034112692890587, 0.6299197743511374, 0.658943440259328, 0.689592660455806, 0.7209190207666561, 0.7519737638647865, 0.781846409829404, 0.809691110316572, 0.834747260202778, 0.8563595985256798, 0.8739994684095749, 0.8872802132542317, 0.89595515418107, 0.8999041056135909, 0.8991274534855754, 0.893752210471573, 0.8840381938290154, 0.8703685155633605, 0.8532332330370677, 0.8332140537704664, 0.8109604664052041, 0.7871554494839493, 0.762480161450421, 0.7375838829088102, 0.7130597237846057, 0.6894250540469401, 0.6671062509245848, 0.6464284417692181, 0.6276116214908459, 0.6107743434176305, 0.5959450738144104, 0.5830797735442427, 0.5720830928916982, 0.5628302199197248, 0.5551868465543036, 0.5490254997331114, 0.5442372166130511, 0.5407379978089197, 0.5384696073605021, 0.5373941796945454, 0.5374819887445039, 0.5386925447539034 ], [ 0.5175468564806412, 0.5182217143381562, 0.5211642859028893, 0.5267852429319977, 0.5354957248825898, 0.5476714464832213, 0.5635961917423913, 0.58338196456663, 0.6068730585982619, 0.6335827090866089, 0.6627611262034783, 0.6935135677696672, 0.7248877716718909, 0.7559330097190062, 0.7857393373505361, 0.81346444031396, 0.8383544367289428, 0.8597633977009027, 0.8771723946718056, 0.8902015434077726, 0.8986066165911734, 0.9022666750499312, 0.9011794845071798, 0.8954693319977061, 0.8853960626076315, 0.87134619624663, 0.8538154248986038, 0.8333917693460033, 0.8107312161425827, 0.7865243863413022, 0.7614618848315479, 0.7362040395216697, 0.7113558478220867, 0.6874462861776279, 0.6649117065829317, 0.6440842249422154, 0.6251868166857746, 0.6083366382062708, 0.5935567842914944, 0.5807948814800687, 0.5699455516522541, 0.560873434318124, 0.5534340133840281, 0.5474904319651377, 0.5429253150433052, 0.539647120765126, 0.537590691216008, 0.5367115960762143, 0.536973881430077, 0.5383319096440426 ], [ 0.518775499166924, 0.5197671331900637, 0.523048259347544, 0.5290205539142339, 0.5380836397931488, 0.5505995629951991, 0.566837569049409, 0.5868970887748718, 0.6106144844287076, 0.6374984168510426, 0.6667959448703255, 0.6976098096709814, 0.7289860413014477, 0.7599737191953, 0.7896649800496242, 0.817222437647793, 0.8419001838157351, 0.8630626914988935, 0.8802017534265953, 0.8929454939418199, 0.9010534528377713, 0.9044047609102812, 0.902994853006787, 0.8969452314945232, 0.8865149001332236, 0.8720922895429706, 0.8541771795092018, 0.8333626779071351, 0.8103104027352848, 0.7857178914320093, 0.7602845912060767, 0.7346814950752796, 0.7095253757041753, 0.6853569393612247, 0.6626227660394436, 0.6416621948775698, 0.622701260540024, 0.6058555798976022, 0.5911425452658345, 0.5785010299142416, 0.5678152273732253, 0.5589389145236247, 0.5517171540087985, 0.5460035691479136, 0.5416722730232597, 0.5386240786479106, 0.5367867816101171, 0.5361092658365132, 0.5365493237379031, 0.5380563271156091 ], [ 0.5201780546739836, 0.5215098539073104, 0.5251531388760788, 0.5314992790559065, 0.5409346797745258, 0.5538056819965169, 0.5703646925983443, 0.5906957941739822, 0.6146245267204042, 0.6416562040229753, 0.6710377154509295, 0.7018718309736562, 0.7332049799844309, 0.764087815199868, 0.7936160832217115, 0.8209586712177919, 0.845378800929098, 0.8662523170533565, 0.8830826956660871, 0.8955073363562626, 0.9032909699299062, 0.9063138221539015, 0.9045694632609145, 0.8981766592359742, 0.8873926159665404, 0.8726060161271549, 0.854319062262456, 0.833128639450028, 0.8097010709233162, 0.7847400649082187, 0.7589533000773302, 0.7330220405137541, 0.707574710876383, 0.6831638699692155, 0.6602465904411967, 0.6391696836974408, 0.6201623259038896, 0.6033384384711762, 0.5887093577883029, 0.5762047683048079, 0.5656980349081447, 0.5570317745174005, 0.5500404435789935, 0.544568043401513, 0.5404801120124164, 0.537669756648776, 0.5360576572418088, 0.5355859682105553, 0.5362062923492182, 0.5378632256668491 ], [ 0.5217474043700101, 0.5234416169276319, 0.5274694418635528, 0.5342106604828734, 0.5440368249614111, 0.5572766368459412, 0.5741634588863882, 0.5947633696434358, 0.6188887808368371, 0.6460424602918337, 0.6754737756452811, 0.7062879916460116, 0.7375340123471419, 0.7682658165654456, 0.7975842725894812, 0.8246658407699824, 0.8487839450443119, 0.8693266890669379, 0.8858101692344522, 0.897882349766157, 0.9053146760621006, 0.9079897004623876, 0.9058997802975586, 0.8991609926625558, 0.8880277627605264, 0.8728872619396728, 0.8542423395628385, 0.8326922678307026, 0.8089070818549816, 0.7835958797765722, 0.7574739451186323, 0.7312324050276611, 0.7055112013728948, 0.6808748671908513, 0.6577912444655554, 0.6366148818198782, 0.6175781837472845, 0.6007932065997437, 0.5862648549601654, 0.5739131749220499, 0.5636003076513896, 0.5551574365264104, 0.548408261957845, 0.5431871006172113, 0.5393508939319389, 0.5367850292498033, 0.5354030598721524, 0.5351404407622855, 0.5359427239962966, 0.5377499842165527 ], [ 0.5234753671173082, 0.5255528838962091, 0.5299861647943157, 0.537142150728276, 0.5473759649161588, 0.5609968226719588, 0.5782169387970622, 0.5990820311935225, 0.6233898173529848, 0.6506408665058007, 0.680089114195155, 0.710844648257158, 0.7419608797081881, 0.7724968496869287, 0.8015600431279675, 0.8283357530058332, 0.85210860523557, 0.8722797791318845, 0.8883789039151205, 0.9000658100830001, 0.9071202673806928, 0.9094285958369577, 0.9069827759981589, 0.8998962003217572, 0.888419519760654, 0.8729365667662494, 0.8539489661207603, 0.8320569137222404, 0.8079330918477211, 0.7822911599364163, 0.7558533551957828, 0.7293202424582758, 0.7033431324674303, 0.6784986513617328, 0.6552656987892461, 0.6340068427276748, 0.6149578084413772, 0.598228603802004, 0.5838173038280644, 0.5716338564804129, 0.5615287962984721, 0.5533216314118053, 0.5468251987696273, 0.541864113149803, 0.5382867438657345, 0.5359707938976386, 0.5348227374623153, 0.5347714277092056, 0.5357565650181286, 0.5377139829367406 ], [ 0.5253527890627663, 0.5278329492517133, 0.5326909224283473, 0.540279586462513, 0.5509361177073079, 0.564948487593956, 0.582505810786559, 0.6036315275229143, 0.6281076880167465, 0.6554327088975054, 0.6848665718466223, 0.7155262830238109, 0.7464717187803359, 0.7767686907958175, 0.8055327764215252, 0.8319593258606268, 0.8553451049357556, 0.8751051218116338, 0.8907834227979184, 0.9020530063815579, 0.9087036400685475, 0.9106270568081156, 0.9078158801794969, 0.9003808071267033, 0.8885676847220023, 0.872755122266895, 0.853441580743459, 0.831226654471365, 0.8067845350848522, 0.7808325581580953, 0.754099231785208, 0.7272941118214069, 0.7010797124036542, 0.676044865128601, 0.6526798254718191, 0.6313554807611155, 0.6123109760812915, 0.5956540735190491, 0.5813756000364119, 0.569374940509257, 0.5594906606052519, 0.551530392535955, 0.5452960524015479, 0.5406025866280368, 0.53728986642174, 0.5352279970695455, 0.5343164778854854, 0.534477717490726, 0.5356458101382446, 0.5377526436916147 ], [ 0.5273696459302495, 0.5302700673989342, 0.5355701067414035, 0.5436073884458266, 0.5546996860641864, 0.5691120753016954, 0.5870088320689272, 0.6083896775198256, 0.6330204365200861, 0.6603972513646483, 0.6897870932346825, 0.7203156729285466, 0.7510511732651688, 0.7810678374238811, 0.8094907858336134, 0.8355266200650802, 0.8584851307210749, 0.8777958465365183, 0.8930180786465917, 0.9038392829585417, 0.9100609323069351, 0.9115820033884445, 0.9083969810447949, 0.9006138928836702, 0.888472682048052, 0.8723447810970082, 0.8527235107963778, 0.830206290346738, 0.8054676094319979, 0.7792275331775012, 0.7522201219444241, 0.7251634510255887, 0.6987310502130515, 0.6735240564381095, 0.6500443852351214, 0.6286715608927193, 0.6096482547620606, 0.5930797719663374, 0.578949254053128, 0.5671450593137574, 0.5574934531011329, 0.549790042185643, 0.5438258221321585, 0.5394061601228346, 0.5363625550984746, 0.5345576523602582, 0.5338841334217996, 0.5342581704562449, 0.535608531600062, 0.5378634612180402 ], [ 0.5295151556649258, 0.5328515923528565, 0.5386090609925361, 0.5471087801704759, 0.55864773619959, 0.5734665871348683, 0.5917032902918945, 0.6133328429665759, 0.6381045647999651, 0.665512122870566, 0.6948300096288927, 0.7251940893428774, 0.7556825335283283, 0.7853796065602592, 0.8134213880010861, 0.8390268966662835, 0.8615197863056653, 0.880344734446821, 0.8950771152192987, 0.9054201048640834, 0.9111886007898554, 0.9122907968710428, 0.9087244814153904, 0.9005951358158691, 0.8881355930408745, 0.8717080787129579, 0.8517987856741714, 0.8290013466965996, 0.8039892643215835, 0.7774843246992521, 0.7502253856187701, 0.72293854254957, 0.6963081243762438, 0.6709476521346973, 0.6473710054557722, 0.6259666793000103, 0.6069809855896751, 0.5905165474925127, 0.5765483676903712, 0.5649533243363944, 0.5555450937390061, 0.5481071697240368, 0.542419692717166, 0.5382785991373362, 0.5355071942969485, 0.5339608505531548, 0.5335256365024644, 0.5341117375969258, 0.5356428997200144, 0.5380440260960223 ], [ 0.531777899136911, 0.5355641265402915, 0.5417922639541976, 0.550766017184982, 0.5627602850925151, 0.577989939038906, 0.596565414182369, 0.6184363437848412, 0.6433354475531462, 0.6707536923663829, 0.6999733343125735, 0.7301415180818535, 0.7603478987475982, 0.7896882562611263, 0.8173109986188464, 0.8424486989030144, 0.8644396702372883, 0.8827442990763047, 0.8969547547884359, 0.9067911512120298, 0.9120835338089379, 0.9127513630596892, 0.908797412408491, 0.9003248998011694, 0.887558209325083, 0.8708482686779928, 0.8506721596037479, 0.8276180814410339, 0.802357189391973, 0.7756119247658623, 0.7481251557531843, 0.7206304695695148, 0.693822740841725, 0.6683279207447888, 0.6446721475129105, 0.623253233398942, 0.6043212530671362, 0.5879759090466943, 0.5741835995486329, 0.5628092897054671, 0.5536538345210184, 0.5464886007693723, 0.5410830109434008, 0.5372237811098349, 0.5347262538820514, 0.5334387618981942, 0.5332410073305679, 0.5340374712500073, 0.5357471958503977, 0.538292040480183 ], [ 0.5341459465599936, 0.5383936754101916, 0.5451035193663153, 0.5545606194636595, 0.5670165842229362, 0.5826592979600789, 0.6015707387726007, 0.6236748224481536, 0.6486876981838651, 0.6760974175275266, 0.7051940566237314, 0.7351368905944256, 0.7650283557246955, 0.7939771270196244, 0.8211452501129772, 0.8457799566694523, 0.867234975573111, 0.88498688907498, 0.8986453111043962, 0.9079484453637474, 0.9127431944605734, 0.9129623652508416, 0.9086156048767212, 0.8998043548869846, 0.8867431058813215, 0.8697693713698837, 0.849349143960155, 0.8260634970639228, 0.8005798022343951, 0.7736200436514216, 0.7459302894328661, 0.7182510608060444, 0.6912874786968382, 0.6656779228170188, 0.6419610619507383, 0.6205443798363262, 0.6016818433415376, 0.5854699822171587, 0.5718661179242545, 0.5607229037577004, 0.5518282132313566, 0.5449413568571239, 0.5398212538620635, 0.5362456733465975, 0.5340222764308111, 0.5329926310175954, 0.5330303541058722, 0.5340345286895531, 0.5359198186827338, 0.5386053274939495 ], [ 0.5366069873174524, 0.5413258046248939, 0.5485261460092069, 0.5584736001759656, 0.5713953908110809, 0.5874513897399516, 0.6066944269878456, 0.6290225634765916, 0.6541354913418599, 0.6815181625004942, 0.7104684254406953, 0.7401583185913241, 0.7697041688441377, 0.7982287990012416, 0.8249091285428555, 0.8490081117270574, 0.8698956096428831, 0.8870648103136834, 0.9001433221232845, 0.9088885017307423, 0.9131657761901792, 0.9129233990268831, 0.9081799022330205, 0.8990356110956961, 0.8856937278063323, 0.8684762355530872, 0.8478360490430019, 0.8243453557778871, 0.798666233171045, 0.7715190681797088, 0.7436523081211911, 0.7158128222170053, 0.6887156216039046, 0.663011446986309, 0.6392517297260262, 0.6178539787644834, 0.599076188631578, 0.5830114511573998, 0.5696075396439364, 0.558704447358618, 0.5500769955447375, 0.5434726052389911, 0.5386399886327815, 0.5353483035247155, 0.5333978575103518, 0.5326237650129337, 0.5328938666421265, 0.5341021694944289, 0.5361592847679484, 0.5389818351259188 ], [ 0.5391484610013871, 0.5443457968576831, 0.5520431643525149, 0.5624856855342817, 0.5758752206471368, 0.592342775514273, 0.611911551196152, 0.6344537732456068, 0.6596528472025833, 0.6869904845928351, 0.7157722169993921, 0.7451833263572841, 0.7743549763062343, 0.802425261233196, 0.8285871268307535, 0.8521202617748797, 0.872411331949459, 0.8889704642422865, 0.9014436950540978, 0.9096084735971182, 0.9133503503851633, 0.9126351495967345, 0.90749232520867, 0.898021843676491, 0.8844144859918043, 0.8669746126262856, 0.8461400348443974, 0.8224721957281186, 0.7966263044632043, 0.7693200101723102, 0.7413033250249453, 0.7133288536203014, 0.6861210729870663, 0.6603429297388763, 0.6365587876154293, 0.6151965225349268, 0.5965182959746291, 0.580613484562528, 0.5674198532364134, 0.5567644579330668, 0.5484091049741938, 0.5420895987112619, 0.5375448241505141, 0.5345357231249056, 0.5328556195098553, 0.5323335154660092, 0.5328318042028048, 0.5342397475584652, 0.5364642240807975, 0.5394196354115399 ], [ 0.5417576876497241, 0.5474388055724398, 0.5556374764339593, 0.5665775217987433, 0.5804345791274794, 0.5973100964696908, 0.6171973386105878, 0.6399428246828053, 0.6652138821713605, 0.6924888920276662, 0.7210809849839521, 0.7501890769348346, 0.7789599886182003, 0.8065480889498936, 0.8321634110764045, 0.8551033213171345, 0.8747719083465841, 0.8906964995463761, 0.9025418582102561, 0.9101062968431273, 0.9132969951397857, 0.912099484915172, 0.9065561393529433, 0.8967674034118124, 0.8829108610125715, 0.8652712443368028, 0.8442691694252624, 0.8204533448931922, 0.7944705007703717, 0.7670344426473156, 0.738895957731619, 0.7108127484247597, 0.6835182529051443, 0.6576873566820458, 0.6338974356559436, 0.6125870467491139, 0.5940226582373335, 0.5782896437045059, 0.5653153248393987, 0.5549136382812859, 0.5468335403882307, 0.5407996056516453, 0.5365413548874175, 0.5338119643699422, 0.5323981797117663, 0.532123255119165, 0.5328444784033846, 0.5344466985814252, 0.5368333714116119, 0.5399169196317946 ], [ 0.5444219953882456, 0.5505900035699394, 0.5592920363635095, 0.5707298668138308, 0.5850521689981617, 0.6023302882746314, 0.6225273842009564, 0.64546447079017, 0.6707930300928983, 0.6979880756903036, 0.7263702928759725, 0.7551525901037229, 0.7834981862965195, 0.8105786255200399, 0.8356219961739436, 0.8579441966437428, 0.8769672798266215, 0.8922359747363433, 0.9034339157512542, 0.9103808264533885, 0.9130069062258617, 0.9113195243153447, 0.9053759168515696, 0.895277926540035, 0.8811895195277574, 0.8633739657102504, 0.8422324915180159, 0.8182989277137935, 0.7922099271481708, 0.7646744214770116, 0.7364432245718348, 0.7082784749143575, 0.6809219746036428, 0.6550601429763963, 0.6312833242830287, 0.6100410213915342, 0.5916041451223624, 0.5760537703882052, 0.5633063852979218, 0.5531627495225069, 0.5453592811807755, 0.5396098307806863, 0.5356350976595587, 0.5331809914547639, 0.5320281134300566, 0.5319943500944041, 0.532932232040161, 0.5347225238530355, 0.5372655543273437, 0.5404719902129308 ], [ 0.5471288439347184, 0.5537847245156124, 0.5629900095649555, 0.5749237645465779, 0.5897070745940921, 0.6073807671449889, 0.6278778343679494, 0.6509940303145434, 0.6763652373979173, 0.7034631178657422, 0.7316159297693134, 0.7600509515378602, 0.7879485147616603, 0.8144981657516402, 0.8389469262103361, 0.8606299705318333, 0.8789877443942476, 0.8935825298310562, 0.904116800832097, 0.9104319633447167, 0.9124824965287205, 0.9102997152152891, 0.9039576286764868, 0.8935604635622079, 0.8792584508901754, 0.8612918245020416, 0.8400400701395943, 0.8160198576016161, 0.7898562505481955, 0.7622523905469747, 0.7339584237012415, 0.7057402380158758, 0.6783472989492273, 0.6524769904843192, 0.6287324186130143, 0.6075742195412046, 0.5892778716752355, 0.5739198525701844, 0.5614054970829392, 0.5515224879151334, 0.5439951806319481, 0.5385273275513874, 0.5348314223194508, 0.5326466470562795, 0.5317479131682222, 0.5319481285677901, 0.5330954147093636, 0.535066771115486, 0.537759678404891, 0.5410832499665514 ], [ 0.549865942680549, 0.5570085960840309, 0.5667149195104701, 0.5791407019586704, 0.594378923176416, 0.612439589698967, 0.633225544187627, 0.6565075483006158, 0.6819061349426476, 0.7088896806172175, 0.736794111432496, 0.7648615136925351, 0.7922900754792097, 0.8182881379834542, 0.842122454418164, 0.8631480886904918, 0.8808241498928241, 0.8947305646795463, 0.9045884196574178, 0.9102607684385124, 0.9117274880871721, 0.9090459190772151, 0.9023087535729369, 0.891623632320489, 0.8771271323387961, 0.8590352131800165, 0.837703047998044, 0.8136278068380718, 0.7874216208827824, 0.7597810691056966, 0.7314549946987794, 0.7032123212278673, 0.6758093653771373, 0.6499537191589458, 0.6262608370874677, 0.6052025609091264, 0.5870590415487392, 0.5719018653241374, 0.5596250000138607, 0.5500033458930561, 0.5427498485636677, 0.5375589035032793, 0.534135477672081, 0.532212595299727, 0.5315599448558121, 0.5319858468574046, 0.5333343560765488, 0.5354790132615532, 0.5383147104016079, 0.5417491892666035 ], [ 0.5526213623166625, 0.5602476637425843, 0.5704507812490145, 0.5833627481844961, 0.5990480244428525, 0.6174855887173218, 0.6385482106022303, 0.6619819337081655, 0.6873921886096489, 0.7142441758319544, 0.7418816684225988, 0.7695620898544661, 0.7965023135239143, 0.8219302833398892, 0.8451332172932666, 0.8654865348414342, 0.882468088123066, 0.8956754131828459, 0.9048477743836063, 0.9098695597554132, 0.9107469964231119, 0.9075654961272935, 0.9004383811926565, 0.8894777740909928, 0.8748067295740498, 0.8566159906967441, 0.8352336487650504, 0.8111351447897488, 0.7849185684565952, 0.7572733209908509, 0.7289463635479906, 0.7007089094413455, 0.6733231987192486, 0.647506070300593, 0.6238846614565846, 0.6029419271777259, 0.5849627620209539, 0.5700135849098424, 0.5579769354031608, 0.5486154584760505, 0.5416315250722672, 0.5367110203821044, 0.5335521142026647, 0.531882262527273, 0.5314664023072226, 0.5321086539092459, 0.5336493376444706, 0.5359588255946873, 0.5389296599896995, 0.5424683717182388 ], [ 0.5553836392018555, 0.5634885045352037, 0.5741822214679324, 0.5875726764357164, 0.6036954895279824, 0.6224984867810474, 0.6438244835421245, 0.667395075774883, 0.6928008300644781, 0.7195039181365094, 0.7468562225211044, 0.7741311432982195, 0.8005652029131151, 0.825406831909125, 0.8479643983960938, 0.8676339785462756, 0.8839120468509344, 0.8964134719877238, 0.904895050609938, 0.9092619878026378, 0.9095476039019593, 0.9058673772627643, 0.8983572850773166, 0.8871350575174735, 0.8723102970510251, 0.8540475338017094, 0.832645125359713, 0.8085548367316128, 0.7823598760707245, 0.7547420067498425, 0.7264457731759246, 0.698243894765637, 0.6709034924566626, 0.6451494796634856, 0.6216197148310585, 0.6008079458197341, 0.5830038274964982, 0.5682683740373763, 0.5564728492888679, 0.5473684372937315, 0.5406479478953448, 0.5359896913009892, 0.5330858054728, 0.5316587773547666, 0.5314692611066948, 0.5323175551746901, 0.534040563846913, 0.5365057623429209, 0.5396035606419385, 0.543239418832465 ], [ 0.5581418718742831, 0.5667183305079194, 0.5778945851619006, 0.5917540693483541, 0.6083033309075754, 0.6274589895797964, 0.6490340566307533, 0.672725940378903, 0.6981105684437243, 0.7246472609386875, 0.75169635161402, 0.7785479733524348, 0.8044594322458106, 0.8287006773128476, 0.8506018792090076, 0.8695798851348888, 0.8851494671755405, 0.8969422375181572, 0.9047316602697564, 0.9084430852742348, 0.9081374180504429, 0.9039621139745018, 0.8960779473468343, 0.8846094690533979, 0.8696527313185873, 0.8513446440942769, 0.8299516326503322, 0.8059002992585855, 0.7797584263214933, 0.7521998212532924, 0.7239661031956994, 0.6958306691774708, 0.6685643707176018, 0.6428988192364378, 0.6194813042988818, 0.5988157387437913, 0.5811964679819387, 0.5666789372619521, 0.555123577044387, 0.5462711958144975, 0.5398062168033645, 0.535400377670244, 0.5327405702681842, 0.5315449116043631, 0.5315702331548076, 0.5326133768638202, 0.5345081332651989, 0.5371193330792672, 0.5403354502150632, 0.544060994179397 ], [ 0.5608858092736001, 0.5699250816383257, 0.5815740292241562, 0.5958914086674311, 0.612854544633321, 0.6323488605119888, 0.6541577388627162, 0.6779546473245658, 0.7033010832440127, 0.7296537149151933, 0.7563817413766344, 0.7827928986720092, 0.8081665943241043, 0.8317955528166515, 0.8530323758960138, 0.8713145884218745, 0.8861747364849509, 0.8972602774265084, 0.9043602411305646, 0.9074192885482772, 0.9065261107674943, 0.9018619006536724, 0.8936145299949034, 0.8819166847834111, 0.8668504654165501, 0.8485233060860453, 0.8271680231905756, 0.8031852136609496, 0.7771270272810931, 0.7496591210758805, 0.7215196840086493, 0.6934819097656868, 0.6663191338834271, 0.6407681082860032, 0.6174839244604938, 0.5969796317475713, 0.5795540589156267, 0.5652570472426446, 0.5539390140133047, 0.5453317709449088, 0.539112659229422, 0.534947889007195, 0.5325198977432749, 0.5315430237539805, 0.5317707231101816, 0.5329967315294198, 0.5350520107222694, 0.5377989796565874, 0.5411243517337276, 0.5449317874513331 ], [ 0.5636059303687758, 0.5730975082910776, 0.5852076034337848, 0.5999701502578487, 0.6173331762924411, 0.6371509780547475, 0.6591775084635663, 0.6830625292622461, 0.7083532983545457, 0.7345040475352909, 0.7608933200353151, 0.7868474338289128, 0.8116693842741979, 0.834676212531982, 0.8552435599226555, 0.8728293341334692, 0.8869831615869406, 0.897367176741163, 0.9037846205511081, 0.9061984293135733, 0.9047249329748931, 0.8995805666765048, 0.8909828008440219, 0.8790738766043009, 0.8639211484283464, 0.8456003627026378, 0.8243095802046689, 0.8004233042637446, 0.7744782224425558, 0.747131747484594, 0.7191181117975642, 0.6912093643800659, 0.6641799960523683, 0.6387701977001032, 0.6156409184399956, 0.5953128203514444, 0.5780887890687276, 0.5640132460881778, 0.5529278799750819, 0.5445571478241319, 0.538572702062739, 0.5346362890038777, 0.5324266778835387, 0.5316550065399994, 0.5320717879222058, 0.5334679858854979, 0.5356720009556889, 0.5385440542142409, 0.5419692548316564, 0.5458504988245612 ], [ 0.5662935149729029, 0.5762252433240359, 0.5887833194019727, 0.6039767854625298, 0.6217243720367389, 0.6418493772810181, 0.6640765500435437, 0.6880321728719138, 0.7132494370822169, 0.7391803619319055, 0.7652133708779452, 0.7906944476769211, 0.814951807849612, 0.8373286141099265, 0.8572241567342457, 0.87411630118606, 0.8875709293203285, 0.8972634633520823, 0.9030097475238464, 0.904789693591791, 0.9027466937455559, 0.8971335370281677, 0.88820002859655, 0.8760995027769539, 0.8608833426411544, 0.8425931630219429, 0.8213917093442233, 0.7976280930923795, 0.7718240930978932, 0.744628852110048, 0.7167720719793202, 0.6890236473213315, 0.662157826665907, 0.6369164379055309, 0.6139640943955929, 0.5938269871318157, 0.5768112850821542, 0.5629565321953676, 0.5520974890161882, 0.5439530961323624, 0.5381907550070761, 0.534468811326833, 0.5324631395743108, 0.531882240280333, 0.5324741005743712, 0.5340272316913826, 0.5363677245055183, 0.5393537987570163, 0.5428690982575757, 0.5468158239655969 ], [ 0.568940705571024, 0.579298864018121, 0.5922902080699207, 0.6078988898168769, 0.6260144159731564, 0.6464292768478062, 0.66883927614637, 0.6928474429917025, 0.7179730581787088, 0.7436661537839929, 0.7693256172938104, 0.7943182848132933, 0.8179993659390955, 0.8397400681078097, 0.8589640137214192, 0.8751686049027337, 0.8879350521988696, 0.8969505033136491, 0.9020415796840303, 0.9032035353139032, 0.9006056820627284, 0.8945377598507882, 0.8852848601809393, 0.8730131020941139, 0.8577562224225025, 0.8395191990507183, 0.8184296094164951, 0.794812644656014, 0.769176062865993, 0.742160733087169, 0.7144911791763837, 0.6869340559264664, 0.6602619126608387, 0.6352163506002074, 0.6124633015756997, 0.5925318651684051, 0.5757301981060836, 0.5620940525844536, 0.5514535401070559, 0.5435240272048019, 0.5379701110115854, 0.5344477885051093, 0.5326307984130432, 0.5322255533476534, 0.5329779190431065, 0.5346742604382949, 0.5371385963761579, 0.54022732674418, 0.5438227538016767, 0.5478264399835266 ], [ 0.5715405599906729, 0.582309944002043, 0.5957183663296489, 0.6117251600832612, 0.6301907551606952, 0.6508770927741033, 0.6734513343638892, 0.6974934905441761, 0.7225090732622376, 0.7479463458490743, 0.7732152777356661, 0.797704835708966, 0.8207991217639906, 0.8418992954256497, 0.8604541331541239, 0.8759802861496295, 0.8880732990544427, 0.8964303574496089, 0.9008868844189298, 0.9014514762205259, 0.8983174981681356, 0.8918115974222437, 0.8822571885133189, 0.86983508736014, 0.8545592532765655, 0.8363957293573345, 0.81543793794383, 0.7919893154230805, 0.7665447147351538, 0.7397366895002027, 0.7122838415447341, 0.6849484190594298, 0.6584997602687611, 0.6336773372987553, 0.611145990962497, 0.5914347416593588, 0.5748517799487596, 0.5614308325708331, 0.5509999461411739, 0.543272881294188, 0.5379128689090661, 0.534574596903117, 0.5329304161247249, 0.5326851910185978, 0.5335830613261363, 0.5354085424615853, 0.5379838079440848, 0.5411636070585655, 0.5448290119441899, 0.5488809925854448 ], [ 0.5740870947085139, 0.5852510952939478, 0.599058993271498, 0.615445440496214, 0.6342420134189516, 0.655180440366959, 0.6778996013296775, 0.7019567453793857, 0.7268437465514184, 0.7520073011195797, 0.7768690923851713, 0.8008415600651128, 0.8233396692236009, 0.8437964069803496, 0.8616866786096266, 0.8765462934947802, 0.8879841155475398, 0.8957056004598482, 0.8995529289609432, 0.8995457193671466, 0.8958987591646058, 0.8889746703818298, 0.8791380133509863, 0.8665865136003484, 0.8513118235786521, 0.8332393839288551, 0.8124304849229819, 0.7891695224050663, 0.7639396307098693, 0.7373649012947961, 0.7101571562790092, 0.6830729872581266, 0.656876955341804, 0.6323044676226554, 0.6100168393899025, 0.5905399044981827, 0.5741795270487073, 0.5609695832787589, 0.5507387188251418, 0.5432010531418647, 0.5380198824949048, 0.5348496201748973, 0.5333619730464187, 0.5332607936610659, 0.5342888872120255, 0.5362292109715181, 0.5389023124876823, 0.5421614506535188, 0.5458865694716917, 0.5499780846469351 ], [ 0.5765753185116469, 0.5881160004882681, 0.6023044164516316, 0.6190507390098469, 0.6381579951198082, 0.6593281257341588, 0.6821721651097468, 0.7062248954853045, 0.7309646783874643, 0.7558368168387873, 0.7802753272940003, 0.8037174848402544, 0.825611102348683, 0.8454228776500882, 0.8626549753629589, 0.8768624694357029, 0.8876665444196711, 0.8947791172849003, 0.8980470989623842, 0.8974986383868322, 0.8933666478969012, 0.8860476164410647, 0.8759492915839531, 0.8632887652077984, 0.8480327985920941, 0.8300657496604628, 0.8094198691394006, 0.7863635446440069, 0.76136926298908, 0.735052340452479, 0.7081168413455675, 0.6813123715336571, 0.6553970965732545, 0.6311003871757945, 0.6090775781881509, 0.5898482841164289, 0.573714022786389, 0.5607106239067743, 0.5506699225076542, 0.543308361532622, 0.5382907388884327, 0.535272232789104, 0.5339246546569376, 0.5339513849104309, 0.5350942872558657, 0.5371350513526909, 0.539892814613419, 0.5432195001005476, 0.5469940192498042, 0.5511162663645989 ], [ 0.579001256120361, 0.59089943497939, 0.6054481084200336, 0.622533234212298, 0.6419296801014566, 0.663310128435217, 0.6862582977663234, 0.7102868543865065, 0.7348607745188558, 0.7594241024103023, 0.7834237633772378, 0.8063231988804775, 0.8276050175063521, 0.8467715535291642, 0.8633535278598277, 0.876925554034163, 0.8871201586200702, 0.8936539010914237, 0.8963765015226722, 0.8953221978762255, 0.8907382779417152, 0.8830516447874471, 0.8727136963861071, 0.8599630666048265, 0.8447399737418901, 0.826888947684506, 0.8064172751126563, 0.783580370235473, 0.7588408435952895, 0.7328047172689106, 0.7061672061479798, 0.6796695338843208, 0.6540618074868293, 0.630065362598379, 0.6083271150577458, 0.5893577487767767, 0.5734530571321377, 0.5606519348045965, 0.5507917021105881, 0.5435930650088613, 0.5387237672753473, 0.5358408042649403, 0.5346168525740216, 0.5347553701434677, 0.5359976792040884, 0.5381244959319367, 0.5409537637473131, 0.5443362221843173, 0.5481498422838984, 0.5522940271132858 ], [ 0.5813619612321831, 0.5935972789355661, 0.6084846935537658, 0.6258862734144047, 0.6455492108063092, 0.6671175769603352, 0.6901484201728612, 0.7141327189461364, 0.7385222035133157, 0.7627597444138499, 0.7863056765144433, 0.8086508561709028, 0.8293145509299058, 0.8478367032450822, 0.8637780725205998, 0.8767332185348377, 0.8863450206295784, 0.8923328794153993, 0.8945475933484788, 0.8930273331718155, 0.8880298648062374, 0.8800077059923765, 0.869453879458888, 0.8566297205724243, 0.8414494298000597, 0.8237212316839311, 0.8034322530403126, 0.7808275992153345, 0.7563603365775304, 0.7306264632219694, 0.7043111611971029, 0.6781458281591953, 0.652870822835387, 0.6291974489664355, 0.6077618715010589, 0.5890636366043354, 0.5733919544634514, 0.560789327574748, 0.5511003808388365, 0.5440519220165569, 0.5393160772524919, 0.5365527237572567, 0.5354361798643321, 0.5356705452059332, 0.5369970118845382, 0.5391956242624906, 0.5420833517550432, 0.5455099036159491, 0.5493524021433143, 0.5535097890875046 ], [ 0.583655518266974, 0.5962065185179022, 0.611409945000513, 0.6291043622276666, 0.6490098726936739, 0.6707427188789589, 0.6938340614976729, 0.7177537202040677, 0.7419403449626598, 0.7658356616317338, 0.7889138123006159, 0.8106941896592401, 0.8307344486893155, 0.8486141167622879, 0.863925677543916, 0.8762841371940788, 0.8853416781373041, 0.8908187917460284, 0.8925658726165009, 0.8906233409046584, 0.8852557428382071, 0.8769352375348105, 0.8661911347834024, 0.8533070633088113, 0.8381748392282652, 0.8205726578936934, 0.8004726055800483, 0.7781114084174063, 0.7539324334725789, 0.7285207494093218, 0.7025502642856016, 0.6767410858300313, 0.6518221362896875, 0.6284927411440582, 0.6073761932651058, 0.5889592346852858, 0.573523978089217, 0.5611166964092916, 0.5515906152100996, 0.5446802901993011, 0.5400636242334511, 0.5374044436718035, 0.5363794999545946, 0.5366941150070466, 0.5380897763570714, 0.540346168824164, 0.5432795146514943, 0.5467386498622858, 0.5505999417705054, 0.5547619027651159 ], [ 0.585881031881998, 0.5987252355816949, 0.6142207712533279, 0.6321831457177922, 0.6523060689061322, 0.6741788876513743, 0.697307816141365, 0.721142170337869, 0.7451077314266258, 0.7686450525151222, 0.7912423561622207, 0.8124485306041697, 0.8318611600847858, 0.8491012525403627, 0.8637948955194602, 0.8755780972424567, 0.8841112004934522, 0.8891141355675889, 0.8904356762294457, 0.8881173628224116, 0.8824273425105973, 0.873850627098695, 0.8629438227690474, 0.8500102404199628, 0.8349268332589188, 0.8174508963023727, 0.7975443795187082, 0.7754365778764282, 0.7515605889617774, 0.7264895371715084, 0.7008847984835468, 0.675453738221604, 0.6509121913855065, 0.627945666201697, 0.6071627497547722, 0.5890361984176581, 0.5738407284627138, 0.5616263100172296, 0.5522555903345919, 0.5454722569775298, 0.5409612979466017, 0.538391540169261, 0.5374429679359876, 0.5378227212816178, 0.5392730239154455, 0.5415735259009027, 0.544539938264107, 0.5480203870196857, 0.5518905826454878, 0.5560486441924208 ], [ 0.5880386030872111, 0.6011525848031521, 0.6169151915511446, 0.6351193809570644, 0.6554332900680611, 0.6774204682285181, 0.7005633012954464, 0.7242914093398073, 0.7480179874154917, 0.7711823372292582, 0.7932868976687205, 0.8139108243080886, 0.8326929358720049, 0.8492974274366207, 0.863385964927159, 0.8746161350178305, 0.882655253771142, 0.8872211893993142, 0.8881601195435722, 0.8855140686107856, 0.8795523215022939, 0.8707655657138577, 0.8597255852246781, 0.8467499989389162, 0.8317125973597156, 0.8143612549989915, 0.7946519633320197, 0.7728065701816813, 0.7492470901734438, 0.7245336555962566, 0.6993138757225961, 0.6742809651757199, 0.6501360970710925, 0.6275492832998173, 0.6071128948799934, 0.589284917272364, 0.5743325050886441, 0.5623091115653102, 0.5530872377315084, 0.5464207922622266, 0.5420030291117406, 0.5395087877964186, 0.5386220826512493, 0.5390524785551039, 0.5405433903522201, 0.5428747712734483, 0.5458620676270148, 0.5493528665976803, 0.5532223262289713, 0.5573682140537493 ], [ 0.5901292905281197, 0.6034887568471812, 0.6194922989381548, 0.6379109005121072, 0.6583880799608144, 0.6804628636482482, 0.7035951186587888, 0.7271957555737225, 0.7506657692217248, 0.7734430965715754, 0.7950443874253333, 0.8150796320408046, 0.8332299034147188, 0.8492040062574877, 0.8627010175687271, 0.8734006751702268, 0.8809762070720631, 0.8851421115045737, 0.8857411960044568, 0.8828156441176973, 0.8766340952899498, 0.8676856131876335, 0.85654329819814, 0.8435318053180876, 0.8285358940782583, 0.8113069450775949, 0.7917982643405136, 0.7702236465507378, 0.7469931505747142, 0.7226528993752128, 0.6978355589420491, 0.6732188599517323, 0.6494878516785247, 0.6272955701953767, 0.6072169846353241, 0.5896948325411875, 0.574988625837538, 0.563155005895682, 0.5540764602595436, 0.5475179141128392, 0.5431819089637462, 0.5407502451005817, 0.5399117476663973, 0.5403790171408007, 0.5418971257518679, 0.5442466802599886, 0.5472431198070262, 0.5507336730219039, 0.5545930575654829, 0.5587187384561321 ], [ 0.5921550552207142, 0.6057349258371467, 0.6219522094345311, 0.6405565661181563, 0.6611679976518683, 0.6833024648305834, 0.706398824155147, 0.7298504650030953, 0.7530467101270879, 0.7754240107698638, 0.7965130859688531, 0.8159551113407314, 0.833474092331033, 0.8488244787437137, 0.8617441869574646, 0.8719356491574355, 0.8790772608075389, 0.8828791040218645, 0.8831800203410681, 0.8800221339270471, 0.873671966624785, 0.8646096503631263, 0.8533957356891222, 0.8403556668005103, 0.8253976206496889, 0.8082894808574248, 0.7889849139228199, 0.7676889994590231, 0.7447990185104093, 0.7208461400987098, 0.6964469957204695, 0.6722626010788975, 0.6489605630343184, 0.6271756857172623, 0.6074646533329187, 0.5902547121200843, 0.575797704945415, 0.5641531231222179, 0.5552133524096394, 0.5487548590575678, 0.5444903163573637, 0.5421093479415269, 0.5413063390647934, 0.5417975318584374, 0.5433301289678789, 0.545685751554609, 0.5486800997998701, 0.5521602336182627, 0.5560005508913303, 0.5600982713310398 ], [ 0.5941186867253243, 0.6078931790135551, 0.6242959954025311, 0.6430562115526626, 0.6637715764628418, 0.6859366256257509, 0.7089709096540225, 0.7322517045272952, 0.7551573764366911, 0.7771228024106388, 0.7976925062977216, 0.8165389720028254, 0.8334294018211682, 0.8481643965181184, 0.8605215910215974, 0.8702265831320987, 0.8769625894121843, 0.8804346276320913, 0.880477168968778, 0.8771319977593202, 0.8706617850769892, 0.8615307083157335, 0.8502749457326697, 0.837216794122917, 0.8222963492651725, 0.8053089791445269, 0.786212447316172, 0.7652028838217181, 0.7426640914230446, 0.7191114445136999, 0.6951445568925221, 0.6714066234422174, 0.6485466563863973, 0.6271802039826045, 0.6078450520162996, 0.5909528868971252, 0.5767478930171499, 0.5652920541217848, 0.5564874079929635, 0.5501222502521911, 0.5459200476294626, 0.5435790072887934, 0.5427997779567795, 0.5433028350909566, 0.5448379858693331, 0.54718823425368, 0.5501698190869783, 0.5536298308021028, 0.5574424770619457, 0.5615047983308138 ], [ 0.5960237084393842, 0.6099664260877933, 0.6265256008933695, 0.6454105736056885, 0.6661982799628353, 0.6883636448103269, 0.7113088005754018, 0.7343965454385962, 0.7569952408840319, 0.7785381893923877, 0.7985833545334712, 0.8168344107753263, 0.8331015205378762, 0.8472312632121389, 0.8590412717531404, 0.8682806601183739, 0.8746374946462473, 0.877811653399616, 0.8776330716026175, 0.8741426382716707, 0.8675968329266978, 0.8584377293077, 0.8471687354539597, 0.8341068230239541, 0.8192284498479336, 0.8023642605161415, 0.7834804347194476, 0.762764736490562, 0.740587029271588, 0.7174461943558765, 0.6939239746635516, 0.6706447827036534, 0.6482380651507584, 0.6272993196758376, 0.6083470527974648, 0.5917774528809003, 0.5778270827345078, 0.5665600570807305, 0.5578877104605545, 0.5516102582980443, 0.5474624450815428, 0.5451517085366276, 0.5443856056593386, 0.5448894127874337, 0.5464160104106177, 0.5487501574243794, 0.551708916409867, 0.5551396161676381, 0.5589164125892951, 0.562936242077525 ], [ 0.5978742593791033, 0.6119582854280756, 0.6286437366353438, 0.6476212101640944, 0.6684484550009375, 0.6905827560854932, 0.7134108726621479, 0.7362829833497659, 0.7585586811331176, 0.7796698556699319, 0.7991874762080873, 0.8168460317917587, 0.8324978145545363, 0.8460344149994711, 0.8573131245314731, 0.8661067602135425, 0.8721085685377775, 0.8750139444478283, 0.8746484342997582, 0.8710509451330315, 0.864468872770357, 0.8553174999225573, 0.8440628115589416, 0.8310151993100365, 0.8161884193180338, 0.7994529126594783, 0.780787580694787, 0.7603732834693322, 0.7385658637834891, 0.7158472038000685, 0.6927804759705387, 0.6699705089397738, 0.6480264019711085, 0.6275230255068214, 0.608959422692454, 0.5927164428131281, 0.5790230838615132, 0.5679452361392365, 0.5594031034958062, 0.5532087511116401, 0.5491085207484362, 0.5468196097320684, 0.5460570596468721, 0.5465514820672251, 0.5480592875819509, 0.5503673615564689, 0.5532938803007862, 0.5566866261541084, 0.560419850062739, 0.5643904686066469 ], [ 0.5996749495127496, 0.6138729438911065, 0.6306537525577605, 0.6496904049279902, 0.6705232816875901, 0.6925941261468798, 0.7152764898756565, 0.7379099907337638, 0.7598470122662409, 0.7805184494792792, 0.7995078184305484, 0.8165797611685948, 0.831627190805437, 0.8445848794507671, 0.8553487992806738, 0.8637154652605195, 0.8693838628918165, 0.8720463664859355, 0.871524683477501, 0.8678539514780771, 0.8612693510090997, 0.8521564184832441, 0.8409425505885172, 0.8279304903031356, 0.8131694742187173, 0.7965714616101993, 0.7781318273475645, 0.7580266411238056, 0.7365981029018236, 0.7143108320197643, 0.6917089080987308, 0.6693769469379086, 0.6479031093086305, 0.6278412637943845, 0.6096709702647648, 0.5937579706546616, 0.5803237708902613, 0.5694356940590187, 0.5610223411800002, 0.5549074305929019, 0.5508490729361286, 0.5485746365416105, 0.5478071485826068, 0.5482830491732404, 0.549762717333683, 0.5520355312463703, 0.5549210729039022, 0.5582677989558105, 0.5619502097158955, 0.5658652948370977 ], [ 0.60143068540892, 0.6157149869092599, 0.6325594866606197, 0.6516210593211819, 0.6724247202432527, 0.6943988595519027, 0.7169060630243059, 0.7392776069082279, 0.7608605631333205, 0.7810856205513671, 0.7995484204605458, 0.8160427635639018, 0.8304999337367889, 0.8428951913140315, 0.8531615435165111, 0.8611189922640186, 0.8664730408135417, 0.8689152265486793, 0.8682643894193007, 0.8645495198858758, 0.8579905979224459, 0.8489420687617308, 0.8377944619736074, 0.8248415107560089, 0.8101642341000506, 0.7937156589316149, 0.7755104868396229, 0.7557224192948069, 0.7346808313992301, 0.7128330895343618, 0.6907038546943209, 0.6688570818550683, 0.6478595899007561, 0.6282440544482476, 0.6104706681060422, 0.5948903520130497, 0.5817172063891753, 0.5710196611881488, 0.5627342180999917, 0.5566959549220798, 0.5526747937711429, 0.5504085722337819, 0.5496287249868756, 0.5500779666490945, 0.5515210586284143, 0.5537502284888056, 0.5565867546296388, 0.5598799923426233, 0.5635048518973831, 0.5673584968927592 ], [ 0.6031464626755692, 0.6174891955681191, 0.6343650911606443, 0.6534165738529633, 0.6741554547835702, 0.6959990074832153, 0.7183011251577415, 0.7403870669205386, 0.7616008069549246, 0.7813741093983289, 0.7993144465417662, 0.8152433666147029, 0.8291275080120766, 0.8409791429894987, 0.85076595693861, 0.8583310068931708, 0.86338741401717, 0.865628564019501, 0.8648715836817009, 0.8611369225087231, 0.8546268145634396, 0.8456646015533019, 0.8346072980708218, 0.821738195477568, 0.8071653660384961, 0.7908808292363071, 0.7729204075336672, 0.7534578313843945, 0.7328108089425553, 0.7114097378040956, 0.6897597412208933, 0.6684038498931306, 0.6478873181128495, 0.6287216016920837, 0.6113477539144331, 0.5961022032886234, 0.5831917428390231, 0.5726856030691203, 0.5645276804103743, 0.558564046131394, 0.554576366665007, 0.5523131413941142, 0.5515145543622706, 0.5519299887641775, 0.5533289728556591, 0.5555069259926668, 0.5582871092042527, 0.5615200020651283, 0.5650810902043565, 0.5688678190982375 ], [ 0.6048271213914833, 0.6192003085439668, 0.636074840881058, 0.655080723480574, 0.6757188343940583, 0.6973975758089577, 0.7194644150350975, 0.741240963860341, 0.7620705557572128, 0.7813879029495246, 0.7988122736562612, 0.8141909960358804, 0.8275223216079249, 0.8388514494313807, 0.848177632603848, 0.8553662801260378, 0.860139716179059, 0.8621960888060148, 0.8613518838062534, 0.8576172119581152, 0.8511746581180952, 0.8423174732089675, 0.8313726554629521, 0.8186121785170314, 0.8041661124104728, 0.78806221913224, 0.7703581617709163, 0.7512298117901923, 0.7309845662223183, 0.7100363818995512, 0.6888709295578223, 0.6680102343055647, 0.6479779335890224, 0.6292643818107702, 0.6122918126794697, 0.5973825220512777, 0.5847361054814268, 0.5744223089707224, 0.5663919192162106, 0.5605015831802913, 0.5565445531431157, 0.5542800864916015, 0.5534573798589273, 0.5538328243716094, 0.5551810659379419, 0.5573010399908754, 0.560018268705395, 0.5631845805328197, 0.566676205043923, 0.5703909834731073 ], [ 0.6064770604702951, 0.6208527512372118, 0.6376929353430147, 0.6566175340329684, 0.677118812199262, 0.6985985264397863, 0.7203999548041699, 0.7418434271141177, 0.7622742228105377, 0.7811324714864757, 0.7980496385020394, 0.8128961190086944, 0.8256974528403326, 0.8365273135270038, 0.8454126690030865, 0.8522401927213074, 0.8567436218124942, 0.8586287538743739, 0.8577124064808221, 0.8539933425484364, 0.84763337400624, 0.8388972612004177, 0.8280850382369109, 0.8154570827042185, 0.8011606681528173, 0.7852553049274873, 0.7678202368305065, 0.7490351369631663, 0.7291984986616045, 0.7087085561223095, 0.6880318018492313, 0.6676693474711826, 0.6481233187865694, 0.6298632140806634, 0.6132928422490138, 0.5987207509186694, 0.5863394584633057, 0.576218963493224, 0.568316447807285, 0.5624986811573184, 0.5585702689176741, 0.5563012367678948, 0.5554499818127425, 0.5557801865473158, 0.5570719285590351, 0.559127962075378, 0.5617763382102932, 0.564870455474086, 0.5682874573981531, 0.5719256995537193 ], [ 0.6080999066439126, 0.6224503490624367, 0.6392233142016432, 0.6580311686980438, 0.6783598834343525, 0.6996067653472755, 0.721113103983317, 0.7422002825589114, 0.7622181349736482, 0.780615101820927, 0.7970358219622263, 0.8113701862496447, 0.8236663622563167, 0.8340218994598543, 0.8424870478330402, 0.8489681506510184, 0.853213193515442, 0.8549382870727574, 0.8539615279949054, 0.8502700656883103, 0.8440045962002993, 0.8354032270477806, 0.824741597193483, 0.8122685714535695, 0.798144406980645, 0.7824560353444748, 0.7653032117089463, 0.7468705441035725, 0.7274489562045863, 0.7074218023306584, 0.6872368339315428, 0.6673745000024713, 0.6483156620226346, 0.630509316886734, 0.6143413043374994, 0.6001068269858053, 0.5879914563388382, 0.5780652032376157, 0.5702911643204582, 0.5645457574806813, 0.560644649409187, 0.5583685692162452, 0.5574852317160626, 0.5577658385166643, 0.5589961740493994, 0.5609830896548177, 0.5635574197242755, 0.5665743483113654, 0.5699121025827307, 0.5734696743794396 ] ], "zauto": true, "zmax": 0.9133503503851633, "zmin": -0.9133503503851633 }, { "autocolorscale": false, "autocontour": true, "colorbar": { "tickfont": { "size": 8 }, "ticksuffix": "", "x": 1, "y": 0.5 }, "colorscale": [ [ 0.0, "rgb(255,247,251)" ], [ 0.14285714285714285, "rgb(236,231,242)" ], [ 0.2857142857142857, "rgb(208,209,230)" ], [ 0.42857142857142855, "rgb(166,189,219)" ], [ 0.5714285714285714, "rgb(116,169,207)" ], [ 0.7142857142857143, "rgb(54,144,192)" ], [ 0.8571428571428571, "rgb(5,112,176)" ], [ 1.0, "rgb(3,78,123)" ] ], "contours": { "coloring": "heatmap" }, "hoverinfo": "x+y+z", "ncontours": 25, "type": "contour", "x": [ 1e-06, 1.3011511650442548e-06, 1.692994354296022e-06, 2.2028415765056147e-06, 2.866229883678204e-06, 3.729398352432554e-06, 4.852511011181743e-06, 6.3138503555892e-06, 8.215273746089953e-06, 1.0689313005882424e-05, 1.390841207112662e-05, 1.809694657026198e-05, 2.354686311364001e-05, 3.063802837345029e-05, 3.986470631277378e-05, 5.1870009063012666e-05, 6.749072272319499e-05, 8.781563250096393e-05, 0.00011426141253772724, 0.00014867137004306603, 0.00019344392634026088, 0.0002516997901283655, 0.0003274994751669172, 0.0004261263236648159, 0.0005544547624925005, 0.0007214294601814526, 0.000938688782612345, 0.0012213760031100258, 0.0015891948094037057, 0.002067782677737912, 0.0026904978401970136, 0.0035007443993213955, 0.004554997653699184, 0.005926740503884541, 0.007711585311544345, 0.010033938212454078, 0.013055670395116691, 0.01698740074503987, 0.02210317627048227, 0.028759573555516536, 0.03742055263793628, 0.04868979566145066, 0.06335278435066323, 0.0824315491666629, 0.10725590623460621, 0.13955614735503497, 0.18158364372009145, 0.23626776957937787, 0.3074200836506151, 0.4 ], "xaxis": "x2", "y": [ 0.0, 0.02040816326530612, 0.04081632653061224, 0.061224489795918366, 0.08163265306122448, 0.1020408163265306, 0.12244897959183673, 0.14285714285714285, 0.16326530612244897, 0.18367346938775508, 0.2040816326530612, 0.22448979591836732, 0.24489795918367346, 0.26530612244897955, 0.2857142857142857, 0.3061224489795918, 0.32653061224489793, 0.3469387755102041, 0.36734693877551017, 0.3877551020408163, 0.4081632653061224, 0.42857142857142855, 0.44897959183673464, 0.4693877551020408, 0.4897959183673469, 0.5102040816326531, 0.5306122448979591, 0.5510204081632653, 0.5714285714285714, 0.5918367346938775, 0.6122448979591836, 0.6326530612244897, 0.6530612244897959, 0.673469387755102, 0.6938775510204082, 0.7142857142857142, 0.7346938775510203, 0.7551020408163265, 0.7755102040816326, 0.7959183673469387, 0.8163265306122448, 0.836734693877551, 0.8571428571428571, 0.8775510204081632, 0.8979591836734693, 0.9183673469387754, 0.9387755102040816, 0.9591836734693877, 0.9795918367346939, 1.0 ], "yaxis": "y2", "z": [ [ 0.11963900033082862, 0.11120502718612327, 0.10236833933703983, 0.09351464531333184, 0.08522000828071864, 0.07823742087219578, 0.0733521652463189, 0.07104269512681603, 0.0711215157044716, 0.07277499861180099, 0.07496004348068795, 0.07676451209001947, 0.07756522531890092, 0.07704393230709297, 0.07515351716325781, 0.07207875800043669, 0.06819716030390856, 0.06402396099121976, 0.060116063637031446, 0.05693980142596851, 0.054825052387451424, 0.054008477748462395, 0.0546256801413763, 0.05669139308557146, 0.06016648617380013, 0.06499997633026526, 0.07107641232215517, 0.07811593866331752, 0.08568681415371496, 0.0933341305533304, 0.10067135983796932, 0.10741353578028373, 0.11337462226693869, 0.1184482927151689, 0.12258340232233154, 0.12576020224871068, 0.1279706966948241, 0.12920514669683758, 0.12944573495288578, 0.1286675782335367, 0.1268467135716976, 0.12397456788186667, 0.12007879152804614, 0.11525099773710244, 0.10968231763754799, 0.10370629309174884, 0.09784200952173126, 0.0928122652659211, 0.08947980314907722, 0.088630510241294 ], [ 0.1189541971465276, 0.11041303152135724, 0.10144100396884981, 0.09241919208840459, 0.08392544666993985, 0.07672927457944152, 0.07165395363318532, 0.06922634561862112, 0.0692816574575042, 0.07098610223801857, 0.07325638338855982, 0.07514729259038597, 0.07601748716877872, 0.07554133470875055, 0.0736696568805988, 0.07058710656323174, 0.06667182206962342, 0.06244108262398428, 0.058457289856717796, 0.05519929140853992, 0.0530165078583715, 0.05216787576165439, 0.05280778391186346, 0.05495694162600182, 0.05856175167866182, 0.06354237598724635, 0.06975344957087316, 0.07689579436942211, 0.08453183026735085, 0.09220936815078887, 0.099549094064882, 0.10627518953013369, 0.11221134907254886, 0.1172607247153624, 0.12138065197309708, 0.12455812431138698, 0.12678954331995354, 0.12806692779640025, 0.1283717039939921, 0.12767623734100966, 0.12595258546298452, 0.12318778428147714, 0.11940537256197156, 0.11469358248684734, 0.10924104112575331, 0.10337944129454686, 0.0976259776470779, 0.09270027907092993, 0.08945930510696357, 0.0886807657450318 ], [ 0.11836383500278834, 0.10974576364159584, 0.10067396056633467, 0.09152346938111927, 0.08287013374268408, 0.07549270655137562, 0.07024264972203614, 0.06768849553780494, 0.06769323727628695, 0.06941380804336264, 0.07173547916533622, 0.0736854262176122, 0.07460694296720852, 0.07416804630565933, 0.07231735999762735, 0.06923828621167152, 0.0653072751752062, 0.061039770693253295, 0.05699866019986124, 0.053672773089162996, 0.05143142203802397, 0.05055578525605705, 0.0512196609318905, 0.053452041485358495, 0.057184092597623, 0.06230306461799206, 0.06862983075230854, 0.07584674101294425, 0.08351521012033421, 0.09118978421424036, 0.09850078992633761, 0.10518291088101173, 0.11107038411645857, 0.11607641988624764, 0.12016729300932188, 0.12333707021979566, 0.1255867083124086, 0.12690994959947802, 0.12728727769714104, 0.12668806356555457, 0.12508020297151923, 0.12244631663974842, 0.118806002886422, 0.11424441407418097, 0.10894787241241541, 0.10324580641777058, 0.09765155208226586, 0.09287671955204907, 0.08976386502373115, 0.08907181558503122 ], [ 0.11786665575767165, 0.1092028069781808, 0.10006824423543142, 0.09083086234142225, 0.08206094542340636, 0.07453899428944581, 0.0691335061224271, 0.06644604411592257, 0.06637131072637162, 0.06806885524750628, 0.0704031094566066, 0.07238022964408343, 0.07333127138855354, 0.07291885061830207, 0.07108907603382053, 0.068022835736752, 0.06409254938637808, 0.05980814039943589, 0.05572869894608569, 0.05235041104415376, 0.05006177341261326, 0.049165646144389945, 0.04985568169728583, 0.05217066142539434, 0.05602583288240893, 0.06127236454846183, 0.06769501815339374, 0.07495890915063987, 0.08262830751148328, 0.09026800371945796, 0.09752014044219022, 0.10413117594869313, 0.10994672906500187, 0.11489073397983572, 0.11893896871846656, 0.12209299581269656, 0.12435853978340006, 0.12573103943495526, 0.12618981340467308, 0.12570093013958658, 0.1242278638699777, 0.1217487165857014, 0.11827923587799968, 0.11390170277637902, 0.10880021716104599, 0.10330138624738283, 0.09791259805947419, 0.0933326453141771, 0.09038156351228185, 0.0897899039787718 ], [ 0.11745983198142561, 0.10878158614131235, 0.09962190767795727, 0.09034066150869756, 0.08149929148022564, 0.07387257646792257, 0.06833418579084635, 0.06550889373802958, 0.0653252748634055, 0.06695754306004262, 0.06926151017209752, 0.07123007420382522, 0.07218553712393516, 0.07178600219706077, 0.06997456817609533, 0.06692820683859492, 0.0630129756531171, 0.058729948768909045, 0.054630975305502545, 0.051216987384543146, 0.048893979428907955, 0.047985373875437436, 0.04870469244986766, 0.051101197354010044, 0.055073635891427324, 0.06043515755960032, 0.0669336953397399, 0.07421857461961724, 0.08185962573792874, 0.08943464758482746, 0.09659947752039157, 0.10311356205270646, 0.10883480725147135, 0.11369866113129411, 0.11769110608593078, 0.12082173844081547, 0.12310133096189396, 0.12452700117781693, 0.12507663665325797, 0.124712616080088, 0.123393636505399, 0.12109307497256981, 0.11782280611515618, 0.11366231412391994, 0.10879338983086498, 0.10353906767744692, 0.09839856491123594, 0.09405322948097951, 0.0912933513514504, 0.0908137399016617 ], [ 0.11713898724937885, 0.10847738398515536, 0.09933002135880523, 0.09004801403743591, 0.08118096146609836, 0.07349075524921724, 0.06784440440158375, 0.06487931581291842, 0.06455815317227004, 0.06608121851319873, 0.06830913169758865, 0.07023037143442341, 0.07116235583487725, 0.07075955043814482, 0.06896140852604664, 0.06593947693842271, 0.06205119274792052, 0.05778591743515158, 0.05368556768529253, 0.05025330795030925, 0.04791023221947717, 0.04699858300621228, 0.047751079035003176, 0.0502274355397346, 0.05430955651610544, 0.059772101282163405, 0.06632693568733247, 0.0736090575168562, 0.08119544024072216, 0.08867876294878552, 0.0957300777522539, 0.10212297499721766, 0.10772864160393478, 0.11249498041979446, 0.11641904227861744, 0.1195191308209113, 0.12181142632707147, 0.12329471641397494, 0.12394513834369034, 0.12372089952010636, 0.12257545027308135, 0.12047711350898925, 0.1174337322478495, 0.11352190395029467, 0.10892082080820656, 0.10394897714166215, 0.09909509476075015, 0.095018777786463, 0.09247457765444013, 0.0921162782319373 ], [ 0.11689825781101762, 0.10828342693430575, 0.09918478911466243, 0.08994407125690598, 0.08109629192949563, 0.07338385240008402, 0.06765596974426627, 0.06455175445297652, 0.06406622502847528, 0.0654359924423301, 0.06754056097422961, 0.0693736872147057, 0.07025216766825089, 0.06982776459638713, 0.06803557783901763, 0.06504017636729305, 0.061188266080842636, 0.05695512800381886, 0.0528706482643633, 0.04943784273748797, 0.04709006446714653, 0.046185999392652784, 0.046976064920335465, 0.049529787559213286, 0.05371233932358245, 0.059260977901274776, 0.06585344465956869, 0.07311172688575934, 0.08062051631840537, 0.08798832116791602, 0.09490251423949524, 0.10115190842423943, 0.10662205633458899, 0.11127442136779293, 0.11511816723939426, 0.11818112928756401, 0.12048534102047545, 0.12203125803986373, 0.12279288368346732, 0.12272366262440414, 0.12177119874988034, 0.11989829275164719, 0.11710844461292838, 0.11347509419810631, 0.10917433172525003, 0.10451894128546256, 0.09998479494183364, 0.09620594988542176, 0.0938966746723001, 0.09366658459401918 ], [ 0.11673039214610696, 0.10819103229041241, 0.09917576786597226, 0.09001631201943441, 0.08123061639736741, 0.07353574483277298, 0.067753203890978, 0.06451302856910063, 0.06383900961552841, 0.06501271378984784, 0.06694662382366737, 0.06864998167151398, 0.06944360226114912, 0.06897762888396229, 0.06718211898709794, 0.06421314692032401, 0.06040479209468658, 0.056216354325065714, 0.052164011109487866, 0.048748360360586496, 0.046411920246052375, 0.045526872251108824, 0.04635903734998369, 0.048986595141046606, 0.053258770810081395, 0.05887803431981496, 0.06549076950561396, 0.07270700199511047, 0.08011885310266978, 0.08735074593864323, 0.09410703418519917, 0.10019272421916615, 0.10550889639866128, 0.11003184519089146, 0.11378408076991109, 0.1168039556404597, 0.1191198930607428, 0.12073401577448711, 0.12161773204319917, 0.12171900667099761, 0.1209788533040927, 0.119353933147937, 0.11684293222769604, 0.11351567972487868, 0.10954445938790909, 0.10523501691936211, 0.1010480905682756, 0.09758903119317304, 0.09552879896595406, 0.09543160422451492 ], [ 0.1166268831326649, 0.1081898076583343, 0.09929017274322625, 0.09024900661070953, 0.08156493604497528, 0.07392470042179364, 0.06811371777447091, 0.06474289471719097, 0.06385958517744564, 0.0647972155810302, 0.06651466978479305, 0.06804696258646804, 0.0687239119811943, 0.06819537334758187, 0.06638579439112506, 0.0634413592492341, 0.059681892979848654, 0.055549223197975096, 0.051544398279682026, 0.048163363081131005, 0.04585454342224863, 0.04500021471685754, 0.045878719893333475, 0.04857532149519707, 0.0529249311340723, 0.058599203031854126, 0.06521639319481833, 0.07237527325703311, 0.07967439602543197, 0.0867534365010045, 0.09333394228641978, 0.09923794278436215, 0.10438325855071598, 0.1087624385044857, 0.11241276243532726, 0.1153842517619457, 0.11771234783489802, 0.11940083294509306, 0.12041796665827757, 0.1207053755720819, 0.1201965845879035, 0.11884134525602975, 0.11663290294403358, 0.11363685498374226, 0.11002080677937162, 0.10608204797690231, 0.10226407909928711, 0.09914113288761751, 0.09733929391035812, 0.09737770472980307 ], [ 0.11657812631519297, 0.10826789029831808, 0.09951324452095246, 0.09062377821266272, 0.08207673926567916, 0.07452443199604916, 0.06870948087914226, 0.06521489143962567, 0.06410521671247899, 0.06477083274690952, 0.06622902826353969, 0.06755053211874527, 0.06807944697932569, 0.0674670074521996, 0.06563170515449873, 0.06270863541413368, 0.05900204017870269, 0.05493513052708831, 0.05099252451161621, 0.047663193955037556, 0.0453980563341421, 0.044585762571092154, 0.045514060629240585, 0.048273506854066034, 0.05268725183070573, 0.058401132250211714, 0.06500865822935066, 0.07209769492861713, 0.0792716760532084, 0.08618425714544997, 0.09257397218216802, 0.09828053274909963, 0.1032397280845356, 0.1074619163023826, 0.11100075268896639, 0.11391924620878936, 0.11626057430812452, 0.11803015378487837, 0.11919243288447046, 0.11968168580582518, 0.11942288897019862, 0.11835796488810892, 0.11647395012082148, 0.11383144908621676, 0.1105924000260536, 0.10704421125920481, 0.10361132437617991, 0.10083523821611577, 0.09929689684774703, 0.09947192161918905 ], [ 0.11657359712966225, 0.10841221321866322, 0.09982865486101118, 0.09112021758906803, 0.08274089861135961, 0.07530528460341218, 0.06950813451241493, 0.06589740275067735, 0.06454830654091458, 0.06491117681220662, 0.06607161111145202, 0.06714530086494794, 0.0674961454232246, 0.06677882795227827, 0.06490584085673289, 0.06200024434762696, 0.05834967694704946, 0.05435787855346671, 0.05049175171385281, 0.0472307535369114, 0.04502466880396622, 0.0442646266271847, 0.045244792503534324, 0.048059471964495025, 0.052523350901019075, 0.058261994850944084, 0.06484749557777504, 0.07185682609652061, 0.07889635040114301, 0.08563197225544311, 0.09181863165453154, 0.09731419110608618, 0.10207361593514389, 0.10612673129182044, 0.10954534375204066, 0.1124069320347526, 0.11476321235187106, 0.11662118032588302, 0.11794068345723721, 0.11864746041875586, 0.1186567166745609, 0.11790148871613462, 0.11636171915637791, 0.1140921582921363, 0.11124803256632952, 0.1081055204214278, 0.10506854775439572, 0.10264505265269992, 0.10137166399877166, 0.10168288848648922 ], [ 0.11660203988979065, 0.10860878505842464, 0.10021892605781627, 0.09171651027061037, 0.08353058016639553, 0.07623546981617026, 0.07047449335245265, 0.06675516651747489, 0.06515771683659763, 0.06519312488565529, 0.06602262326627546, 0.06681513983504113, 0.06696001444773245, 0.06611787921293574, 0.06419554108585647, 0.06130335771070844, 0.057711637346808634, 0.05380403487535645, 0.050028409135157006, 0.0468518185488467, 0.04471901355427139, 0.04401968368978297, 0.04505184418543991, 0.0479128816857231, 0.052412661467992026, 0.058162078078306305, 0.06471495843102334, 0.07163711534603308, 0.07853563357094813, 0.08508661419994049, 0.09106051122622534, 0.09633360655302688, 0.10088119160729522, 0.10475428702058655, 0.10804477894066265, 0.1108462551344263, 0.11321985049932165, 0.11517403774622927, 0.1166631288789934, 0.11760296441555507, 0.11789759813511075, 0.11747000582905728, 0.11629206755840947, 0.11441176622079978, 0.11197658139483195, 0.10925026695798355, 0.10661519387283375, 0.10454564754623547, 0.10353562332033069, 0.1039814722247902 ], [ 0.1166516617813259, 0.10884297201625046, 0.10066584501642775, 0.09239004221001636, 0.08441811080490051, 0.07728226174455315, 0.07157211634268165, 0.0677512016606845, 0.0659003660593551, 0.06558994044374838, 0.06606133174033696, 0.0665437398864161, 0.06645758105292653, 0.06547235113700947, 0.06348986052734434, 0.0606073690230037, 0.05707737958817479, 0.0532630422503922, 0.04959178920007955, 0.04651499409613978, 0.04446813983740064, 0.043835758710141594, 0.044917826649916934, 0.047815263752859216, 0.052336880470297484, 0.058084174841990834, 0.06459557685897538, 0.07142523756213874, 0.07817861822265419, 0.08453977840977225, 0.09029354962457689, 0.09533470066998344, 0.09965990821597757, 0.10334315256859994, 0.10649845923972379, 0.10923731235582075, 0.11163121324492768, 0.11368994669055249, 0.11536119062857646, 0.11654933845414157, 0.11714576485383209, 0.11706212080694568, 0.11626121283456507, 0.11478334379284945, 0.11276728447118219, 0.11046338616305604, 0.10823186526186843, 0.10651390935166317, 0.10576318844197947, 0.10634115595433566 ], [ 0.11671032592548407, 0.1090997719408385, 0.10115085528817236, 0.09311795803147722, 0.08537576141418476, 0.07841307729372632, 0.07276476717567289, 0.06884858286103765, 0.06674281904395879, 0.06607441241022424, 0.06616683731718086, 0.06631514948794381, 0.06597629654544025, 0.06483190606274257, 0.06277983737113747, 0.05990408884180709, 0.056439064733431854, 0.05272712918628483, 0.04917387469425461, 0.046211355936668225, 0.04426121821371125, 0.04369961456003264, 0.04482739199971579, 0.04775040542917589, 0.052280250987825565, 0.058013804439914864, 0.06447655776128658, 0.07121029883158551, 0.07781649245119064, 0.08398484536964147, 0.08951325285386037, 0.09431484321959081, 0.09840861662225671, 0.10189327678790021, 0.10490715589352992, 0.10758155843351201, 0.10999935664181162, 0.11217140056620103, 0.11403745430920957, 0.11548872728375109, 0.11640226082028658, 0.11667706404918356, 0.11626586327120295, 0.1152004228472503, 0.11360997257989561, 0.11173074400937255, 0.10990063208103401, 0.1085288196486569, 0.10803137734303826, 0.1087382209785724 ], [ 0.11676573862634008, 0.10936407289719517, 0.10165541545032074, 0.09387765439712527, 0.08637641798259577, 0.07959638258048488, 0.0740176161783086, 0.07001183827399736, 0.06765264083520288, 0.06661990003555975, 0.06631879544473035, 0.0661142651841023, 0.06550488247460313, 0.06418793071544351, 0.06205866941484901, 0.059187833030244386, 0.05579151723298735, 0.05219108510654242, 0.048768878860358106, 0.04593385500182634, 0.04408903327577316, 0.043599755339661844, 0.04476731962642066, 0.04770452765658869, 0.05222968271042428, 0.05793929085543416, 0.06434785662435293, 0.07098392844770797, 0.07744266396804347, 0.0834171329273324, 0.08871686609664418, 0.09327303912093993, 0.0971277660732409, 0.10040620003978078, 0.10327322750966096, 0.10588202034835308, 0.10832787032941257, 0.1106223450535697, 0.11269581907856123, 0.1144243987809273, 0.11566904032702152, 0.11631478533265698, 0.11630332754627666, 0.11565713931987216, 0.11449525267466455, 0.11303934700959407, 0.11160523077353371, 0.11057159670583667, 0.11031988091746878, 0.11115177830055338 ], [ 0.11680562708562905, 0.10962089079727055, 0.10216131643704518, 0.09464719884109757, 0.08739412629664714, 0.08080239563915313, 0.07529812297286965, 0.07120794853253123, 0.06859942418050732, 0.06720120278055687, 0.06649804284196748, 0.06592725431623857, 0.06503360992758345, 0.0635337123260383, 0.06132180372549478, 0.058455420433760764, 0.05513210182475861, 0.05165196810636518, 0.04837271649563306, 0.045676573475094136, 0.04394338553970768, 0.04352607831756444, 0.04472635633182078, 0.047666227865133255, 0.052174722518076054, 0.057851725171458535, 0.06420214591820667, 0.07074027695405771, 0.07705280325686976, 0.08283398445587128, 0.08790349928006341, 0.09221008539066806, 0.09581958877521965, 0.09888526096431184, 0.10160083952688224, 0.10414351690171653, 0.10662208313495777, 0.10904835597406647, 0.11134163869194226, 0.11336084874151323, 0.11494904778130997, 0.11597602686319294, 0.1163716000256544, 0.11614834366337685, 0.11541464278894241, 0.11437748128658665, 0.11333116905667047, 0.11262572962593208, 0.11261102255913766, 0.11356369414978414 ], [ 0.11681790504922332, 0.109855582839753, 0.102650954282363, 0.09540567117025922, 0.08840450849612914, 0.08200358404212733, 0.07657661082050211, 0.07240698239453808, 0.06955548837814977, 0.06779521677497767, 0.06668710119958343, 0.06574189566404819, 0.06455450731069746, 0.0628645395822914, 0.06056694479511461, 0.05770609175685581, 0.05446054282967596, 0.05110880321978213, 0.04798255158337924, 0.04543408442904632, 0.043816569373302264, 0.04346943899016711, 0.044694868694227924, 0.04762624042523387, 0.05210740338610352, 0.05774483715875591, 0.06403470151888424, 0.07047593738692355, 0.07664481782360007, 0.0822347993461043, 0.08707420794409423, 0.09112869650009645, 0.09448826533034763, 0.09733579488557308, 0.0998961827280432, 0.10237287995254345, 0.10488926790693821, 0.1074568101451191, 0.1099818482020764, 0.11230388578216398, 0.11424627489227002, 0.11566237241469994, 0.11646941947446889, 0.11666967773821861, 0.11636066086372482, 0.115734789482986, 0.11506575545861215, 0.11467693323793775, 0.11488964327467746, 0.1159584457331623 ], [ 0.11679082496633866, 0.11005403541832391, 0.10310755794935021, 0.09613342991980048, 0.08938505965755104, 0.08317497740399657, 0.07782658009478194, 0.0735824317743239, 0.07049628843950158, 0.06838137508571426, 0.066870544894883, 0.06554782979750864, 0.06406149432059412, 0.062177729636072515, 0.059793983850563295, 0.05694135470068824, 0.053778698783549184, 0.050562303123971274, 0.04759651403191625, 0.045201270083182844, 0.04370105723546104, 0.043421207514304526, 0.04466437275213948, 0.04757709348448046, 0.0520220128915919, 0.05761479939186953, 0.0638432246627862, 0.0701898045630457, 0.07621876896342944, 0.08162101243776823, 0.08623203115673031, 0.09003359615180867, 0.09314006685903989, 0.09576531886098134, 0.09816768561072636, 0.10057917172918246, 0.10313883911247887, 0.10585704183475149, 0.10862506878400321, 0.11126068982662221, 0.11356579043923475, 0.11537626953666838, 0.11659629975369956, 0.11721761867390564, 0.1173268713713022, 0.11710229514814308, 0.1167980705719692, 0.11671304754734883, 0.11714293956766288, 0.1183229350447905 ], [ 0.11671311624559913, 0.11020282676679365, 0.10351537443432317, 0.09681231028646632, 0.09031533925275309, 0.08429432643224453, 0.07902481973222025, 0.07471131641770332, 0.07140058946711379, 0.06894189168927685, 0.06703523260700706, 0.06533671629470497, 0.06355044176725254, 0.06147258255436057, 0.05900484961504608, 0.05616475341567778, 0.053090290953668455, 0.05001460966744947, 0.04721355470357418, 0.04497335466475983, 0.04358940242117319, 0.043372925730699195, 0.04462704253232934, 0.04751276062572287, 0.05191482569570969, 0.05745998423937213, 0.06362761344044746, 0.06988288529854585, 0.07577674162433953, 0.08099602866029365, 0.08538198779003045, 0.08893157246764702, 0.09178346785504571, 0.09418369598800877, 0.0964262126758736, 0.09877388970527591, 0.10138253381963344, 0.10426047485042118, 0.10728168132179605, 0.11023983677360687, 0.11291373776485346, 0.11512102227871442, 0.11675253186548318, 0.11778949119442467, 0.11830789451181375, 0.11847238425241456, 0.11851889515725235, 0.1187239008400708, 0.119360274578252, 0.12064628111777162 ], [ 0.11657411005456224, 0.11028936589453911, 0.10385981516813529, 0.09742576250361114, 0.09117707581523792, 0.08534214366956223, 0.08015137324855072, 0.07577412373248228, 0.07225046281902153, 0.06946183986237407, 0.0671704116425019, 0.06510229910352926, 0.06301915837005734, 0.060750264956995335, 0.058203279702347455, 0.0553815551880808, 0.052400572557580974, 0.049469028407060855, 0.046833346545996996, 0.04474596210426181, 0.04347430714924238, 0.04331621608941289, 0.044575460028222824, 0.047428397828679636, 0.05178383569346207, 0.05728068935162624, 0.06338969535376265, 0.06955807146091927, 0.07532267735732733, 0.08036511861258858, 0.08453103166762475, 0.08783149208994413, 0.09042922137518157, 0.09260326828329157, 0.09468523688597758, 0.09697114654310113, 0.09963456287956386, 0.10268071715952039, 0.10596385740455173, 0.10925128119032432, 0.1122972952734877, 0.11490075247016508, 0.11693915747236695, 0.11838345065671628, 0.11929938317001627, 0.11983875285978136, 0.12022060780096235, 0.12070115112080376, 0.12153297723569144, 0.12291960499686692 ], [ 0.11636385182195864, 0.11030201033841668, 0.10412756895645456, 0.09795894095225266, 0.09195420402917638, 0.08630165922872303, 0.08118940761406561, 0.07675463880716844, 0.07303115485758428, 0.06992909837312226, 0.06726770875581005, 0.06484038408740646, 0.062467306741513434, 0.06001362556969357, 0.05739451320379906, 0.05459834629866418, 0.051715919825293936, 0.048929718859058856, 0.04645616321734286, 0.04451517871954508, 0.043348803656763205, 0.04324298018516142, 0.04450273442450774, 0.04732020007271184, 0.05162850604441158, 0.05707884169176432, 0.06313293175869177, 0.06921988810198411, 0.07486218017148615, 0.07973528022648481, 0.08368796509027122, 0.08674426688463789, 0.08909038514142471, 0.09103894322892572, 0.09296096957619754, 0.09518780745203957, 0.09791171440333797, 0.10113360135583982, 0.10468553443526794, 0.10830628843671312, 0.1117245956644248, 0.11472032834956501, 0.1171579147676171, 0.11899843961502818, 0.12029797288579169, 0.1211963290329598, 0.12189706240676446, 0.12263811677202967, 0.12365413962686782, 0.1251358169900862 ], [ 0.11607320316794167, 0.11023016594829868, 0.10430668740141634, 0.09839875467327107, 0.09263285254371692, 0.08715872031832315, 0.08212502337225963, 0.0776397078686773, 0.07373087047332952, 0.07033419701723582, 0.06732102326336852, 0.06454873439294118, 0.06189625162719771, 0.0592669472162408, 0.056584909787530974, 0.053822534623655924, 0.051043328361920215, 0.04840130931104371, 0.046082704659961865, 0.044277609341987934, 0.04320649763047493, 0.043145738062596334, 0.044402737685141275, 0.047185343120948846, 0.05144953670780286, 0.05685768576962748, 0.0628621070829539, 0.06887423059908086, 0.07440230527168287, 0.07911507109624491, 0.0828633092252615, 0.08568276505030666, 0.08778228377245444, 0.0895082140876277, 0.0912724250339414, 0.09344356104707915, 0.09623338616004674, 0.0996371505552398, 0.10346232000007867, 0.1074173077556712, 0.11120460051738212, 0.1145852603077021, 0.11741115833749591, 0.11963412114718665, 0.12130120993846985, 0.12254117596280056, 0.1235434535962165, 0.12452960424291826, 0.1257184193295276, 0.12728941226467505 ], [ 0.11569393541798062, 0.1100643723724502, 0.10438664903341341, 0.09873388962923592, 0.09320129888006468, 0.08790165805296168, 0.08294703514319171, 0.07841896776989093, 0.07434050493353196, 0.07067008945062275, 0.06732633859609383, 0.0642268901246049, 0.0613088442091871, 0.058515642224942745, 0.05578150879380863, 0.05306177148184741, 0.050389809091284016, 0.04788842117917976, 0.04571386769553928, 0.04403041882175486, 0.04304182420768926, 0.043017972773964845, 0.044270324035209425, 0.04702195373277911, 0.0512486378053098, 0.056621461572302044, 0.06258302145006005, 0.0685281072180398, 0.07395134081839741, 0.0785144155045877, 0.08206912901783622, 0.08466165686086428, 0.08652238913763888, 0.0880310887607743, 0.08964139016611179, 0.09176089280849332, 0.09462151779103813, 0.09821144599331877, 0.1023113099881871, 0.10659777887588771, 0.11074692829942545, 0.11450156471902262, 0.11770175544595624, 0.12029079250427348, 0.12230746236986723, 0.12387038224456635, 0.12515617621856368, 0.12637173818940967, 0.12772185093699137, 0.12937627836347404 ], [ 0.1152188171654904, 0.10979637815875186, 0.10435840835675059, 0.09895481230851956, 0.09364990543963814, 0.0885211399913213, 0.08364674401235501, 0.07908456553829518, 0.07485335033543093, 0.0709318776972046, 0.06728146835925382, 0.06387591945666679, 0.060709147052801546, 0.05776590087023572, 0.05499155191395349, 0.05232333129191917, 0.049761698508119144, 0.04739511216181083, 0.04535048625766683, 0.043771357501682026, 0.04285027948606503, 0.04285443243735098, 0.04410152559767029, 0.04682907970858543, 0.05102829689456209, 0.0563750869583551, 0.06230221449970196, 0.06818940557183172, 0.07351859250084479, 0.07794438958426148, 0.08131880989734377, 0.08369718453352055, 0.08533009852325406, 0.08662989718179023, 0.08809226302825106, 0.09016492345318511, 0.0931003886994646, 0.09687837062928885, 0.1012508058796776, 0.10586186690353806, 0.11036163597290824, 0.11447559829388948, 0.11803296198659961, 0.12096928290283261, 0.12331581844274847, 0.1251819442469747, 0.12673268363963258, 0.12816179769736358, 0.12966166922157965, 0.13139351650302125 ], [ 0.11464169855247888, 0.10941920947888256, 0.10421443576774636, 0.09905376326995861, 0.0939710482320338, 0.08901002252237519, 0.08421771743789866, 0.07963088566498652, 0.07526479749444251, 0.07111651011250852, 0.06718575352610327, 0.06349810940603146, 0.06010210530136081, 0.05702430473686388, 0.054222001933501296, 0.05161352031101061, 0.049163947372888074, 0.04692429959567761, 0.044993091425246344, 0.04349877582897901, 0.042628608395764205, 0.04265138516249372, 0.04389374581172733, 0.046606684143657894, 0.05079153733172338, 0.05612388869865331, 0.062026757243722516, 0.06786670122079543, 0.07311417930535918, 0.07741698738820982, 0.08062678389653655, 0.08280684649404713, 0.08422639006585435, 0.08532894447514194, 0.08665171909785611, 0.08868306959057766, 0.09169624471223739, 0.09566120374638781, 0.10029991994102871, 0.10522412402523633, 0.11005895718290844, 0.11451386705137047, 0.11840828216555736, 0.12167083948304545, 0.12432597669218262, 0.12647464467145342, 0.128271348275181, 0.12989806091964218, 0.13153614516844428, 0.13333927737460913 ], [ 0.11395759505035398, 0.10892723645011226, 0.10394875391282347, 0.09902474813699243, 0.09415904773655302, 0.08936321373604818, 0.08465558800556224, 0.08005429833906678, 0.07557205033896544, 0.07122247326978022, 0.06703972938932502, 0.06309660684026883, 0.059493171318694894, 0.05629741844772795, 0.05347909414857592, 0.05093720540885313, 0.04859959210969428, 0.04647735129534913, 0.04464175103232754, 0.04321163545833461, 0.04237494353449747, 0.04240683717038633, 0.04364598244758136, 0.0463557452698178, 0.05054173275770464, 0.055873478833100655, 0.061764149468171735, 0.06756912227765186, 0.07274884602724041, 0.07694486993460169, 0.08000820418248018, 0.08200898935875896, 0.08323333821208262, 0.084153978505611, 0.08534816349153053, 0.08734448348658808, 0.09043671834731323, 0.09458404651965056, 0.09947806359533458, 0.10469908196745453, 0.10984900341267063, 0.11462281580134598, 0.11883131676017526, 0.12239700560655811, 0.12533813139691036, 0.12774793068521867, 0.12977132690946006, 0.13157965953735437, 0.13334443530952544, 0.1352126114413081 ], [ 0.11316277355459806, 0.10831624090040336, 0.10355697557133756, 0.09886353245498317, 0.09421010937480649, 0.08957755469208091, 0.08495787926949713, 0.08035293874208245, 0.07577386737407028, 0.07124949799468261, 0.06684478477979455, 0.06267502509105218, 0.05888789254018272, 0.05559137431458125, 0.05276795547254601, 0.05029753474639252, 0.04806964123922023, 0.046054039415028836, 0.044296023908830445, 0.04290952031547848, 0.04208889685476118, 0.04212072784252803, 0.04335910500579217, 0.04607854863124678, 0.050282831317476656, 0.055629881742958215, 0.061522343564466654, 0.06730627462517326, 0.07243379387101946, 0.07654109868321596, 0.0794785696672811, 0.08132230692564742, 0.0823734803851215, 0.08313144866339903, 0.08421093404495678, 0.08617923586408865, 0.08935001640251093, 0.09367106952438878, 0.09880432453193715, 0.10430078615124833, 0.1097414382959775, 0.11480860579252655, 0.11930560548012596, 0.12314949575030128, 0.1263528569798992, 0.1290017944209772, 0.13123243264307938, 0.1332064438202964, 0.13508644427930014, 0.13701333328266502 ], [ 0.11225484356121569, 0.10758348921394781, 0.10303634760653231, 0.09856764576685208, 0.09412227940365522, 0.08965172488007417, 0.08512386466990275, 0.08052652530693585, 0.07587034294270369, 0.07119829995273355, 0.066602841601084, 0.06223704080766048, 0.0582914762107246, 0.054911466342746816, 0.052092322937406524, 0.04969587169233529, 0.04757327765080938, 0.045652760441175985, 0.04395501614993684, 0.04259264428293466, 0.04177161010265686, 0.041795113216924924, 0.043036188983547614, 0.04577916389586374, 0.050019985377100555, 0.05539989844215673, 0.061309882868463624, 0.06708821962918585, 0.07218052550403992, 0.0762188546233872, 0.07905330543186738, 0.08076525465901195, 0.08166904077767308, 0.08228755049522578, 0.08326923763666101, 0.08521722375974591, 0.08846386707131525, 0.09294559153721124, 0.09829675215956245, 0.10404229054255834, 0.10974513908307731, 0.11507688973714864, 0.11983446946400604, 0.12393007124571834, 0.1273709945345679, 0.13023665814739918, 0.13265501479455333, 0.1347788586336483, 0.1367627002086706, 0.13874189929160075 ], [ 0.11123285606500681, 0.10672781361051943, 0.10238580494014986, 0.0981363992455273, 0.09389542058708608, 0.08958617596486604, 0.0851544637903337, 0.08057622308015976, 0.07586273927938672, 0.07107037480278681, 0.06631608841035316, 0.06178602237966499, 0.05770835057033949, 0.054261776262064694, 0.05145439497157868, 0.04913192465844336, 0.04710820076941937, 0.04527085272217838, 0.04361749747938376, 0.042261848598014115, 0.04142577641817798, 0.04143434528413684, 0.04268288334653264, 0.04546399750085294, 0.04976019064125761, 0.055191570860926825, 0.06113610886420649, 0.06692548334719074, 0.07200069620741095, 0.0759911444889998, 0.07874730998882228, 0.08035540014377394, 0.08114103830452456, 0.08164707991212328, 0.08255083266219439, 0.08448681656484072, 0.08780424815631134, 0.0924290250404194, 0.09797158774852167, 0.10393514034236984, 0.10986786254245756, 0.11543259454159081, 0.12042086019584582, 0.12474042099128707, 0.12839354335312622, 0.13145326600393178, 0.13403984769181365, 0.13629783045560762, 0.13837424240349835, 0.14039929790810518 ], [ 0.11009741259344448, 0.10574970484256091, 0.10160603786257337, 0.09757092022276677, 0.09353121069538119, 0.08938309646386562, 0.08505217874850574, 0.0805045566744031, 0.07575337825078414, 0.07086786589335431, 0.06598680433120149, 0.0613247556031682, 0.05714175875231906, 0.053644881843565174, 0.05085485683919919, 0.0486040489457226, 0.04667106473572319, 0.044904973042660896, 0.04328205011692068, 0.041918587061416615, 0.041055660613429724, 0.04104525006908894, 0.04230776905355231, 0.0451423017081343, 0.04951281880634168, 0.055014611075483555, 0.061011373172066466, 0.0668290668718244, 0.07190595949098424, 0.07587049678320418, 0.07857448638554408, 0.08010874334102, 0.0808083312129632, 0.08123216065267022, 0.08208051409483522, 0.08401329627670519, 0.08739395739107474, 0.09213975599656835, 0.09784249264887553, 0.10398887644866112, 0.11011593473291229, 0.11587972253607881, 0.12106722105179239, 0.12558205100806297, 0.1294215599926439, 0.13265258383782674, 0.13538802899566843, 0.1377646652869698, 0.1399225206715531, 0.14198695153459914 ], [ 0.10885078643317057, 0.10465141880955599, 0.10069957525905153, 0.09687420592361906, 0.09303316561830356, 0.08904640865471092, 0.0848210721080452, 0.08031537554178757, 0.07554559894294433, 0.07059351804868723, 0.06561730474331362, 0.06085534940829195, 0.0565935283552748, 0.05306178914433462, 0.05029312317926125, 0.048109702348725907, 0.04625800946896861, 0.04455155393635459, 0.042947285659880985, 0.041564930471656644, 0.040667168719091595, 0.04063730157545955, 0.04192265765925964, 0.044826554738831785, 0.04928999416252239, 0.054880711300690305, 0.06094719021546837, 0.06681042394152738, 0.07190779467313074, 0.07586865234707497, 0.0785472798589471, 0.08003905266967672, 0.08068667589016133, 0.08106095471996573, 0.08187851175880852, 0.08381719637636288, 0.08725113327822306, 0.0920920592043021, 0.097919841492106, 0.10421059936766462, 0.11049398451971942, 0.11642118163294272, 0.12177536724065012, 0.1264561862999028, 0.13045606704213736, 0.13383570836071085, 0.1367008879667682, 0.1391809572141599, 0.14140930563211948, 0.14350662928595367 ], [ 0.107497057566058, 0.1034370989374487, 0.0996708854563686, 0.09605119758181137, 0.09240668756388681, 0.08858179762759603, 0.08446678620967292, 0.08001387232325477, 0.07524378386646542, 0.070250724202596, 0.06521002566129339, 0.060379380635893896, 0.05606431714766807, 0.05251228398858053, 0.04976781433024236, 0.04764603349578966, 0.04586527892106443, 0.04420736959683301, 0.04261225461422751, 0.04120377864709638, 0.04026802711032962, 0.040222778265834196, 0.04154277459551935, 0.0445326512992031, 0.04910677166520321, 0.05480368254598932, 0.06095627769932308, 0.06688137469055254, 0.07201730595151684, 0.07599625694840086, 0.07867624895947577, 0.08015727072405292, 0.0807878947539193, 0.0811465065579116, 0.0819589772570499, 0.08391268589081254, 0.08738789105504426, 0.09229517711583826, 0.09821015260368057, 0.10460462942867702, 0.1110047390354727, 0.11705865356801844, 0.12254638907238831, 0.127363687922048, 0.13149797334477112, 0.13500378654170583, 0.1379799038986889, 0.1405485073072382, 0.14283660934304512, 0.1449603697614063 ], [ 0.10604226202856204, 0.10211291526945197, 0.09852649528649159, 0.09510887477697838, 0.09165913733282799, 0.08799677085123733, 0.08399660213553062, 0.07960665279116565, 0.07485345270607807, 0.06984366347730157, 0.06476774257342548, 0.05989826443565809, 0.05555427927143365, 0.051995654914633156, 0.0492774279207617, 0.0472105683939991, 0.04548990848588681, 0.04387021356074223, 0.04227712582764028, 0.040839463096231376, 0.03986809688057369, 0.03981687494049871, 0.04118676734102367, 0.044279857665072093, 0.048981081946325715, 0.054799386398550184, 0.061052447404643474, 0.06705393244353254, 0.07224498730066505, 0.0762625663477182, 0.0789696985069117, 0.08047104373048092, 0.08111924990218249, 0.08149588937275233, 0.08232881392636644, 0.08430617696848247, 0.08780930531700946, 0.09275269574992302, 0.09871572086273755, 0.10517229392825156, 0.11164889542144019, 0.11779250723320542, 0.12338058230986632, 0.12830498746980837, 0.1325480069846415, 0.13615794586451116, 0.13922663477651925, 0.14186925247706192, 0.1442066155953379, 0.14635041307183888 ], [ 0.10449455522776928, 0.10068721994438758, 0.0972751265024344, 0.09405636817632056, 0.09079992788415812, 0.08730074478958909, 0.08341953453711017, 0.07910185338660694, 0.07438141854140895, 0.06937752134875563, 0.06429389963058514, 0.059413771049351465, 0.055063828976322345, 0.051511540161907024, 0.04882111607451328, 0.046801944691177495, 0.04513044938866108, 0.0435396529102782, 0.04194402051858057, 0.040478517058833756, 0.03947978502853959, 0.03943773948534929, 0.040876476179534264, 0.044090505467470745, 0.04893341809816278, 0.05488543702806873, 0.06125032545151133, 0.06734003238595435, 0.07260045448057098, 0.07667517688046659, 0.07943340052044622, 0.08098441992457447, 0.08168310047372761, 0.08210979582433722, 0.0829871501927849, 0.08499560215440098, 0.08851300207763195, 0.0934623219961164, 0.09943449751707473, 0.10591186052097497, 0.11242507745611335, 0.11862176126385326, 0.12427740790963547, 0.12928004040512367, 0.1336066618759752, 0.1372992357927118, 0.14044265607274878, 0.14314520387015586, 0.14552161924591225, 0.14767914140524466 ], [ 0.10286438703550371, 0.09917071693618017, 0.09592784670507226, 0.09290508672925694, 0.08984063420031833, 0.08650515285701828, 0.0827464561644623, 0.0785092995212791, 0.07383599755183427, 0.06885877617788791, 0.06379301354359054, 0.05892859741537596, 0.054594363074117316, 0.0510607330614771, 0.048399461691015416, 0.04642063335381158, 0.04478768761845753, 0.043217797292736705, 0.04161784310365694, 0.04013041327255907, 0.03911847214474157, 0.03910643356057755, 0.04063645186398824, 0.04398942616879312, 0.0489862495492705, 0.05508066462819867, 0.06156489915791847, 0.06775116538653193, 0.07309215346656278, 0.07723979609160139, 0.08007042319003202, 0.08169774452200095, 0.0824768839689175, 0.08298263943027358, 0.08392562024395121, 0.08597095674434284, 0.08948949356646872, 0.09441610001586044, 0.10036023002287414, 0.10681862143191892, 0.11332987879929085, 0.11954409679356778, 0.12523548185405242, 0.13028829878819836, 0.1346741583086239, 0.13842858051123644, 0.14162950945480687, 0.14437839433945038, 0.14678397397914153, 0.14894902746053076 ], [ 0.10116468403924767, 0.09757664138101496, 0.0944982292450918, 0.09166885257764135, 0.08879511166954512, 0.08562356619664761, 0.08199024300226944, 0.07784069456715931, 0.07322725942518463, 0.06829553211595825, 0.06327111475632458, 0.05844692632672033, 0.054148890155721253, 0.05064586534879824, 0.04801516063022282, 0.046069587728182086, 0.04446531160389036, 0.04291001659620193, 0.041307015808652654, 0.03980819182660275, 0.038802861607499205, 0.038846833948163156, 0.0404934048691761, 0.04400313748683956, 0.049163160843408577, 0.055404349852633955, 0.062010907673803975, 0.06829793581720515, 0.07372706238146408, 0.07796006781164146, 0.08088107872640747, 0.08260775603962982, 0.08349341405733622, 0.08410313165556887, 0.08512932148226547, 0.0872155671382974, 0.09072312949350815, 0.09560102172111087, 0.10148284016173398, 0.10788511753803717, 0.11435798752454412, 0.12055591799359372, 0.1262525941578951, 0.13132870312812892, 0.13575041732662135, 0.13954674274636805, 0.1427886610493622, 0.14557083449068642, 0.14799604790468812, 0.1501625901223943 ], [ 0.0994110309178265, 0.09592094027030351, 0.09300251291677186, 0.0903640333208441, 0.08767961169007257, 0.08467181528396736, 0.0811659273050079, 0.07710982562257768, 0.07256730178562015, 0.06769787562075859, 0.06273619257805627, 0.057974933977846574, 0.05373253608351838, 0.05027191859376717, 0.04767354038517999, 0.045754769861261174, 0.04417048526455469, 0.042625548281299684, 0.041024053681395244, 0.03952890389902268, 0.038555139601457775, 0.03868538295544497, 0.04047556529058977, 0.044158764641111355, 0.049487732409444714, 0.05587526278870632, 0.06260211582046414, 0.0689895772143038, 0.07451041009123537, 0.07883746398917954, 0.08186299003104018, 0.08370786526505616, 0.08472144275886294, 0.08545521557781023, 0.08657816154806884, 0.08870760158340658, 0.09219339867517098, 0.09699992155443936, 0.10278898940474748, 0.10910147795265289, 0.11550238039314316, 0.12165245504403352, 0.12732575463986584, 0.1323996922665661, 0.13683504836762872, 0.1406542982184324, 0.14392146878804216, 0.1467244767670527, 0.14916018641175416, 0.1513223567868573 ], [ 0.0976218382777312, 0.09422244146025636, 0.09145974742262954, 0.0890096566572523, 0.08651287887193125, 0.08366809611707415, 0.08029084140821978, 0.07633276750487938, 0.07187052749625601, 0.06707823022184399, 0.062198615640630374, 0.05752122585288471, 0.05335291518958222, 0.049946530964469, 0.0473828687895216, 0.045485521328521204, 0.04391428688237318, 0.04237794090909801, 0.040785916993203185, 0.039313782345705796, 0.03840082017710458, 0.03865048738077134, 0.040611645969400846, 0.044482671792665913, 0.04998221348251864, 0.05651056927158055, 0.06335053384188195, 0.06983347182460108, 0.07544543768912977, 0.07987125168177103, 0.08301126545677605, 0.08498857910603919, 0.08614640470786407, 0.08701919855735775, 0.08824834161214522, 0.09042164375228244, 0.09387634504586476, 0.09859251995471216, 0.10426276337178501, 0.11045584172858249, 0.11675457076174726, 0.12282790185121795, 0.12845126180699093, 0.13349922951343024, 0.13792734919124836, 0.14175162005784844, 0.14502915824980436, 0.14784118699912233, 0.15027868170875472, 0.15243083177818817 ], [ 0.09581847822094108, 0.09250299160284002, 0.08989190438716141, 0.0876274867550168, 0.08531620909563575, 0.08263303979485837, 0.07938472994870202, 0.075528060803048, 0.07115389818462663, 0.0664516802993825, 0.06167150023150254, 0.057097189514505556, 0.05302037960422615, 0.049680089321485106, 0.04715443494109546, 0.04527476676591823, 0.043711973826499335, 0.042185278425886714, 0.04061407204587517, 0.039188039784593676, 0.038368160175863926, 0.03877143882472943, 0.04092936526058555, 0.04499885993773781, 0.050666084511977606, 0.05732470017601035, 0.06426566702244768, 0.07083472678047605, 0.07653322819276544, 0.08105853831071741, 0.08431876226209069, 0.08643802028247789, 0.08775125063676156, 0.08877293232773785, 0.09011379751909677, 0.09233018494476936, 0.09574593428552723, 0.10035648977761827, 0.10588640418990528, 0.11193482472137285, 0.11810489157858294, 0.12407557933139765, 0.12962479029798546, 0.1346248427043502, 0.1390263167915243, 0.14283887232409753, 0.14611280631977513, 0.148922722814892, 0.15135374848353558, 0.15349047032408586 ], [ 0.09402536135727471, 0.0907875356539376, 0.08832392614993624, 0.08624203515272316, 0.08411344204318731, 0.08158971887168909, 0.07846980224082742, 0.0747168332447984, 0.07043713049008606, 0.06583622852132165, 0.06117099480287427, 0.05671725027836736, 0.05274817509917582, 0.049485675957498795, 0.047002465840367824, 0.045139051471170026, 0.043583024377095714, 0.04207012087079087, 0.040534191104292294, 0.03918020195312794, 0.03848708063301667, 0.03907690834221318, 0.041453677473156446, 0.045727333907156546, 0.051554653521475806, 0.05832831775762343, 0.06535389234465154, 0.07199585817875694, 0.07777262462642859, 0.0823943923385341, 0.08577641390100514, 0.0880424913658037, 0.08951728448611809, 0.09069292170403331, 0.09214748213606608, 0.09440493118888345, 0.09777526686951726, 0.10226844973858291, 0.10764102809672918, 0.11352399623359741, 0.1195427946549326, 0.12538811449373644, 0.13084149181608626, 0.13577367547185545, 0.14013066773679642, 0.14391601161471504, 0.14717333190608434, 0.1499707182775888, 0.15238750512408406, 0.15450365757819456 ], [ 0.09226992020124343, 0.0891041013352741, 0.08678367608149645, 0.08488047213062494, 0.0829308562940022, 0.08056355865585065, 0.0775706906048853, 0.07392282651529666, 0.06974279355570127, 0.06525294320108153, 0.06071643951369875, 0.056399001926703114, 0.05255251773924645, 0.04937906593683329, 0.046944053692377634, 0.04509839978440019, 0.04355089143556596, 0.042059086459191784, 0.04057543539787631, 0.03932093746746502, 0.03878763377060293, 0.03959326026930027, 0.042205090979457585, 0.04668276565254613, 0.05265788341642064, 0.0595275624815136, 0.0666180548108561, 0.07331662312840795, 0.07916024784297897, 0.0838720307863126, 0.08737359443683893, 0.0897870367825391, 0.09142493753721775, 0.09275528501335131, 0.09432242433764237, 0.09661786822918406, 0.0999375811463701, 0.10430482405577768, 0.10950728100176678, 0.11520833550829505, 0.1210571489850105, 0.12675762581536648, 0.1320961043527549, 0.13694254682574036, 0.1412388662157569, 0.14498279564202543, 0.14821149289771413, 0.1509866740987954, 0.15338195994476106, 0.15547269219845314 ], [ 0.09058245479193724, 0.08748364224566874, 0.08530174582879309, 0.08357239854188336, 0.08179693088851744, 0.07958211777667545, 0.07671427559685247, 0.07317228410975199, 0.06909625825302004, 0.06472594251303898, 0.06033034752275765, 0.056163163559415814, 0.0524525526752873, 0.049378773570798944, 0.04699908325618115, 0.045175935139091156, 0.043642387725073596, 0.04218199722818197, 0.040769291487229894, 0.039641495907698124, 0.039298238040353886, 0.04034299870670649, 0.04319859040780965, 0.04787375716579918, 0.053979872514457974, 0.060923772714452, 0.06805734432567796, 0.07479401979626966, 0.08069061442997003, 0.08548306027456617, 0.08909849475841872, 0.09165596644766542, 0.09345443654081899, 0.09493652875189187, 0.09661254311751337, 0.09894206898846265, 0.10220702821497456, 0.10644253916387447, 0.11146590212186651, 0.11697264573698431, 0.12263652378301561, 0.12817590634978787, 0.13338306472113373, 0.13812801611566153, 0.14234915799302206, 0.14603879759103416, 0.14922788850947202, 0.15197195276297357, 0.1543390018698677, 0.1563997740076416 ], [ 0.08899578876982621, 0.08595968518787647, 0.08391106881314364, 0.08234943513154429, 0.08074193630970172, 0.07867470172809703, 0.07592933728936449, 0.07249365185228099, 0.06852544279046391, 0.0642821551753867, 0.060038144685395145, 0.05603329495519781, 0.05247010113988844, 0.04950587613657333, 0.04718990272288056, 0.04539716521573953, 0.0438866252522652, 0.04247053019680409, 0.041147965670129384, 0.04017196830748852, 0.040043997816774826, 0.04134353693344814, 0.04444313792204391, 0.049302786731333016, 0.05551917902414451, 0.06251372147480959, 0.0696674581634831, 0.07642245078672484, 0.08235634393063691, 0.08721775514294425, 0.09093848903507888, 0.09363331672348339, 0.0955863452528103, 0.09721412996905433, 0.09899322678351682, 0.1013522584897774, 0.10455922793451355, 0.10865955280596357, 0.11349818174216574, 0.11880191136518199, 0.12426944549256516, 0.12963459743336816, 0.1346966199133701, 0.13932645058062143, 0.14345960848570072, 0.1470834250520963, 0.1502229661456498, 0.15292777789780784, 0.15526039503607603, 0.157286995279844 ], [ 0.08754468107630656, 0.08456772399727377, 0.0826462888406772, 0.08124458941373024, 0.07979732266725052, 0.07787177814805009, 0.075245995727682, 0.07191704447580184, 0.06806029930962272, 0.06395079648363046, 0.059867602932375544, 0.05603519850505269, 0.05262909060361316, 0.04978344863061286, 0.047540581131966145, 0.04578885453441456, 0.044313472129725556, 0.042956373617800456, 0.041742395135315774, 0.04093944188805092, 0.04104520783811816, 0.04260635038199747, 0.0459416050650265, 0.05096668556707127, 0.05726955054285529, 0.06429021346305651, 0.07144099638619865, 0.07819402227129069, 0.08414843554729852, 0.08906535497549956, 0.09288047490382469, 0.0957032359589892, 0.09780197670454545, 0.09956694069269706, 0.10144170638306772, 0.10382516921169371, 0.10697163227920353, 0.11093522719112453, 0.11558631172769389, 0.12068159189483557, 0.12594462173805762, 0.13112534752731442, 0.13603093348471107, 0.1405340929456424, 0.14456814325427153, 0.1481159423437497, 0.1511970319213786, 0.15385523723354064, 0.15614777678881075, 0.1581363352143002 ], [ 0.08626494414896536, 0.08334430837779198, 0.08154284238590824, 0.08029137285138889, 0.0789948862744551, 0.07720417594137595, 0.07469491658533155, 0.0714734430536459, 0.06773199645143696, 0.06376251019765164, 0.059847917210820924, 0.05619595560282494, 0.0529546019743581, 0.05023557261060268, 0.04807573551965971, 0.046377488707727316, 0.04495157794006984, 0.04366898840520362, 0.04258003190319696, 0.04196614149044989, 0.0423160900421376, 0.0441365209864647, 0.047691056757993136, 0.052857416571970205, 0.05922080949419312, 0.06624284659870798, 0.07336800398209789, 0.0800989360736191, 0.08605658907224185, 0.09101436449312206, 0.0949111762573547, 0.0978502914018806, 0.10008368575547769, 0.10197544016423792, 0.10393726115968206, 0.10633972730481615, 0.1094237300456171, 0.113250567109728, 0.11771363607064274, 0.12259785138897486, 0.12765112860285696, 0.1326399524513549, 0.137380183962951, 0.1417471268706438, 0.14567258934294625, 0.14913549509945312, 0.1521502640086788, 0.15475528851363504, 0.15700265856563503, 0.15894965717696324 ], [ 0.08519223872888193, 0.08232579727053518, 0.08063573788996918, 0.07952266662015052, 0.07836571984751235, 0.07670207388262637, 0.07430628249409826, 0.07119361293989034, 0.06757177822220559, 0.06374815609943169, 0.060008412191823804, 0.05654259333382873, 0.05347153725927645, 0.05088595896719528, 0.04881900270616167, 0.047187450272490834, 0.04582615343050283, 0.044633202070663576, 0.043682681761115796, 0.04326781254874923, 0.043863916988899965, 0.04593264282664069, 0.04968328409629308, 0.054962973341986854, 0.06135976915659173, 0.0683588031150042, 0.07543657127039763, 0.08212592714618797, 0.08806954359271538, 0.09305284095200879, 0.09701740246647843, 0.1000597011468318, 0.10241505834966995, 0.10442186486254426, 0.10646129491234631, 0.1088771101532133, 0.11189712813168079, 0.11558834756653705, 0.11986481381932661, 0.1245377272258873, 0.12937856047150137, 0.13417047487228476, 0.13873865310577, 0.1429617384544506, 0.1467707160910202, 0.1501411360804685, 0.15308272802038025, 0.15562876774812945, 0.15782642918227466, 0.15972870831047675 ], [ 0.08436055175803873, 0.08154678631063793, 0.07995805870794873, 0.0789693736324867, 0.07793898608056303, 0.07639381776156358, 0.07410856697753067, 0.07110677226462589, 0.06760952213459125, 0.06393727220838505, 0.0603769240219561, 0.05710044983089626, 0.0542029948117053, 0.05175629928316772, 0.0497913107509894, 0.04823913215835613, 0.046956857636771474, 0.045867025327565196, 0.04506476252015178, 0.04485262556279052, 0.04568867768594825, 0.04798704699433129, 0.05190551814930903, 0.05726828783610815, 0.06367109196787939, 0.07062358926688725, 0.0776334223218828, 0.08426270339580882, 0.09017541006879964, 0.09516865739882471, 0.0991862619960694, 0.10231749903285471, 0.10478101655909337, 0.1068902456747695, 0.10899731862247063, 0.11142071150204963, 0.1143755417624024, 0.11793315539834723, 0.12202590893598823, 0.12648924440033285, 0.131117143848627, 0.13570934230026016, 0.1401008026169362, 0.14417417241479075, 0.1478602742514797, 0.1511318512884658, 0.15399439370638132, 0.15647639923890078, 0.15862036006399313, 0.16047512113548884 ], [ 0.0838004169932082, 0.0810382785902067, 0.0795392771532657, 0.07865894414394574, 0.07774059536664005, 0.07630464449849912, 0.0741271926894328, 0.0712390949230374, 0.06787207707752446, 0.06435630514763375, 0.06097797744040814, 0.057891390824941984, 0.055168522548928206, 0.05286452831066852, 0.051009163270240535, 0.049547264989406, 0.048356231766535425, 0.04738027476731494, 0.046732296579338646, 0.0467207733396528, 0.0477833507588911, 0.05028649108117715, 0.05434131593668564, 0.05975608701284085, 0.06613804017881855, 0.07302167935963706, 0.07994444567805765, 0.08649635505309688, 0.09236197897922217, 0.09734973380882207, 0.10140533149491898, 0.10461064246987098, 0.10716785828216574, 0.10936637823364503, 0.11153086903991895, 0.113956044197331, 0.11684472119153373, 0.12027136703747301, 0.12418442181490556, 0.12844148308410835, 0.13285781806743513, 0.13724942396370604, 0.14146133863940724, 0.14538078197498816, 0.14893903248120183, 0.15210658557805254, 0.15488515230964034, 0.15729880685130568, 0.1593856119961968, 0.16119041679026694 ], [ 0.08353700072530629, 0.08082574298145714, 0.07940353659525298, 0.07861391220006739, 0.07779190731071922, 0.07645542863080164, 0.07438319959765205, 0.07161219107253029, 0.06838152566265802, 0.06502676845781792, 0.06183095489549894, 0.05893210287723757, 0.05638248252518383, 0.05422322111490241, 0.05248316443278273, 0.05111970488626937, 0.0500289285570411, 0.04917422129238938, 0.04868276097892823, 0.048864756732047945, 0.050134689908977986, 0.05281333595893773, 0.05697156212667137, 0.06240766457049821, 0.06874309988700945, 0.07553704636567925, 0.08235514391612955, 0.08881371283414072, 0.09461699042663604, 0.09958423175333721, 0.10366278357939623, 0.10692707364788397, 0.10956324857181253, 0.11183774795169352, 0.1140493867172649, 0.11647060408262672, 0.11929233698343024, 0.12259108123679978, 0.12632927612035919, 0.13038460756349485, 0.13459228670097623, 0.1387840877670244, 0.14281526394116625, 0.14657807186818766, 0.15000481049517014, 0.15306426710405038, 0.1557548340101711, 0.15809652605765762, 0.1601232430027745, 0.16187600958134718 ], [ 0.08358823217089935, 0.08092728891904012, 0.07956811769751071, 0.07885061480020279, 0.07810860036189961, 0.07686158970403334, 0.07489207809022584, 0.07224175642399162, 0.06915357913030575, 0.06596353578636271, 0.06294851460602502, 0.0602327219539263, 0.05785278044334172, 0.05583835149588736, 0.05421697343242076, 0.05295684223883683, 0.05197171332323665, 0.051242025631802864, 0.050905703015364685, 0.051270194253853686, 0.05272429642441361, 0.05554676693240496, 0.05977543611773267, 0.06520354391898335, 0.07146847943251394, 0.07815357962132549, 0.08485099536026393, 0.09120164514840604, 0.09692836013918245, 0.10186071141848552, 0.10594747781967935, 0.10925574402949723, 0.11195617690341539, 0.11429342665507665, 0.1165420711974095, 0.11895371265869958, 0.1217078408247654, 0.12488202197198936, 0.1284507728732036, 0.1323098643857008, 0.13631304390304505, 0.1403072391009608, 0.14415791818608376, 0.14776273420549085, 0.15105550839441806, 0.1540038300743774, 0.15660322497053847, 0.15887001633291045, 0.16083421699795747, 0.16253321254516534 ] ] }, { "hoverinfo": "text", "legendgroup": "In-sample", "marker": { "color": "black", "opacity": 0.5, "symbol": 1 }, "mode": "markers", "name": "In-sample", "text": [ "Arm 0_0
accuracy: 0.841833 (SEM: None)
lr: 2.6e-05
momentum: 0.58", "Arm 1_0
accuracy: 0.100333 (SEM: None)
lr: 0.00995509
momentum: 0.633423", "Arm 2_0
accuracy: 0.318667 (SEM: None)
lr: 5.18336e-06
momentum: 0.0228515", "Arm 3_0
accuracy: 0.4585 (SEM: None)
lr: 6.63497e-06
momentum: 0.176948", "Arm 4_0
accuracy: 0.926 (SEM: None)
lr: 8.15482e-05
momentum: 0.90883", "Arm 5_0
accuracy: 0.929 (SEM: None)
lr: 0.000302077
momentum: 0.341904", "Arm 6_0
accuracy: 0.92 (SEM: None)
lr: 0.000137311
momentum: 0.590917", "Arm 7_0
accuracy: 0.860167 (SEM: None)
lr: 1.03262e-05
momentum: 1", "Arm 8_0
accuracy: 0.888833 (SEM: None)
lr: 0.000246295
momentum: 0", "Arm 9_0
accuracy: 0.901667 (SEM: None)
lr: 0.000149439
momentum: 0.286357", "Arm 10_0
accuracy: 0.560333 (SEM: None)
lr: 1e-06
momentum: 1", "Arm 11_0
accuracy: 0.759167 (SEM: None)
lr: 3.73218e-05
momentum: 1", "Arm 12_0
accuracy: 0.945333 (SEM: None)
lr: 0.000261046
momentum: 0.91355", "Arm 13_0
accuracy: 0.942833 (SEM: None)
lr: 0.000170431
momentum: 0.799113", "Arm 14_0
accuracy: 0.842833 (SEM: None)
lr: 0.000158728
momentum: 1", "Arm 15_0
accuracy: 0.945833 (SEM: None)
lr: 0.000310546
momentum: 0.665002", "Arm 16_0
accuracy: 0.870333 (SEM: None)
lr: 0.000836774
momentum: 0", "Arm 17_0
accuracy: 0.943833 (SEM: None)
lr: 0.000572854
momentum: 0.790425", "Arm 18_0
accuracy: 0.912333 (SEM: None)
lr: 0.000563705
momentum: 0.479924", "Arm 19_0
accuracy: 0.101 (SEM: None)
lr: 0.4
momentum: 0", "Arm 20_0
accuracy: 0.203667 (SEM: None)
lr: 0.000314755
momentum: 0.777594", "Arm 21_0
accuracy: 0.881833 (SEM: None)
lr: 3.47393e-05
momentum: 0.76736", "Arm 22_0
accuracy: 0.873167 (SEM: None)
lr: 0.000100187
momentum: 0", "Arm 23_0
accuracy: 0.9015 (SEM: None)
lr: 7.63425e-05
momentum: 0.471701", "Arm 24_0
accuracy: 0.913333 (SEM: None)
lr: 0.00035258
momentum: 0.183083" ], "type": "scatter", "x": [ 2.6e-05, 0.00995509203921593, 5.183362665905935e-06, 6.6349677287433636e-06, 8.154817047148868e-05, 0.000302076728574865, 0.00013731096665846605, 1.0326238819187764e-05, 0.00024629479479540775, 0.0001494385653033887, 1e-06, 3.732184410050388e-05, 0.0002610460173347585, 0.0001704311281854437, 0.00015872762260578225, 0.00031054584535304577, 0.0008367735334414118, 0.0005728536520635165, 0.0005637049589970559, 0.4, 0.0003147550527241065, 3.473928755567143e-05, 0.00010018704633631847, 7.634249037888073e-05, 0.0003525795556828365 ], "xaxis": "x", "y": [ 0.58, 0.6334228515625, 0.022851496934890747, 0.1769482558593154, 0.9088304992765188, 0.34190391190350056, 0.5909173150416861, 1.0, 0.0, 0.2863569208604894, 1.0, 1.0, 0.9135498623676301, 0.7991127998404972, 1.0, 0.665001895649106, 0.0, 0.7904247793656229, 0.479924033515976, 0.0, 0.7775944991992201, 0.7673599449304487, 0.0, 0.4717011028463039, 0.18308293645958593 ], "yaxis": "y" }, { "hoverinfo": "text", "legendgroup": "In-sample", "marker": { "color": "black", "opacity": 0.5, "symbol": 1 }, "mode": "markers", "name": "In-sample", "showlegend": false, "text": [ "Arm 0_0
accuracy: 0.841833 (SEM: None)
lr: 2.6e-05
momentum: 0.58", "Arm 1_0
accuracy: 0.100333 (SEM: None)
lr: 0.00995509
momentum: 0.633423", "Arm 2_0
accuracy: 0.318667 (SEM: None)
lr: 5.18336e-06
momentum: 0.0228515", "Arm 3_0
accuracy: 0.4585 (SEM: None)
lr: 6.63497e-06
momentum: 0.176948", "Arm 4_0
accuracy: 0.926 (SEM: None)
lr: 8.15482e-05
momentum: 0.90883", "Arm 5_0
accuracy: 0.929 (SEM: None)
lr: 0.000302077
momentum: 0.341904", "Arm 6_0
accuracy: 0.92 (SEM: None)
lr: 0.000137311
momentum: 0.590917", "Arm 7_0
accuracy: 0.860167 (SEM: None)
lr: 1.03262e-05
momentum: 1", "Arm 8_0
accuracy: 0.888833 (SEM: None)
lr: 0.000246295
momentum: 0", "Arm 9_0
accuracy: 0.901667 (SEM: None)
lr: 0.000149439
momentum: 0.286357", "Arm 10_0
accuracy: 0.560333 (SEM: None)
lr: 1e-06
momentum: 1", "Arm 11_0
accuracy: 0.759167 (SEM: None)
lr: 3.73218e-05
momentum: 1", "Arm 12_0
accuracy: 0.945333 (SEM: None)
lr: 0.000261046
momentum: 0.91355", "Arm 13_0
accuracy: 0.942833 (SEM: None)
lr: 0.000170431
momentum: 0.799113", "Arm 14_0
accuracy: 0.842833 (SEM: None)
lr: 0.000158728
momentum: 1", "Arm 15_0
accuracy: 0.945833 (SEM: None)
lr: 0.000310546
momentum: 0.665002", "Arm 16_0
accuracy: 0.870333 (SEM: None)
lr: 0.000836774
momentum: 0", "Arm 17_0
accuracy: 0.943833 (SEM: None)
lr: 0.000572854
momentum: 0.790425", "Arm 18_0
accuracy: 0.912333 (SEM: None)
lr: 0.000563705
momentum: 0.479924", "Arm 19_0
accuracy: 0.101 (SEM: None)
lr: 0.4
momentum: 0", "Arm 20_0
accuracy: 0.203667 (SEM: None)
lr: 0.000314755
momentum: 0.777594", "Arm 21_0
accuracy: 0.881833 (SEM: None)
lr: 3.47393e-05
momentum: 0.76736", "Arm 22_0
accuracy: 0.873167 (SEM: None)
lr: 0.000100187
momentum: 0", "Arm 23_0
accuracy: 0.9015 (SEM: None)
lr: 7.63425e-05
momentum: 0.471701", "Arm 24_0
accuracy: 0.913333 (SEM: None)
lr: 0.00035258
momentum: 0.183083" ], "type": "scatter", "x": [ 2.6e-05, 0.00995509203921593, 5.183362665905935e-06, 6.6349677287433636e-06, 8.154817047148868e-05, 0.000302076728574865, 0.00013731096665846605, 1.0326238819187764e-05, 0.00024629479479540775, 0.0001494385653033887, 1e-06, 3.732184410050388e-05, 0.0002610460173347585, 0.0001704311281854437, 0.00015872762260578225, 0.00031054584535304577, 0.0008367735334414118, 0.0005728536520635165, 0.0005637049589970559, 0.4, 0.0003147550527241065, 3.473928755567143e-05, 0.00010018704633631847, 7.634249037888073e-05, 0.0003525795556828365 ], "xaxis": "x2", "y": [ 0.58, 0.6334228515625, 0.022851496934890747, 0.1769482558593154, 0.9088304992765188, 0.34190391190350056, 0.5909173150416861, 1.0, 0.0, 0.2863569208604894, 1.0, 1.0, 0.9135498623676301, 0.7991127998404972, 1.0, 0.665001895649106, 0.0, 0.7904247793656229, 0.479924033515976, 0.0, 0.7775944991992201, 0.7673599449304487, 0.0, 0.4717011028463039, 0.18308293645958593 ], "yaxis": "y2" } ], "layout": { "annotations": [ { "font": { "size": 14 }, "showarrow": false, "text": "Mean", "x": 0.25, "xanchor": "center", "xref": "paper", "y": 1, "yanchor": "bottom", "yref": "paper" }, { "font": { "size": 14 }, "showarrow": false, "text": "Standard Error", "x": 0.8, "xanchor": "center", "xref": "paper", "y": 1, "yanchor": "bottom", "yref": "paper" } ], "autosize": false, "height": 450, "hovermode": "closest", "legend": { "orientation": "h", "x": 0, "y": -0.25 }, "margin": { "b": 100, "l": 35, "pad": 0, "r": 35, "t": 35 }, "template": { "data": { "bar": [ { "error_x": { "color": "#2a3f5f" }, "error_y": { "color": "#2a3f5f" }, "marker": { "line": { "color": "#E5ECF6", "width": 0.5 }, "pattern": { "fillmode": "overlay", "size": 10, "solidity": 0.2 } }, "type": "bar" } ], "barpolar": [ { "marker": { "line": { "color": "#E5ECF6", "width": 0.5 }, "pattern": { "fillmode": "overlay", "size": 10, "solidity": 0.2 } }, "type": "barpolar" } ], "carpet": [ { "aaxis": { "endlinecolor": "#2a3f5f", "gridcolor": "white", "linecolor": "white", "minorgridcolor": "white", "startlinecolor": "#2a3f5f" }, "baxis": { "endlinecolor": "#2a3f5f", "gridcolor": "white", "linecolor": "white", "minorgridcolor": "white", "startlinecolor": "#2a3f5f" }, "type": "carpet" } ], "choropleth": [ { "colorbar": { "outlinewidth": 0, "ticks": "" }, "type": "choropleth" } ], "contour": [ { "colorbar": { "outlinewidth": 0, "ticks": "" }, "colorscale": [ [ 0.0, "#0d0887" ], [ 0.1111111111111111, "#46039f" ], [ 0.2222222222222222, "#7201a8" ], [ 0.3333333333333333, "#9c179e" ], [ 0.4444444444444444, "#bd3786" ], [ 0.5555555555555556, "#d8576b" ], [ 0.6666666666666666, "#ed7953" ], [ 0.7777777777777778, "#fb9f3a" ], [ 0.8888888888888888, "#fdca26" ], [ 1.0, "#f0f921" ] ], "type": "contour" } ], "contourcarpet": [ { "colorbar": { "outlinewidth": 0, "ticks": "" }, "type": "contourcarpet" } ], "heatmap": [ { "colorbar": { "outlinewidth": 0, "ticks": "" }, "colorscale": [ [ 0.0, "#0d0887" ], [ 0.1111111111111111, "#46039f" ], [ 0.2222222222222222, "#7201a8" ], [ 0.3333333333333333, "#9c179e" ], [ 0.4444444444444444, "#bd3786" ], [ 0.5555555555555556, "#d8576b" ], [ 0.6666666666666666, "#ed7953" ], [ 0.7777777777777778, "#fb9f3a" ], [ 0.8888888888888888, "#fdca26" ], [ 1.0, "#f0f921" ] ], "type": "heatmap" } ], "heatmapgl": [ { "colorbar": { "outlinewidth": 0, "ticks": "" }, "colorscale": [ [ 0.0, "#0d0887" ], [ 0.1111111111111111, "#46039f" ], [ 0.2222222222222222, "#7201a8" ], [ 0.3333333333333333, "#9c179e" ], [ 0.4444444444444444, "#bd3786" ], [ 0.5555555555555556, "#d8576b" ], [ 0.6666666666666666, "#ed7953" ], [ 0.7777777777777778, "#fb9f3a" ], [ 0.8888888888888888, "#fdca26" ], [ 1.0, "#f0f921" ] ], "type": "heatmapgl" } ], "histogram": [ { "marker": { "pattern": { "fillmode": "overlay", "size": 10, "solidity": 0.2 } }, "type": "histogram" } ], "histogram2d": [ { "colorbar": { "outlinewidth": 0, "ticks": "" }, "colorscale": [ [ 0.0, "#0d0887" ], [ 0.1111111111111111, "#46039f" ], [ 0.2222222222222222, "#7201a8" ], [ 0.3333333333333333, "#9c179e" ], [ 0.4444444444444444, "#bd3786" ], [ 0.5555555555555556, "#d8576b" ], [ 0.6666666666666666, "#ed7953" ], [ 0.7777777777777778, "#fb9f3a" ], [ 0.8888888888888888, "#fdca26" ], [ 1.0, "#f0f921" ] ], "type": "histogram2d" } ], "histogram2dcontour": [ { "colorbar": { "outlinewidth": 0, "ticks": "" }, "colorscale": [ [ 0.0, "#0d0887" ], [ 0.1111111111111111, "#46039f" ], [ 0.2222222222222222, "#7201a8" ], [ 0.3333333333333333, "#9c179e" ], [ 0.4444444444444444, "#bd3786" ], [ 0.5555555555555556, "#d8576b" ], [ 0.6666666666666666, "#ed7953" ], [ 0.7777777777777778, "#fb9f3a" ], [ 0.8888888888888888, "#fdca26" ], [ 1.0, "#f0f921" ] ], "type": "histogram2dcontour" } ], "mesh3d": [ { "colorbar": { "outlinewidth": 0, "ticks": "" }, "type": "mesh3d" } ], "parcoords": [ { "line": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "type": "parcoords" } ], "pie": [ { "automargin": true, "type": "pie" } ], "scatter": [ { "fillpattern": { "fillmode": "overlay", "size": 10, "solidity": 0.2 }, "type": "scatter" } ], "scatter3d": [ { "line": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "marker": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "type": "scatter3d" } ], "scattercarpet": [ { "marker": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "type": "scattercarpet" } ], "scattergeo": [ { "marker": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "type": "scattergeo" } ], "scattergl": [ { "marker": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "type": "scattergl" } ], "scattermapbox": [ { "marker": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "type": "scattermapbox" } ], "scatterpolar": [ { "marker": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "type": "scatterpolar" } ], "scatterpolargl": [ { "marker": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "type": "scatterpolargl" } ], "scatterternary": [ { "marker": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "type": "scatterternary" } ], "surface": [ { "colorbar": { "outlinewidth": 0, "ticks": "" }, "colorscale": [ [ 0.0, "#0d0887" ], [ 0.1111111111111111, "#46039f" ], [ 0.2222222222222222, "#7201a8" ], [ 0.3333333333333333, "#9c179e" ], [ 0.4444444444444444, "#bd3786" ], [ 0.5555555555555556, "#d8576b" ], [ 0.6666666666666666, "#ed7953" ], [ 0.7777777777777778, "#fb9f3a" ], [ 0.8888888888888888, "#fdca26" ], [ 1.0, "#f0f921" ] ], "type": "surface" } ], "table": [ { "cells": { "fill": { "color": "#EBF0F8" }, "line": { "color": "white" } }, "header": { "fill": { "color": "#C8D4E3" }, "line": { "color": "white" } }, "type": "table" } ] }, "layout": { "annotationdefaults": { "arrowcolor": "#2a3f5f", "arrowhead": 0, "arrowwidth": 1 }, "autotypenumbers": "strict", "coloraxis": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "colorscale": { "diverging": [ [ 0, "#8e0152" ], [ 0.1, "#c51b7d" ], [ 0.2, "#de77ae" ], [ 0.3, "#f1b6da" ], [ 0.4, "#fde0ef" ], [ 0.5, "#f7f7f7" ], [ 0.6, "#e6f5d0" ], [ 0.7, "#b8e186" ], [ 0.8, "#7fbc41" ], [ 0.9, "#4d9221" ], [ 1, "#276419" ] ], "sequential": [ [ 0.0, "#0d0887" ], [ 0.1111111111111111, "#46039f" ], [ 0.2222222222222222, "#7201a8" ], [ 0.3333333333333333, "#9c179e" ], [ 0.4444444444444444, "#bd3786" ], [ 0.5555555555555556, "#d8576b" ], [ 0.6666666666666666, "#ed7953" ], [ 0.7777777777777778, "#fb9f3a" ], [ 0.8888888888888888, "#fdca26" ], [ 1.0, "#f0f921" ] ], "sequentialminus": [ [ 0.0, "#0d0887" ], [ 0.1111111111111111, "#46039f" ], [ 0.2222222222222222, "#7201a8" ], [ 0.3333333333333333, "#9c179e" ], [ 0.4444444444444444, "#bd3786" ], [ 0.5555555555555556, "#d8576b" ], [ 0.6666666666666666, "#ed7953" ], [ 0.7777777777777778, "#fb9f3a" ], [ 0.8888888888888888, "#fdca26" ], [ 1.0, "#f0f921" ] ] }, "colorway": [ "#636efa", "#EF553B", "#00cc96", "#ab63fa", "#FFA15A", "#19d3f3", "#FF6692", "#B6E880", "#FF97FF", "#FECB52" ], "font": { "color": "#2a3f5f" }, "geo": { "bgcolor": "white", "lakecolor": "white", "landcolor": "#E5ECF6", "showlakes": true, "showland": true, "subunitcolor": "white" }, "hoverlabel": { "align": "left" }, "hovermode": "closest", "mapbox": { "style": "light" }, "paper_bgcolor": "white", "plot_bgcolor": "#E5ECF6", "polar": { "angularaxis": { "gridcolor": "white", "linecolor": "white", "ticks": "" }, "bgcolor": "#E5ECF6", "radialaxis": { "gridcolor": "white", "linecolor": "white", "ticks": "" } }, "scene": { "xaxis": { "backgroundcolor": "#E5ECF6", "gridcolor": "white", "gridwidth": 2, "linecolor": "white", "showbackground": true, "ticks": "", "zerolinecolor": "white" }, "yaxis": { "backgroundcolor": "#E5ECF6", "gridcolor": "white", "gridwidth": 2, "linecolor": "white", "showbackground": true, "ticks": "", "zerolinecolor": "white" }, "zaxis": { "backgroundcolor": "#E5ECF6", "gridcolor": "white", "gridwidth": 2, "linecolor": "white", "showbackground": true, "ticks": "", "zerolinecolor": "white" } }, "shapedefaults": { "line": { "color": "#2a3f5f" } }, "ternary": { "aaxis": { "gridcolor": "white", "linecolor": "white", "ticks": "" }, "baxis": { "gridcolor": "white", "linecolor": "white", "ticks": "" }, "bgcolor": "#E5ECF6", "caxis": { "gridcolor": "white", "linecolor": "white", "ticks": "" } }, "title": { "x": 0.05 }, "xaxis": { "automargin": true, "gridcolor": "white", "linecolor": "white", "ticks": "", "title": { "standoff": 15 }, "zerolinecolor": "white", "zerolinewidth": 2 }, "yaxis": { "automargin": true, "gridcolor": "white", "linecolor": "white", "ticks": "", "title": { "standoff": 15 }, "zerolinecolor": "white", "zerolinewidth": 2 } } }, "title": { "text": "accuracy" }, "width": 950, "xaxis": { "anchor": "y", "autorange": false, "domain": [ 0.05, 0.45 ], "exponentformat": "e", "range": [ -6.0, -0.3979400086720376 ], "tickfont": { "size": 11 }, "tickmode": "auto", "title": { "text": "lr" }, "type": "log" }, "xaxis2": { "anchor": "y2", "autorange": false, "domain": [ 0.6, 1 ], "exponentformat": "e", "range": [ -6.0, -0.3979400086720376 ], "tickfont": { "size": 11 }, "tickmode": "auto", "title": { "text": "lr" }, "type": "log" }, "yaxis": { "anchor": "x", "autorange": false, "domain": [ 0, 1 ], "exponentformat": "e", "range": [ 0.0, 1.0 ], "tickfont": { "size": 11 }, "tickmode": "auto", "title": { "text": "momentum" }, "type": "linear" }, "yaxis2": { "anchor": "x2", "autorange": false, "domain": [ 0, 1 ], "exponentformat": "e", "range": [ 0.0, 1.0 ], "tickfont": { "size": 11 }, "tickmode": "auto", "type": "linear" } } }, "text/html": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "render(ax_client.get_contour_plot(param_x=\"lr\", param_y=\"momentum\", metric_name=\"accuracy\"))" ] }, { "cell_type": "markdown", "id": "b2dd9ec1", "metadata": { "customInput": null, "originalKey": "5c91d83a-9a90-4ea0-8df9-9d242d998cb3", "papermill": { "duration": 0.054524, "end_time": "2024-05-13T05:22:13.024062", "exception": false, "start_time": "2024-05-13T05:22:12.969538", "status": "completed" }, "showInput": false, "tags": [] }, "source": [ "Here we plot the optimization trace, showing the progression of finding the point with the optimal objective:\n", "\n" ] }, { "cell_type": "code", "execution_count": 15, "id": "6eaf3ac5", "metadata": { "customInput": null, "customOutput": null, "execution": { "iopub.execute_input": "2024-05-13T05:22:13.133725Z", "iopub.status.busy": "2024-05-13T05:22:13.133435Z", "iopub.status.idle": "2024-05-13T05:22:13.183140Z", "shell.execute_reply": "2024-05-13T05:22:13.182452Z" }, "executionStartTime": 1690415953760, "executionStopTime": 1690415954260, "originalKey": "3a767bdf-7ef3-48e7-b853-6fae5e9e02ff", "papermill": { "duration": 0.106035, "end_time": "2024-05-13T05:22:13.184432", "exception": false, "start_time": "2024-05-13T05:22:13.078397", "status": "completed" }, "requestMsgId": "043de459-6a28-4796-b237-808385c9e54c", "showInput": true, "tags": [] }, "outputs": [ { "data": { "application/vnd.plotly.v1+json": { "config": { "linkText": "Export to plot.ly", "plotlyServerURL": "https://plot.ly", "showLink": false }, "data": [ { "hoverinfo": "none", "legendgroup": "", "line": { "width": 0 }, "mode": "lines", "showlegend": false, "type": "scatter", "x": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26 ], "y": [ 0.8418333333333333, 0.8418333333333333, 0.8418333333333333, 0.8418333333333333, 0.926, 0.929, 0.929, 0.929, 0.929, 0.929, 0.929, 0.929, 0.9453333333333334, 0.9453333333333334, 0.9453333333333334, 0.9458333333333333, 0.9458333333333333, 0.9458333333333333, 0.9458333333333333, 0.9458333333333333, 0.9458333333333333, 0.9458333333333333, 0.9458333333333333, 0.9458333333333333, 0.9458333333333333, 0.9458333333333333 ] }, { "fill": "tonexty", "fillcolor": "rgba(128,177,211,0.3)", "legendgroup": "objective value", "line": { "color": "rgba(128,177,211,1)" }, "mode": "lines", "name": "objective value", "text": [ "
Parameterization:
lr: 2.6e-05
momentum: 0.58", "
Parameterization:
lr: 0.00995509203921593
momentum: 0.6334228515625", "
Parameterization:
lr: 5.183362665905935e-06
momentum: 0.022851496934890747", "
Parameterization:
lr: 6.6349677287433636e-06
momentum: 0.1769482558593154", "
Parameterization:
lr: 8.154817047148868e-05
momentum: 0.9088304992765188", "
Parameterization:
lr: 0.000302076728574865
momentum: 0.34190391190350056", "
Parameterization:
lr: 0.00013731096665846605
momentum: 0.5909173150416861", "
Parameterization:
lr: 1.0326238819187764e-05
momentum: 1.0", "
Parameterization:
lr: 0.00024629479479540775
momentum: 0.0", "
Parameterization:
lr: 0.0001494385653033887
momentum: 0.2863569208604894", "
Parameterization:
lr: 1e-06
momentum: 1.0", "
Parameterization:
lr: 3.732184410050388e-05
momentum: 1.0", "
Parameterization:
lr: 0.0002610460173347585
momentum: 0.9135498623676301", "
Parameterization:
lr: 0.0001704311281854437
momentum: 0.7991127998404972", "
Parameterization:
lr: 0.00015872762260578225
momentum: 1.0", "
Parameterization:
lr: 0.00031054584535304577
momentum: 0.665001895649106", "
Parameterization:
lr: 0.0008367735334414118
momentum: 0.0", "
Parameterization:
lr: 0.0005728536520635165
momentum: 0.7904247793656229", "
Parameterization:
lr: 0.0005637049589970559
momentum: 0.479924033515976", "
Parameterization:
lr: 0.4
momentum: 0.0", "
Parameterization:
lr: 0.0003147550527241065
momentum: 0.7775944991992201", "
Parameterization:
lr: 3.473928755567143e-05
momentum: 0.7673599449304487", "
Parameterization:
lr: 0.00010018704633631847
momentum: 0.0", "
Parameterization:
lr: 7.634249037888073e-05
momentum: 0.4717011028463039", "
Parameterization:
lr: 0.0003525795556828365
momentum: 0.18308293645958593", "
Parameterization:
lr: 0.0010716582482886152
momentum: 1.0" ], "type": "scatter", "x": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26 ], "y": [ 0.8418333333333333, 0.8418333333333333, 0.8418333333333333, 0.8418333333333333, 0.926, 0.929, 0.929, 0.929, 0.929, 0.929, 0.929, 0.929, 0.9453333333333334, 0.9453333333333334, 0.9453333333333334, 0.9458333333333333, 0.9458333333333333, 0.9458333333333333, 0.9458333333333333, 0.9458333333333333, 0.9458333333333333, 0.9458333333333333, 0.9458333333333333, 0.9458333333333333, 0.9458333333333333, 0.9458333333333333 ] }, { "fill": "tonexty", "fillcolor": "rgba(128,177,211,0.3)", "hoverinfo": "none", "legendgroup": "", "line": { "width": 0 }, "mode": "lines", "showlegend": false, "type": "scatter", "x": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26 ], "y": [ 0.8418333333333333, 0.8418333333333333, 0.8418333333333333, 0.8418333333333333, 0.926, 0.929, 0.929, 0.929, 0.929, 0.929, 0.929, 0.929, 0.9453333333333334, 0.9453333333333334, 0.9453333333333334, 0.9458333333333333, 0.9458333333333333, 0.9458333333333333, 0.9458333333333333, 0.9458333333333333, 0.9458333333333333, 0.9458333333333333, 0.9458333333333333, 0.9458333333333333, 0.9458333333333333, 0.9458333333333333 ] } ], "layout": { "showlegend": true, "template": { "data": { "bar": [ { "error_x": { "color": "#2a3f5f" }, "error_y": { "color": "#2a3f5f" }, "marker": { "line": { "color": "#E5ECF6", "width": 0.5 }, "pattern": { "fillmode": "overlay", "size": 10, "solidity": 0.2 } }, "type": "bar" } ], "barpolar": [ { "marker": { "line": { "color": "#E5ECF6", "width": 0.5 }, "pattern": { "fillmode": "overlay", "size": 10, "solidity": 0.2 } }, "type": "barpolar" } ], "carpet": [ { "aaxis": { "endlinecolor": "#2a3f5f", "gridcolor": "white", "linecolor": "white", "minorgridcolor": "white", "startlinecolor": "#2a3f5f" }, "baxis": { "endlinecolor": "#2a3f5f", "gridcolor": "white", "linecolor": "white", "minorgridcolor": "white", "startlinecolor": "#2a3f5f" }, "type": "carpet" } ], "choropleth": [ { "colorbar": { "outlinewidth": 0, "ticks": "" }, "type": "choropleth" } ], "contour": [ { "colorbar": { "outlinewidth": 0, "ticks": "" }, "colorscale": [ [ 0.0, "#0d0887" ], [ 0.1111111111111111, "#46039f" ], [ 0.2222222222222222, "#7201a8" ], [ 0.3333333333333333, "#9c179e" ], [ 0.4444444444444444, "#bd3786" ], [ 0.5555555555555556, "#d8576b" ], [ 0.6666666666666666, "#ed7953" ], [ 0.7777777777777778, "#fb9f3a" ], [ 0.8888888888888888, "#fdca26" ], [ 1.0, "#f0f921" ] ], "type": "contour" } ], "contourcarpet": [ { "colorbar": { "outlinewidth": 0, "ticks": "" }, "type": "contourcarpet" } ], "heatmap": [ { "colorbar": { "outlinewidth": 0, "ticks": "" }, "colorscale": [ [ 0.0, "#0d0887" ], [ 0.1111111111111111, "#46039f" ], [ 0.2222222222222222, "#7201a8" ], [ 0.3333333333333333, "#9c179e" ], [ 0.4444444444444444, "#bd3786" ], [ 0.5555555555555556, "#d8576b" ], [ 0.6666666666666666, "#ed7953" ], [ 0.7777777777777778, "#fb9f3a" ], [ 0.8888888888888888, "#fdca26" ], [ 1.0, "#f0f921" ] ], "type": "heatmap" } ], "heatmapgl": [ { "colorbar": { "outlinewidth": 0, "ticks": "" }, "colorscale": [ [ 0.0, "#0d0887" ], [ 0.1111111111111111, "#46039f" ], [ 0.2222222222222222, "#7201a8" ], [ 0.3333333333333333, "#9c179e" ], [ 0.4444444444444444, "#bd3786" ], [ 0.5555555555555556, "#d8576b" ], [ 0.6666666666666666, "#ed7953" ], [ 0.7777777777777778, "#fb9f3a" ], [ 0.8888888888888888, "#fdca26" ], [ 1.0, "#f0f921" ] ], "type": "heatmapgl" } ], "histogram": [ { "marker": { "pattern": { "fillmode": "overlay", "size": 10, "solidity": 0.2 } }, "type": "histogram" } ], "histogram2d": [ { "colorbar": { "outlinewidth": 0, "ticks": "" }, "colorscale": [ [ 0.0, "#0d0887" ], [ 0.1111111111111111, "#46039f" ], [ 0.2222222222222222, "#7201a8" ], [ 0.3333333333333333, "#9c179e" ], [ 0.4444444444444444, "#bd3786" ], [ 0.5555555555555556, "#d8576b" ], [ 0.6666666666666666, "#ed7953" ], [ 0.7777777777777778, "#fb9f3a" ], [ 0.8888888888888888, "#fdca26" ], [ 1.0, "#f0f921" ] ], "type": "histogram2d" } ], "histogram2dcontour": [ { "colorbar": { "outlinewidth": 0, "ticks": "" }, "colorscale": [ [ 0.0, "#0d0887" ], [ 0.1111111111111111, "#46039f" ], [ 0.2222222222222222, "#7201a8" ], [ 0.3333333333333333, "#9c179e" ], [ 0.4444444444444444, "#bd3786" ], [ 0.5555555555555556, "#d8576b" ], [ 0.6666666666666666, "#ed7953" ], [ 0.7777777777777778, "#fb9f3a" ], [ 0.8888888888888888, "#fdca26" ], [ 1.0, "#f0f921" ] ], "type": "histogram2dcontour" } ], "mesh3d": [ { "colorbar": { "outlinewidth": 0, "ticks": "" }, "type": "mesh3d" } ], "parcoords": [ { "line": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "type": "parcoords" } ], "pie": [ { "automargin": true, "type": "pie" } ], "scatter": [ { "fillpattern": { "fillmode": "overlay", "size": 10, "solidity": 0.2 }, "type": "scatter" } ], "scatter3d": [ { "line": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "marker": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "type": "scatter3d" } ], "scattercarpet": [ { "marker": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "type": "scattercarpet" } ], "scattergeo": [ { "marker": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "type": "scattergeo" } ], "scattergl": [ { "marker": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "type": "scattergl" } ], "scattermapbox": [ { "marker": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "type": "scattermapbox" } ], "scatterpolar": [ { "marker": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "type": "scatterpolar" } ], "scatterpolargl": [ { "marker": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "type": "scatterpolargl" } ], "scatterternary": [ { "marker": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "type": "scatterternary" } ], "surface": [ { "colorbar": { "outlinewidth": 0, "ticks": "" }, "colorscale": [ [ 0.0, "#0d0887" ], [ 0.1111111111111111, "#46039f" ], [ 0.2222222222222222, "#7201a8" ], [ 0.3333333333333333, "#9c179e" ], [ 0.4444444444444444, "#bd3786" ], [ 0.5555555555555556, "#d8576b" ], [ 0.6666666666666666, "#ed7953" ], [ 0.7777777777777778, "#fb9f3a" ], [ 0.8888888888888888, "#fdca26" ], [ 1.0, "#f0f921" ] ], "type": "surface" } ], "table": [ { "cells": { "fill": { "color": "#EBF0F8" }, "line": { "color": "white" } }, "header": { "fill": { "color": "#C8D4E3" }, "line": { "color": "white" } }, "type": "table" } ] }, "layout": { "annotationdefaults": { "arrowcolor": "#2a3f5f", "arrowhead": 0, "arrowwidth": 1 }, "autotypenumbers": "strict", "coloraxis": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "colorscale": { "diverging": [ [ 0, "#8e0152" ], [ 0.1, "#c51b7d" ], [ 0.2, "#de77ae" ], [ 0.3, "#f1b6da" ], [ 0.4, "#fde0ef" ], [ 0.5, "#f7f7f7" ], [ 0.6, "#e6f5d0" ], [ 0.7, "#b8e186" ], [ 0.8, "#7fbc41" ], [ 0.9, "#4d9221" ], [ 1, "#276419" ] ], "sequential": [ [ 0.0, "#0d0887" ], [ 0.1111111111111111, "#46039f" ], [ 0.2222222222222222, "#7201a8" ], [ 0.3333333333333333, "#9c179e" ], [ 0.4444444444444444, "#bd3786" ], [ 0.5555555555555556, "#d8576b" ], [ 0.6666666666666666, "#ed7953" ], [ 0.7777777777777778, "#fb9f3a" ], [ 0.8888888888888888, "#fdca26" ], [ 1.0, "#f0f921" ] ], "sequentialminus": [ [ 0.0, "#0d0887" ], [ 0.1111111111111111, "#46039f" ], [ 0.2222222222222222, "#7201a8" ], [ 0.3333333333333333, "#9c179e" ], [ 0.4444444444444444, "#bd3786" ], [ 0.5555555555555556, "#d8576b" ], [ 0.6666666666666666, "#ed7953" ], [ 0.7777777777777778, "#fb9f3a" ], [ 0.8888888888888888, "#fdca26" ], [ 1.0, "#f0f921" ] ] }, "colorway": [ "#636efa", "#EF553B", "#00cc96", "#ab63fa", "#FFA15A", "#19d3f3", "#FF6692", "#B6E880", "#FF97FF", "#FECB52" ], "font": { "color": "#2a3f5f" }, "geo": { "bgcolor": "white", "lakecolor": "white", "landcolor": "#E5ECF6", "showlakes": true, "showland": true, "subunitcolor": "white" }, "hoverlabel": { "align": "left" }, "hovermode": "closest", "mapbox": { "style": "light" }, "paper_bgcolor": "white", "plot_bgcolor": "#E5ECF6", "polar": { "angularaxis": { "gridcolor": "white", "linecolor": "white", "ticks": "" }, "bgcolor": "#E5ECF6", "radialaxis": { "gridcolor": "white", "linecolor": "white", "ticks": "" } }, "scene": { "xaxis": { "backgroundcolor": "#E5ECF6", "gridcolor": "white", "gridwidth": 2, "linecolor": "white", "showbackground": true, "ticks": "", "zerolinecolor": "white" }, "yaxis": { "backgroundcolor": "#E5ECF6", "gridcolor": "white", "gridwidth": 2, "linecolor": "white", "showbackground": true, "ticks": "", "zerolinecolor": "white" }, "zaxis": { "backgroundcolor": "#E5ECF6", "gridcolor": "white", "gridwidth": 2, "linecolor": "white", "showbackground": true, "ticks": "", "zerolinecolor": "white" } }, "shapedefaults": { "line": { "color": "#2a3f5f" } }, "ternary": { "aaxis": { "gridcolor": "white", "linecolor": "white", "ticks": "" }, "baxis": { "gridcolor": "white", "linecolor": "white", "ticks": "" }, "bgcolor": "#E5ECF6", "caxis": { "gridcolor": "white", "linecolor": "white", "ticks": "" } }, "title": { "x": 0.05 }, "xaxis": { "automargin": true, "gridcolor": "white", "linecolor": "white", "ticks": "", "title": { "standoff": 15 }, "zerolinecolor": "white", "zerolinewidth": 2 }, "yaxis": { "automargin": true, "gridcolor": "white", "linecolor": "white", "ticks": "", "title": { "standoff": 15 }, "zerolinecolor": "white", "zerolinewidth": 2 } } }, "title": { "text": "Best objective found vs. # of iterations" }, "xaxis": { "title": { "text": "Iteration" } }, "yaxis": { "title": { "text": "Accuracy" } } } }, "text/html": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "render(\n", " ax_client.get_optimization_trace()\n", ") " ] }, { "cell_type": "markdown", "id": "e2a64675", "metadata": { "customInput": null, "executionStartTime": 1689617061294, "executionStopTime": 1689617061325, "originalKey": "09aaec9d-c178-42e2-b549-663cd17f8c3d", "papermill": { "duration": 0.057662, "end_time": "2024-05-13T05:22:13.300257", "exception": false, "start_time": "2024-05-13T05:22:13.242595", "status": "completed" }, "requestMsgId": "09aaec9d-c178-42e2-b549-663cd17f8c3d", "showInput": false, "tags": [] }, "source": [ "## 8. Train CNN with best hyperparameters and evaluate on test set\n", "Note that the resulting accuracy on the test set generally won't be the same as the maximum accuracy achieved on the evaluation set throughout optimization. " ] }, { "cell_type": "code", "execution_count": 16, "id": "e3c8ec7c", "metadata": { "customInput": null, "customOutput": null, "execution": { "iopub.execute_input": "2024-05-13T05:22:13.418019Z", "iopub.status.busy": "2024-05-13T05:22:13.417726Z", "iopub.status.idle": "2024-05-13T05:22:13.441055Z", "shell.execute_reply": "2024-05-13T05:22:13.440520Z" }, "executionStartTime": 1690415954397, "executionStopTime": 1690415954452, "originalKey": "27f92d16-93c4-43bb-a37f-e7a1aeecd856", "papermill": { "duration": 0.083546, "end_time": "2024-05-13T05:22:13.442335", "exception": false, "start_time": "2024-05-13T05:22:13.358789", "status": "completed" }, "requestMsgId": "07eba5ce-bebe-4588-8dbb-07553efeb2b0", "showInput": true, "tags": [] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "[WARNING 05-13 05:22:13] ax.service.utils.report_utils: Column reason missing for all trials. Not appending column.\n" ] }, { "data": { "text/plain": [ "{'lr': 0.00031054584535304577, 'momentum': 0.665001895649106}" ] }, "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df = ax_client.get_trials_data_frame()\n", "best_arm_idx = df.trial_index[df[\"accuracy\"] == df[\"accuracy\"].max()].values[0]\n", "best_arm = ax_client.get_trial_parameters(best_arm_idx)\n", "best_arm" ] }, { "cell_type": "code", "execution_count": 17, "id": "8cdf96f7", "metadata": { "customInput": null, "customOutput": null, "execution": { "iopub.execute_input": "2024-05-13T05:22:13.559639Z", "iopub.status.busy": "2024-05-13T05:22:13.559132Z", "iopub.status.idle": "2024-05-13T05:22:13.563464Z", "shell.execute_reply": "2024-05-13T05:22:13.562902Z" }, "executionStartTime": 1690415954677, "executionStopTime": 1690415954681, "originalKey": "d0c7c645-c230-4654-a3b5-a01c61a09393", "papermill": { "duration": 0.064086, "end_time": "2024-05-13T05:22:13.564717", "exception": false, "start_time": "2024-05-13T05:22:13.500631", "status": "completed" }, "requestMsgId": "0a962cef-65a1-4f95-9410-37a9a8e5c5ac", "showInput": true, "tags": [] }, "outputs": [], "source": [ "combined_train_valid_set = torch.utils.data.ConcatDataset(\n", " [\n", " train_loader.dataset.dataset,\n", " valid_loader.dataset.dataset,\n", " ]\n", ")\n", "combined_train_valid_loader = torch.utils.data.DataLoader(\n", " combined_train_valid_set,\n", " batch_size=BATCH_SIZE,\n", " shuffle=True,\n", ")" ] }, { "cell_type": "code", "execution_count": 18, "id": "2a8744c4", "metadata": { "customInput": null, "customOutput": null, "execution": { "iopub.execute_input": "2024-05-13T05:22:13.681951Z", "iopub.status.busy": "2024-05-13T05:22:13.681419Z", "iopub.status.idle": "2024-05-13T05:22:43.127141Z", "shell.execute_reply": "2024-05-13T05:22:43.126469Z" }, "executionStartTime": 1690415954791, "executionStopTime": 1690416061340, "originalKey": "5695c78b-4c6e-4d35-ab08-6c60781bd8f1", "papermill": { "duration": 29.506064, "end_time": "2024-05-13T05:22:43.128802", "exception": false, "start_time": "2024-05-13T05:22:13.622738", "status": "completed" }, "requestMsgId": "e22fa0c7-88cc-4d8a-bb7d-4f96fbae9a42", "showInput": true, "tags": [] }, "outputs": [], "source": [ "net = train(\n", " net=CNN(),\n", " train_loader=combined_train_valid_loader,\n", " parameters=best_arm,\n", " dtype=dtype,\n", " device=device,\n", ")\n", "test_accuracy = evaluate(\n", " net=net,\n", " data_loader=test_loader,\n", " dtype=dtype,\n", " device=device,\n", ")" ] }, { "cell_type": "code", "execution_count": 19, "id": "91b0f0a5", "metadata": { "customInput": null, "customOutput": null, "execution": { "iopub.execute_input": "2024-05-13T05:22:43.246271Z", "iopub.status.busy": "2024-05-13T05:22:43.245757Z", "iopub.status.idle": "2024-05-13T05:22:43.249763Z", "shell.execute_reply": "2024-05-13T05:22:43.249110Z" }, "executionStartTime": 1690416061460, "executionStopTime": 1690416061467, "originalKey": "7522e229-9641-4383-a892-12c3f0a8011c", "papermill": { "duration": 0.064311, "end_time": "2024-05-13T05:22:43.251106", "exception": false, "start_time": "2024-05-13T05:22:43.186795", "status": "completed" }, "requestMsgId": "5552d77d-9c9d-4712-9256-2cb3da836f2c", "showInput": true, "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Classification Accuracy (test set): 98.22%\n" ] } ], "source": [ "print(f\"Classification Accuracy (test set): {round(test_accuracy*100, 2)}%\")" ] }, { "cell_type": "markdown", "id": "71efca0f", "metadata": { "customInput": null, "originalKey": "c8232211-4837-4677-b86c-bce730635fff", "papermill": { "duration": 0.057973, "end_time": "2024-05-13T05:22:43.367371", "exception": false, "start_time": "2024-05-13T05:22:43.309398", "status": "completed" }, "showInput": false, "tags": [] }, "source": [ "## 9. Save / reload optimization to JSON / SQL\n", "We can serialize the state of optimization to JSON and save it to a `.json` file or save it to the SQL backend. For the former:" ] }, { "cell_type": "code", "execution_count": 20, "id": "1b559f84", "metadata": { "customInput": null, "customOutput": null, "execution": { "iopub.execute_input": "2024-05-13T05:22:43.484185Z", "iopub.status.busy": "2024-05-13T05:22:43.483610Z", "iopub.status.idle": "2024-05-13T05:22:43.510487Z", "shell.execute_reply": "2024-05-13T05:22:43.509911Z" }, "executionStartTime": 1690416061571, "executionStopTime": 1690416061657, "originalKey": "6afddb45-c980-4b14-b5e9-927747ea98ea", "papermill": { "duration": 0.086913, "end_time": "2024-05-13T05:22:43.511777", "exception": false, "start_time": "2024-05-13T05:22:43.424864", "status": "completed" }, "requestMsgId": "bab02be8-706c-4422-b97b-c222b5084bba", "showInput": true, "tags": [] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:22:43] ax.service.ax_client: Saved JSON-serialized state of optimization to `ax_client_snapshot.json`.\n" ] } ], "source": [ "ax_client.save_to_json_file() # For custom filepath, pass `filepath` argument." ] }, { "cell_type": "code", "execution_count": 21, "id": "e58e83bb", "metadata": { "customInput": null, "customOutput": null, "execution": { "iopub.execute_input": "2024-05-13T05:22:43.629985Z", "iopub.status.busy": "2024-05-13T05:22:43.629480Z", "iopub.status.idle": "2024-05-13T05:22:43.758087Z", "shell.execute_reply": "2024-05-13T05:22:43.757387Z" }, "executionStartTime": 1690416061758, "executionStopTime": 1690416062132, "originalKey": "31e6f7b4-cf6b-4967-95ff-f76d03657fb2", "papermill": { "duration": 0.189834, "end_time": "2024-05-13T05:22:43.759553", "exception": false, "start_time": "2024-05-13T05:22:43.569719", "status": "completed" }, "requestMsgId": "f2d10848-f995-420d-88e7-9036894d7b1b", "showInput": true, "tags": [] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:22:43] ax.service.ax_client: Starting optimization with verbose logging. To disable logging, set the `verbose_logging` argument to `False`. Note that float values in the logs are rounded to 6 decimal points.\n" ] } ], "source": [ "restored_ax_client = (\n", " AxClient.load_from_json_file()\n", ") # For custom filepath, pass `filepath` argument." ] }, { "cell_type": "markdown", "id": "abde618f", "metadata": { "customInput": null, "originalKey": "122510f5-5b9e-4b1c-9f5e-8c8ea2e08848", "papermill": { "duration": 0.057583, "end_time": "2024-05-13T05:22:43.875942", "exception": false, "start_time": "2024-05-13T05:22:43.818359", "status": "completed" }, "showInput": false, "tags": [] }, "source": [ "To store state of optimization to an SQL backend, first follow [setup instructions](https://ax.dev/docs/storage.html#sql) on Ax website." ] }, { "cell_type": "markdown", "id": "e06125c0", "metadata": { "customInput": null, "originalKey": "bd80e639-aa0f-4dc1-8542-0caf0d674fda", "papermill": { "duration": 0.05804, "end_time": "2024-05-13T05:22:43.992232", "exception": false, "start_time": "2024-05-13T05:22:43.934192", "status": "completed" }, "showInput": false, "tags": [] }, "source": [ "Having set up the SQL backend, pass `DBSettings` to `AxClient` on instantiation (note that `SQLAlchemy` dependency will have to be installed – for installation, refer to [optional dependencies](https://ax.dev/docs/installation.html#optional-dependencies) on Ax website):" ] }, { "cell_type": "code", "execution_count": 22, "id": "5851a948", "metadata": { "customInput": null, "customOutput": null, "execution": { "iopub.execute_input": "2024-05-13T05:22:44.110188Z", "iopub.status.busy": "2024-05-13T05:22:44.109668Z", "iopub.status.idle": "2024-05-13T05:22:44.120053Z", "shell.execute_reply": "2024-05-13T05:22:44.119534Z" }, "executionStartTime": 1690416062222, "executionStopTime": 1690416062314, "originalKey": "80eb6a2e-6564-405e-b5d4-d448e32dbf60", "papermill": { "duration": 0.071101, "end_time": "2024-05-13T05:22:44.121271", "exception": false, "start_time": "2024-05-13T05:22:44.050170", "status": "completed" }, "requestMsgId": "65f2307f-b800-4415-b9e7-11734a2a6889", "showInput": true, "tags": [] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:22:44] ax.service.ax_client: Starting optimization with verbose logging. To disable logging, set the `verbose_logging` argument to `False`. Note that float values in the logs are rounded to 6 decimal points.\n" ] } ], "source": [ "from ax.storage.sqa_store.structs import DBSettings\n", "\n", "# URL is of the form \"dialect+driver://username:password@host:port/database\".\n", "db_settings = DBSettings(url=\"sqlite:///foo.db\")\n", "# Instead of URL, can provide a `creator function`; can specify custom encoders/decoders if necessary.\n", "new_ax = AxClient(db_settings=db_settings)" ] }, { "cell_type": "markdown", "id": "8388ee9b", "metadata": { "customInput": null, "originalKey": "adafd3aa-b84e-4e86-9694-a29f94c6d5f3", "papermill": { "duration": 0.057589, "end_time": "2024-05-13T05:22:44.236784", "exception": false, "start_time": "2024-05-13T05:22:44.179195", "status": "completed" }, "showInput": false, "tags": [] }, "source": [ "When valid `DBSettings` are passed into `AxClient`, a unique experiment name is a required argument (`name`) to `ax_client.create_experiment`. The **state of the optimization is auto-saved** any time it changes (i.e. a new trial is added or completed, etc). \n", "\n", "To reload an optimization state later, instantiate `AxClient` with the same `DBSettings` and use `ax_client.load_experiment_from_database(experiment_name=\"my_experiment\")`." ] }, { "cell_type": "markdown", "id": "edd8a4f7", "metadata": { "customInput": null, "originalKey": "2f4a875b-1e18-4352-955d-576d6b01c5ed", "papermill": { "duration": 0.057525, "end_time": "2024-05-13T05:22:44.352272", "exception": false, "start_time": "2024-05-13T05:22:44.294747", "status": "completed" }, "showInput": false, "tags": [] }, "source": [ "# Special Cases" ] }, { "cell_type": "markdown", "id": "01c9d333", "metadata": { "customInput": null, "originalKey": "0d49e448-4768-401d-ac1d-810aee633c9a", "papermill": { "duration": 0.057521, "end_time": "2024-05-13T05:22:44.467322", "exception": false, "start_time": "2024-05-13T05:22:44.409801", "status": "completed" }, "showInput": false, "tags": [] }, "source": [ "**Evaluation failure**: should any optimization iterations fail during evaluation, `log_trial_failure` will ensure that the same trial is not proposed again." ] }, { "cell_type": "code", "execution_count": 23, "id": "cc6fc8d1", "metadata": { "customInput": null, "customOutput": null, "execution": { "iopub.execute_input": "2024-05-13T05:22:44.584465Z", "iopub.status.busy": "2024-05-13T05:22:44.583781Z", "iopub.status.idle": "2024-05-13T05:22:45.404500Z", "shell.execute_reply": "2024-05-13T05:22:45.403884Z" }, "executionStartTime": 1690416062420, "executionStopTime": 1690416064316, "originalKey": "faa83f1d-31da-481a-96e4-ccbc12f30b91", "papermill": { "duration": 0.88101, "end_time": "2024-05-13T05:22:45.406023", "exception": false, "start_time": "2024-05-13T05:22:44.525013", "status": "completed" }, "requestMsgId": "80a40c3a-76ed-4e1d-aa77-3652fadbe69f", "showInput": true, "tags": [] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:22:45] ax.service.ax_client: Generated new trial 26 with parameters {'lr': 7.7e-05, 'momentum': 0.643066} using model BoTorch.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:22:45] ax.service.ax_client: Registered failure of trial 26.\n" ] } ], "source": [ "_, trial_index = ax_client.get_next_trial()\n", "ax_client.log_trial_failure(trial_index=trial_index)" ] }, { "cell_type": "markdown", "id": "e48b34f7", "metadata": { "customInput": null, "originalKey": "c826a96e-9431-49bd-87d7-62b517537a15", "papermill": { "duration": 0.057858, "end_time": "2024-05-13T05:22:45.522557", "exception": false, "start_time": "2024-05-13T05:22:45.464699", "status": "completed" }, "showInput": false, "tags": [] }, "source": [ "**Need to run many trials in parallel**: for optimal results and optimization efficiency, we strongly recommend sequential optimization (generating a few trials, then waiting for them to be completed with evaluation data). However, if your use case needs to dispatch many trials in parallel before they are updated with data and you are running into the *\"All trials for current model have been generated, but not enough data has been observed to fit next model\"* error, instantiate `AxClient` as `AxClient(enforce_sequential_optimization=False)`." ] }, { "cell_type": "markdown", "id": "7293463b", "metadata": { "customInput": null, "originalKey": "683378e0-893b-49a1-b090-084dc394da1a", "papermill": { "duration": 0.057463, "end_time": "2024-05-13T05:22:45.637658", "exception": false, "start_time": "2024-05-13T05:22:45.580195", "status": "completed" }, "showInput": false, "tags": [] }, "source": [ "# Service API Exceptions Meaning and Handling\n", "[**`DataRequiredError`**](https://ax.dev/api/exceptions.html#ax.exceptions.core.DataRequiredError): Ax generation strategy needs to be updated with more data to proceed to the next optimization model. When the optimization moves from initialization stage to the Bayesian optimization stage, the underlying BayesOpt model needs sufficient data to train. For optimal results and optimization efficiency (finding the optimal point in the least number of trials), we recommend sequential optimization (generating a few trials, then waiting for them to be completed with evaluation data). Therefore, the correct way to handle this exception is to wait until more trial evaluations complete and log their data via `ax_client.complete_trial(...)`. \n", "\n", "However, if there is strong need to generate more trials before more data is available, instantiate `AxClient` as `AxClient(enforce_sequential_optimization=False)`. With this setting, as many trials will be generated from the initialization stage as requested, and the optimization will move to the BayesOpt stage whenever enough trials are completed." ] }, { "cell_type": "markdown", "id": "bdc872a0", "metadata": { "customInput": null, "originalKey": "4602d41d-43aa-46d2-9ca6-392c414d0b5f", "papermill": { "duration": 0.05788, "end_time": "2024-05-13T05:22:45.753893", "exception": false, "start_time": "2024-05-13T05:22:45.696013", "status": "completed" }, "showInput": false, "tags": [] }, "source": [ "[**`MaxParallelismReachedException`**](https://ax.dev/api/modelbridge.html#ax.modelbridge.generation_strategy.MaxParallelismReachedException): generation strategy restricts the number of trials that can be run simultaneously (to encourage sequential optimization), and the parallelism limit has been reached. The correct way to handle this exception is the same as `DataRequiredError` – to wait until more trial evluations complete and log their data via `ax_client.complete_trial(...)`.\n", " \n", "In some cases higher parallelism is important, so `enforce_sequential_optimization=False` kwarg to AxClient allows the user to suppress limiting of parallelism. It's also possible to override the default parallelism setting for all stages of the optimization by passing `choose_generation_strategy_kwargs` to `ax_client.create_experiment`:" ] }, { "cell_type": "code", "execution_count": 24, "id": "6b7a6a9e", "metadata": { "customInput": null, "customOutput": null, "execution": { "iopub.execute_input": "2024-05-13T05:22:45.873899Z", "iopub.status.busy": "2024-05-13T05:22:45.873255Z", "iopub.status.idle": "2024-05-13T05:22:45.884489Z", "shell.execute_reply": "2024-05-13T05:22:45.883862Z" }, "executionStartTime": 1690416064534, "executionStopTime": 1690416064564, "originalKey": "d62e6cfd-5127-450e-80b7-d0edcaf97d6c", "papermill": { "duration": 0.072764, "end_time": "2024-05-13T05:22:45.885890", "exception": false, "start_time": "2024-05-13T05:22:45.813126", "status": "completed" }, "requestMsgId": "cb9a17f9-5734-41c6-9018-c0635c61d8b3", "showInput": true, "tags": [] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:22:45] ax.service.ax_client: Starting optimization with verbose logging. To disable logging, set the `verbose_logging` argument to `False`. Note that float values in the logs are rounded to 6 decimal points.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:22:45] ax.service.utils.instantiation: Inferred value type of ParameterType.FLOAT for parameter x. If that is not the expected value type, you can explicitly specify 'value_type' ('int', 'float', 'bool' or 'str') in parameter dict.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:22:45] ax.service.utils.instantiation: Inferred value type of ParameterType.FLOAT for parameter y. If that is not the expected value type, you can explicitly specify 'value_type' ('int', 'float', 'bool' or 'str') in parameter dict.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:22:45] ax.service.utils.instantiation: Created search space: SearchSpace(parameters=[RangeParameter(name='x', parameter_type=FLOAT, range=[-5.0, 10.0]), RangeParameter(name='y', parameter_type=FLOAT, range=[0.0, 15.0])], parameter_constraints=[]).\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:22:45] ax.modelbridge.dispatch_utils: Using Models.BOTORCH_MODULAR since there is at least one ordered parameter and there are no unordered categorical parameters.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:22:45] ax.modelbridge.dispatch_utils: Calculating the number of remaining initialization trials based on num_initialization_trials=None max_initialization_trials=None num_tunable_parameters=2 num_trials=None use_batch_trials=False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:22:45] ax.modelbridge.dispatch_utils: calculated num_initialization_trials=5\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:22:45] ax.modelbridge.dispatch_utils: num_completed_initialization_trials=0 num_remaining_initialization_trials=5\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:22:45] ax.modelbridge.dispatch_utils: `verbose`, `disable_progbar`, and `jit_compile` are not yet supported when using `choose_generation_strategy` with ModularBoTorchModel, dropping these arguments.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 05-13 05:22:45] ax.modelbridge.dispatch_utils: Using Bayesian Optimization generation strategy: GenerationStrategy(name='Sobol+BoTorch', steps=[Sobol for 5 trials, BoTorch for subsequent trials]). Iterations after 5 will take longer to generate due to model-fitting.\n" ] } ], "source": [ "ax_client = AxClient()\n", "ax_client.create_experiment(\n", " parameters=[\n", " {\"name\": \"x\", \"type\": \"range\", \"bounds\": [-5.0, 10.0]},\n", " {\"name\": \"y\", \"type\": \"range\", \"bounds\": [0.0, 15.0]},\n", " ],\n", " # Sets max parallelism to 10 for all steps of the generation strategy.\n", " choose_generation_strategy_kwargs={\"max_parallelism_override\": 10},\n", ")" ] }, { "cell_type": "code", "execution_count": 25, "id": "c0afe95d", "metadata": { "customInput": null, "customOutput": null, "execution": { "iopub.execute_input": "2024-05-13T05:22:46.006374Z", "iopub.status.busy": "2024-05-13T05:22:46.006039Z", "iopub.status.idle": "2024-05-13T05:22:46.010722Z", "shell.execute_reply": "2024-05-13T05:22:46.010184Z" }, "executionStartTime": 1690416064679, "executionStopTime": 1690416064702, "originalKey": "bc15d2cf-8ddc-4d66-83b6-7469cd15aa4d", "papermill": { "duration": 0.066723, "end_time": "2024-05-13T05:22:46.012061", "exception": false, "start_time": "2024-05-13T05:22:45.945338", "status": "completed" }, "requestMsgId": "996c4bd3-b296-4cf9-8f95-cbf488639c2f", "showInput": true, "tags": [] }, "outputs": [ { "data": { "text/plain": [ "[(5, 10), (-1, 10)]" ] }, "execution_count": 25, "metadata": {}, "output_type": "execute_result" } ], "source": [ "ax_client.get_max_parallelism() # Max parallelism is now 10 for all stages of the optimization." ] } ], "metadata": { "fileHeader": "", "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.14" }, "papermill": { "default_parameters": {}, "duration": 236.079303, "end_time": "2024-05-13T05:22:48.825652", "environment_variables": {}, "exception": null, "input_path": "/tmp/tmp.gf1AMIEaOC/Ax-main/tutorials/tune_cnn_service.ipynb", "output_path": "/tmp/tmp.gf1AMIEaOC/Ax-main/tutorials/tune_cnn_service.ipynb", "parameters": {}, "start_time": "2024-05-13T05:18:52.746349", "version": "2.6.0" } }, "nbformat": 4, "nbformat_minor": 5 }