{ "cells": [ { "cell_type": "markdown", "id": "413010b9", "metadata": { "collapsed": true, "customInput": null, "originalKey": "ac61b043-8ebf-43b9-9fa5-ed9a42a184ce", "papermill": { "duration": 0.00648, "end_time": "2024-07-23T19:39:46.075302", "exception": false, "start_time": "2024-07-23T19:39:46.068822", "status": "completed" }, "showInput": false, "tags": [] }, "source": [ "# Tune a CNN on MNIST\n", "\n", "This tutorial walks through using Ax to tune two hyperparameters (learning rate and momentum) for a PyTorch CNN on the MNIST dataset trained using SGD with momentum." ] }, { "cell_type": "code", "execution_count": 1, "id": "75661350", "metadata": { "customInput": null, "customOutput": null, "execution": { "iopub.execute_input": "2024-07-23T19:39:46.088670Z", "iopub.status.busy": "2024-07-23T19:39:46.088139Z", "iopub.status.idle": "2024-07-23T19:39:49.676530Z", "shell.execute_reply": "2024-07-23T19:39:49.675580Z" }, "executionStartTime": 1690415246079, "executionStopTime": 1690415266324, "originalKey": "c2b37f0f-3644-4367-912f-f775082f6676", "papermill": { "duration": 3.609031, "end_time": "2024-07-23T19:39:49.690255", "exception": false, "start_time": "2024-07-23T19:39:46.081224", "status": "completed" }, "requestMsgId": "0b481630-f0f4-436a-a205-a25aa163a364", "showInput": true, "tags": [] }, "outputs": [ { "data": { "text/html": [ "" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:39:49] ax.utils.notebook.plotting: Injecting Plotly library into cell. Do not overwrite or delete cell.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:39:49] ax.utils.notebook.plotting: Please see\n", " (https://ax.dev/tutorials/visualizations.html#Fix-for-plots-that-are-not-rendering)\n", " if visualizations are not rendering.\n" ] }, { "data": { "text/html": [ " \n", " " ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "import torch\n", "\n", "import torch.nn as nn\n", "import torch.nn.functional as F\n", "\n", "from ax.service.ax_client import AxClient, ObjectiveProperties\n", "from ax.service.utils.report_utils import exp_to_df\n", "from ax.utils.notebook.plotting import init_notebook_plotting, render\n", "from ax.utils.tutorials.cnn_utils import evaluate, load_mnist, train\n", "from torch._tensor import Tensor\n", "from torch.utils.data import DataLoader\n", "\n", "init_notebook_plotting()" ] }, { "cell_type": "code", "execution_count": 2, "id": "4fcd1861", "metadata": { "customInput": null, "customOutput": null, "execution": { "iopub.execute_input": "2024-07-23T19:39:49.765944Z", "iopub.status.busy": "2024-07-23T19:39:49.765450Z", "iopub.status.idle": "2024-07-23T19:39:49.770209Z", "shell.execute_reply": "2024-07-23T19:39:49.769657Z" }, "executionStartTime": 1690415266521, "executionStopTime": 1690415266529, "originalKey": "4d0a27c4-a6ce-4b7d-97eb-1c229aabb375", "papermill": { "duration": 0.044482, "end_time": "2024-07-23T19:39:49.771509", "exception": false, "start_time": "2024-07-23T19:39:49.727027", "status": "completed" }, "requestMsgId": "fd975d25-a185-4b09-a50f-7b2bcd89f93f", "showInput": true, "tags": [] }, "outputs": [], "source": [ "torch.manual_seed(42)\n", "dtype = torch.float\n", "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")" ] }, { "cell_type": "markdown", "id": "df55ede8", "metadata": { "customInput": null, "originalKey": "10384e51-444c-4265-b56d-ad078d05d2a1", "papermill": { "duration": 0.037675, "end_time": "2024-07-23T19:39:49.846056", "exception": false, "start_time": "2024-07-23T19:39:49.808381", "status": "completed" }, "showInput": false, "tags": [] }, "source": [ "## 1. Load MNIST data\n", "First, we need to load the MNIST data and partition it into training, validation, and test sets.\n", "\n", "Note: this will download the dataset if necessary." ] }, { "cell_type": "code", "execution_count": 3, "id": "b583ad4b", "metadata": { "customInput": null, "customOutput": null, "execution": { "iopub.execute_input": "2024-07-23T19:39:49.922051Z", "iopub.status.busy": "2024-07-23T19:39:49.921439Z", "iopub.status.idle": "2024-07-23T19:39:51.868204Z", "shell.execute_reply": "2024-07-23T19:39:51.867419Z" }, "executionStartTime": 1690415266733, "executionStopTime": 1690415266902, "originalKey": "6f0949e2-1064-44b8-99c0-f6ce23df7c63", "papermill": { "duration": 1.986622, "end_time": "2024-07-23T19:39:51.869697", "exception": false, "start_time": "2024-07-23T19:39:49.883075", "status": "completed" }, "requestMsgId": "8ce7dd21-9afb-4379-ad11-4112b4d27f8a", "showInput": true, "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "Failed to download (trying next):\n", "HTTP Error 403: Forbidden\n", "\n", "Downloading https://ossci-datasets.s3.amazonaws.com/mnist/train-images-idx3-ubyte.gz\n", "Downloading https://ossci-datasets.s3.amazonaws.com/mnist/train-images-idx3-ubyte.gz to ./data/MNIST/raw/train-images-idx3-ubyte.gz\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "\r", " 0%| | 0/9912422 [00:00= p2\" or \"p1 + p2 <= some_bound\".\n", " # outcome_constraints: Optional, a list of strings of form \"constrained_metric <= some_bound\".\n", ")" ] }, { "cell_type": "markdown", "id": "0d42c3d7", "metadata": { "customInput": null, "originalKey": "af441a83-50fd-4385-a380-d8ebc570c0e5", "papermill": { "duration": 0.038601, "end_time": "2024-07-23T19:39:52.273717", "exception": false, "start_time": "2024-07-23T19:39:52.235116", "status": "completed" }, "showInput": false, "tags": [] }, "source": [ "## 4. Define how to evaluate trials\n" ] }, { "cell_type": "markdown", "id": "0a130114", "metadata": { "customInput": null, "originalKey": "c7630dfd-548b-408a-badf-b6abf79275e2", "papermill": { "duration": 0.038446, "end_time": "2024-07-23T19:39:52.350679", "exception": false, "start_time": "2024-07-23T19:39:52.312233", "status": "completed" }, "showInput": false, "tags": [] }, "source": [ "First we define a simple CNN class to classify the MNIST images" ] }, { "cell_type": "code", "execution_count": 6, "id": "ef823d6c", "metadata": { "customInput": null, "customOutput": null, "execution": { "iopub.execute_input": "2024-07-23T19:39:52.429571Z", "iopub.status.busy": "2024-07-23T19:39:52.429052Z", "iopub.status.idle": "2024-07-23T19:39:52.434002Z", "shell.execute_reply": "2024-07-23T19:39:52.433377Z" }, "executionStartTime": 1690415267282, "executionStopTime": 1690415267286, "originalKey": "e41fea0a-ae71-4e6f-8c0a-6eb6ae143fb0", "papermill": { "duration": 0.045868, "end_time": "2024-07-23T19:39:52.435236", "exception": false, "start_time": "2024-07-23T19:39:52.389368", "status": "completed" }, "requestMsgId": "60f14ec9-eb1b-4e88-95c5-15c91f999c90", "showInput": true, "tags": [] }, "outputs": [], "source": [ "class CNN(nn.Module):\n", " \n", " def __init__(self) -> None:\n", " super().__init__()\n", " self.conv1 = nn.Conv2d(1, 20, kernel_size=5, stride=1)\n", " self.fc1 = nn.Linear(8 * 8 * 20, 64)\n", " self.fc2 = nn.Linear(64, 10)\n", "\n", " def forward(self, x: Tensor) -> Tensor:\n", " x = F.relu(self.conv1(x))\n", " x = F.max_pool2d(x, 3, 3)\n", " x = x.view(-1, 8 * 8 * 20)\n", " x = F.relu(self.fc1(x))\n", " x = self.fc2(x)\n", " return F.log_softmax(x, dim=-1)" ] }, { "cell_type": "markdown", "id": "8ac0a9cd", "metadata": { "customInput": null, "originalKey": "8ef6bcb9-c492-4874-b8c7-a07f7e6291ad", "papermill": { "duration": 0.038841, "end_time": "2024-07-23T19:39:52.512643", "exception": false, "start_time": "2024-07-23T19:39:52.473802", "status": "completed" }, "showInput": false, "tags": [] }, "source": [ "In this tutorial, we want to optimize classification accuracy on the validation set as a function of the learning rate and momentum. The `train_evaluate` function takes in a parameterization (set of parameter values), computes the classification accuracy, and returns that metric. " ] }, { "cell_type": "code", "execution_count": 7, "id": "398240cf", "metadata": { "customInput": null, "customOutput": null, "execution": { "iopub.execute_input": "2024-07-23T19:39:52.592603Z", "iopub.status.busy": "2024-07-23T19:39:52.591840Z", "iopub.status.idle": "2024-07-23T19:39:52.596411Z", "shell.execute_reply": "2024-07-23T19:39:52.595875Z" }, "executionStartTime": 1690415267388, "executionStopTime": 1690415267395, "originalKey": "a7e4bcc4-7494-429b-bb93-7ad84d0985af", "papermill": { "duration": 0.046025, "end_time": "2024-07-23T19:39:52.597801", "exception": false, "start_time": "2024-07-23T19:39:52.551776", "status": "completed" }, "requestMsgId": "5d486dbf-60cb-453d-8f24-8605f974b0a7", "showInput": true, "tags": [] }, "outputs": [], "source": [ "def train_evaluate(parameterization):\n", " \"\"\"\n", " Train the model and then compute an evaluation metric.\n", "\n", " In this tutorial, the CNN utils package is doing a lot of work\n", " under the hood:\n", " - `train` initializes the network, defines the loss function\n", " and optimizer, performs the training loop, and returns the\n", " trained model.\n", " - `evaluate` computes the accuracy of the model on the\n", " evaluation dataset and returns the metric.\n", "\n", " For your use case, you can define training and evaluation functions\n", " of your choosing.\n", "\n", " \"\"\"\n", " net = CNN()\n", " net = train(\n", " net=net,\n", " train_loader=train_loader,\n", " parameters=parameterization,\n", " dtype=dtype,\n", " device=device,\n", " )\n", "\n", " return evaluate(\n", " net=net, \n", " data_loader=valid_loader, \n", " dtype=dtype, \n", " device=device,\n", " )\n" ] }, { "cell_type": "markdown", "id": "a7cc01f1", "metadata": { "customInput": null, "originalKey": "9ab127a8-021f-4ec8-9f4e-f4256a2e322a", "papermill": { "duration": 0.038565, "end_time": "2024-07-23T19:39:52.676856", "exception": false, "start_time": "2024-07-23T19:39:52.638291", "status": "completed" }, "showInput": false, "tags": [] }, "source": [ "## 5. Run optimization loop\n" ] }, { "cell_type": "markdown", "id": "9ebcbea1", "metadata": { "customInput": null, "originalKey": "411a2fb4-e8a3-4414-bc17-09f0b5ba3e74", "papermill": { "duration": 0.038572, "end_time": "2024-07-23T19:39:52.754192", "exception": false, "start_time": "2024-07-23T19:39:52.715620", "status": "completed" }, "showInput": false, "tags": [] }, "source": [ "First we use `attach_trial` to attach a custom trial with manually-chosen parameters. This step is optional, but we include it here to demonstrate adding manual trials and to serve as a baseline model with decent performance. " ] }, { "cell_type": "code", "execution_count": 8, "id": "91a93a5f", "metadata": { "customInput": null, "execution": { "iopub.execute_input": "2024-07-23T19:39:52.872517Z", "iopub.status.busy": "2024-07-23T19:39:52.872230Z", "iopub.status.idle": "2024-07-23T19:39:59.676724Z", "shell.execute_reply": "2024-07-23T19:39:59.676108Z" }, "executionStartTime": 1690415267533, "executionStopTime": 1690415287786, "originalKey": "1388ef55-5642-46ab-b297-c76a73a48aca", "papermill": { "duration": 6.885564, "end_time": "2024-07-23T19:39:59.678168", "exception": false, "start_time": "2024-07-23T19:39:52.792604", "status": "completed" }, "requestMsgId": "b32a4981-ad59-46e1-b701-fa5a5f118d8b", "showInput": true, "tags": [] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:39:52] ax.core.experiment: Attached custom parameterizations [{'lr': 2.6e-05, 'momentum': 0.58}] as trial 0.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:39:59] ax.service.ax_client: Completed trial 0 with data: {'accuracy': (0.841833, None)}.\n" ] } ], "source": [ "# Attach the trial\n", "ax_client.attach_trial(\n", " parameters={\"lr\": 0.000026, \"momentum\": 0.58}\n", ")\n", "\n", "# Get the parameters and run the trial \n", "baseline_parameters = ax_client.get_trial_parameters(trial_index=0)\n", "ax_client.complete_trial(trial_index=0, raw_data=train_evaluate(baseline_parameters))" ] }, { "cell_type": "markdown", "id": "8cd6a487", "metadata": { "customInput": null, "originalKey": "f0f886a1-c5c8-44bb-b2fd-9fa3f140357a", "papermill": { "duration": 0.03784, "end_time": "2024-07-23T19:39:59.753593", "exception": false, "start_time": "2024-07-23T19:39:59.715753", "status": "completed" }, "showInput": false, "tags": [] }, "source": [ "Now we start the optimization loop.\n", "\n", "At each step, the user queries the client for a new trial then submits the evaluation of that trial back to the client.\n", "\n", "Note that Ax auto-selects an appropriate optimization algorithm based on the search space. For more advanced use cases that require a specific optimization algorithm, pass a `generation_strategy` argument into the `AxClient` constructor. Note that when Bayesian Optimization is used, generating new trials may take a few minutes." ] }, { "cell_type": "code", "execution_count": 9, "id": "4df25cb3", "metadata": { "customInput": null, "customOutput": null, "execution": { "iopub.execute_input": "2024-07-23T19:39:59.829753Z", "iopub.status.busy": "2024-07-23T19:39:59.829189Z", "iopub.status.idle": "2024-07-23T19:43:06.359407Z", "shell.execute_reply": "2024-07-23T19:43:06.358784Z" }, "executionStartTime": 1690415287908, "executionStopTime": 1690415945107, "originalKey": "bff5d714-1ab3-43d3-b9b3-8c3a53c81dcb", "papermill": { "duration": 186.569834, "end_time": "2024-07-23T19:43:06.360764", "exception": false, "start_time": "2024-07-23T19:39:59.790930", "status": "completed" }, "requestMsgId": "a203534f-85dd-4dfa-9fa6-6aa46a0200a3", "showInput": true, "tags": [] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "/tmp/tmp.DL1QmpHQMI/Ax-main/ax/modelbridge/cross_validation.py:462: UserWarning:\n", "\n", "Encountered exception in computing model fit quality: RandomModelBridge does not support prediction.\n", "\n", "[INFO 07-23 19:39:59] ax.service.ax_client: Generated new trial 1 with parameters {'lr': 0.009955, 'momentum': 0.633423} using model Sobol.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:40:06] ax.service.ax_client: Completed trial 1 with data: {'accuracy': (0.100333, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/tmp/tmp.DL1QmpHQMI/Ax-main/ax/modelbridge/cross_validation.py:462: UserWarning:\n", "\n", "Encountered exception in computing model fit quality: RandomModelBridge does not support prediction.\n", "\n", "[INFO 07-23 19:40:06] ax.service.ax_client: Generated new trial 2 with parameters {'lr': 5e-06, 'momentum': 0.022851} using model Sobol.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:40:13] ax.service.ax_client: Completed trial 2 with data: {'accuracy': (0.318667, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/tmp/tmp.DL1QmpHQMI/Ax-main/ax/modelbridge/cross_validation.py:462: UserWarning:\n", "\n", "Encountered exception in computing model fit quality: RandomModelBridge does not support prediction.\n", "\n", "[INFO 07-23 19:40:13] ax.service.ax_client: Generated new trial 3 with parameters {'lr': 7e-06, 'momentum': 0.176948} using model Sobol.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:40:20] ax.service.ax_client: Completed trial 3 with data: {'accuracy': (0.4585, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/tmp/tmp.DL1QmpHQMI/Ax-main/ax/modelbridge/cross_validation.py:462: UserWarning:\n", "\n", "Encountered exception in computing model fit quality: RandomModelBridge does not support prediction.\n", "\n", "[INFO 07-23 19:40:20] ax.service.ax_client: Generated new trial 4 with parameters {'lr': 8.2e-05, 'momentum': 0.90883} using model Sobol.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:40:27] ax.service.ax_client: Completed trial 4 with data: {'accuracy': (0.926, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "/tmp/tmp.DL1QmpHQMI/Ax-main/ax/modelbridge/cross_validation.py:462: UserWarning:\n", "\n", "Encountered exception in computing model fit quality: RandomModelBridge does not support prediction.\n", "\n", "[INFO 07-23 19:40:27] ax.service.ax_client: Generated new trial 5 with parameters {'lr': 0.000302, 'momentum': 0.341904} using model Sobol.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:40:33] ax.service.ax_client: Completed trial 5 with data: {'accuracy': (0.929, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:40:34] ax.service.ax_client: Generated new trial 6 with parameters {'lr': 0.000137, 'momentum': 0.590917} using model BoTorch.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:40:41] ax.service.ax_client: Completed trial 6 with data: {'accuracy': (0.92, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:40:41] ax.service.ax_client: Generated new trial 7 with parameters {'lr': 1e-05, 'momentum': 1.0} using model BoTorch.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:40:48] ax.service.ax_client: Completed trial 7 with data: {'accuracy': (0.860167, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:40:48] ax.service.ax_client: Generated new trial 8 with parameters {'lr': 0.000246, 'momentum': 0.0} using model BoTorch.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:40:55] ax.service.ax_client: Completed trial 8 with data: {'accuracy': (0.888833, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:40:56] ax.service.ax_client: Generated new trial 9 with parameters {'lr': 0.000149, 'momentum': 0.286357} using model BoTorch.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:41:03] ax.service.ax_client: Completed trial 9 with data: {'accuracy': (0.901667, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:41:03] ax.service.ax_client: Generated new trial 10 with parameters {'lr': 1e-06, 'momentum': 1.0} using model BoTorch.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:41:10] ax.service.ax_client: Completed trial 10 with data: {'accuracy': (0.560333, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:41:11] ax.service.ax_client: Generated new trial 11 with parameters {'lr': 3.7e-05, 'momentum': 1.0} using model BoTorch.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:41:17] ax.service.ax_client: Completed trial 11 with data: {'accuracy': (0.759167, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:41:18] ax.service.ax_client: Generated new trial 12 with parameters {'lr': 0.000261, 'momentum': 0.913551} using model BoTorch.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:41:25] ax.service.ax_client: Completed trial 12 with data: {'accuracy': (0.945667, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:41:26] ax.service.ax_client: Generated new trial 13 with parameters {'lr': 0.000171, 'momentum': 0.799543} using model BoTorch.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:41:33] ax.service.ax_client: Completed trial 13 with data: {'accuracy': (0.942167, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:41:34] ax.service.ax_client: Generated new trial 14 with parameters {'lr': 0.000159, 'momentum': 1.0} using model BoTorch.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:41:41] ax.service.ax_client: Completed trial 14 with data: {'accuracy': (0.854, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:41:42] ax.service.ax_client: Generated new trial 15 with parameters {'lr': 0.000312, 'momentum': 0.665533} using model BoTorch.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:41:48] ax.service.ax_client: Completed trial 15 with data: {'accuracy': (0.947667, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:41:49] ax.service.ax_client: Generated new trial 16 with parameters {'lr': 0.000824, 'momentum': 0.0} using model BoTorch.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:41:56] ax.service.ax_client: Completed trial 16 with data: {'accuracy': (0.906833, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:41:57] ax.service.ax_client: Generated new trial 17 with parameters {'lr': 0.000694, 'momentum': 0.350825} using model BoTorch.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:42:04] ax.service.ax_client: Completed trial 17 with data: {'accuracy': (0.928333, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:42:05] ax.service.ax_client: Generated new trial 18 with parameters {'lr': 0.000582, 'momentum': 0.803374} using model BoTorch.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:42:11] ax.service.ax_client: Completed trial 18 with data: {'accuracy': (0.1095, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:42:12] ax.modelbridge.base: Untransformed parameter 0.40000000000000013 greater than upper bound 0.4, clamping\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:42:12] ax.service.ax_client: Generated new trial 19 with parameters {'lr': 0.4, 'momentum': 0.0} using model BoTorch.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:42:19] ax.service.ax_client: Completed trial 19 with data: {'accuracy': (0.101, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:42:20] ax.service.ax_client: Generated new trial 20 with parameters {'lr': 7.2e-05, 'momentum': 0.703681} using model BoTorch.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:42:27] ax.service.ax_client: Completed trial 20 with data: {'accuracy': (0.914, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:42:28] ax.service.ax_client: Generated new trial 21 with parameters {'lr': 0.000451, 'momentum': 0.134976} using model BoTorch.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:42:35] ax.service.ax_client: Completed trial 21 with data: {'accuracy': (0.809333, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:42:36] ax.service.ax_client: Generated new trial 22 with parameters {'lr': 8.5e-05, 'momentum': 0.453801} using model BoTorch.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:42:42] ax.service.ax_client: Completed trial 22 with data: {'accuracy': (0.906833, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:42:43] ax.service.ax_client: Generated new trial 23 with parameters {'lr': 0.002828, 'momentum': 0.0} using model BoTorch.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:42:50] ax.service.ax_client: Completed trial 23 with data: {'accuracy': (0.1095, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:42:51] ax.service.ax_client: Generated new trial 24 with parameters {'lr': 6.5e-05, 'momentum': 0.0} using model BoTorch.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:42:58] ax.service.ax_client: Completed trial 24 with data: {'accuracy': (0.830833, None)}.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:42:59] ax.service.ax_client: Generated new trial 25 with parameters {'lr': 4e-06, 'momentum': 1.0} using model BoTorch.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:43:06] ax.service.ax_client: Completed trial 25 with data: {'accuracy': (0.767833, None)}.\n" ] } ], "source": [ "for i in range(25):\n", " parameters, trial_index = ax_client.get_next_trial()\n", " # Local evaluation here can be replaced with deployment to external system.\n", " ax_client.complete_trial(trial_index=trial_index, raw_data=train_evaluate(parameters))" ] }, { "cell_type": "markdown", "id": "a09d11e5", "metadata": { "customInput": null, "originalKey": "ccd16059-db9f-475b-b527-75afb320e0f4", "papermill": { "duration": 0.039967, "end_time": "2024-07-23T19:43:06.441168", "exception": false, "start_time": "2024-07-23T19:43:06.401201", "status": "completed" }, "showInput": false, "tags": [] }, "source": [ "### How many trials can run in parallel?\n", "By default, Ax restricts number of trials that can run in parallel for some optimization stages, in order to improve the optimization performance and reduce the number of trials that the optimization will require. To check the maximum parallelism for each optimization stage:" ] }, { "cell_type": "code", "execution_count": 10, "id": "b27f3576", "metadata": { "customInput": null, "customOutput": null, "execution": { "iopub.execute_input": "2024-07-23T19:43:06.522669Z", "iopub.status.busy": "2024-07-23T19:43:06.521978Z", "iopub.status.idle": "2024-07-23T19:43:06.528919Z", "shell.execute_reply": "2024-07-23T19:43:06.528327Z" }, "executionStartTime": 1690415945269, "executionStopTime": 1690415945336, "originalKey": "7182d2f9-912c-464c-b5ad-f65ce6f00017", "papermill": { "duration": 0.049166, "end_time": "2024-07-23T19:43:06.530249", "exception": false, "start_time": "2024-07-23T19:43:06.481083", "status": "completed" }, "requestMsgId": "4cb4ff79-e45b-4c7d-86a1-7f8007eb2c81", "showInput": true, "tags": [] }, "outputs": [ { "data": { "text/plain": [ "[(5, 5), (-1, 3)]" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "ax_client.get_max_parallelism()" ] }, { "cell_type": "markdown", "id": "850e107a", "metadata": { "customInput": null, "originalKey": "e2f429e6-2ec8-4af2-906b-52a36a53d329", "papermill": { "duration": 0.039802, "end_time": "2024-07-23T19:43:06.609792", "exception": false, "start_time": "2024-07-23T19:43:06.569990", "status": "completed" }, "showInput": false, "tags": [] }, "source": [ "The output of this function is a list of tuples of form (number of trials, max parallelism), so the example above means \"the max parallelism is 5 for the first 5 trials and 3 for all subsequent trials.\" This is because the first 5 trials are produced quasi-randomly and can all be evaluated at once, and subsequent trials are produced via Bayesian optimization, which converges on optimal point in fewer trials when parallelism is limited. MaxParallelismReachedException indicates that the parallelism limit has been reached –– refer to the 'Service API Exceptions Meaning and Handling' section at the end of the tutorial for handling.\n", "\n" ] }, { "cell_type": "markdown", "id": "b4c0f283", "metadata": { "customInput": null, "originalKey": "86c7aef9-993a-411e-add5-05839b00d3cf", "papermill": { "duration": 0.039618, "end_time": "2024-07-23T19:43:06.689208", "exception": false, "start_time": "2024-07-23T19:43:06.649590", "status": "completed" }, "showInput": false, "tags": [] }, "source": [ "### How to view all existing trials during optimization?" ] }, { "cell_type": "code", "execution_count": 11, "id": "eca0d1a2", "metadata": { "customInput": null, "execution": { "iopub.execute_input": "2024-07-23T19:43:06.769971Z", "iopub.status.busy": "2024-07-23T19:43:06.769470Z", "iopub.status.idle": "2024-07-23T19:43:06.800008Z", "shell.execute_reply": "2024-07-23T19:43:06.799364Z" }, "executionStartTime": 1690415945532, "executionStopTime": 1690415946199, "originalKey": "3fbad5dc-863a-494e-b04f-d7dc1e47936c", "papermill": { "duration": 0.072399, "end_time": "2024-07-23T19:43:06.801355", "exception": false, "start_time": "2024-07-23T19:43:06.728956", "status": "completed" }, "requestMsgId": "905ea8b6-add0-473e-8516-5be6ad7d7658", "showInput": true, "tags": [] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "[WARNING 07-23 19:43:06] ax.service.utils.report_utils: Column reason missing for all trials. Not appending column.\n" ] }, { "data": { "text/html": [ "
\n", "\n", "\n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", " \n", "
trial_indexarm_nametrial_statusgeneration_methodaccuracylrmomentum
000_0COMPLETEDManual0.8418330.0000260.580000
111_0COMPLETEDSobol0.1003330.0099550.633423
222_0COMPLETEDSobol0.3186670.0000050.022851
333_0COMPLETEDSobol0.4585000.0000070.176948
444_0COMPLETEDSobol0.9260000.0000820.908830
555_0COMPLETEDSobol0.9290000.0003020.341904
666_0COMPLETEDBoTorch0.9200000.0001370.590917
777_0COMPLETEDBoTorch0.8601670.0000101.000000
888_0COMPLETEDBoTorch0.8888330.0002460.000000
999_0COMPLETEDBoTorch0.9016670.0001490.286357
101010_0COMPLETEDBoTorch0.5603330.0000011.000000
111111_0COMPLETEDBoTorch0.7591670.0000371.000000
121212_0COMPLETEDBoTorch0.9456670.0002610.913551
131313_0COMPLETEDBoTorch0.9421670.0001710.799543
141414_0COMPLETEDBoTorch0.8540000.0001591.000000
151515_0COMPLETEDBoTorch0.9476670.0003120.665533
161616_0COMPLETEDBoTorch0.9068330.0008240.000000
171717_0COMPLETEDBoTorch0.9283330.0006940.350825
181818_0COMPLETEDBoTorch0.1095000.0005820.803374
191919_0COMPLETEDBoTorch0.1010000.4000000.000000
202020_0COMPLETEDBoTorch0.9140000.0000720.703681
212121_0COMPLETEDBoTorch0.8093330.0004510.134976
222222_0COMPLETEDBoTorch0.9068330.0000850.453801
232323_0COMPLETEDBoTorch0.1095000.0028280.000000
242424_0COMPLETEDBoTorch0.8308330.0000650.000000
252525_0COMPLETEDBoTorch0.7678330.0000041.000000
\n", "
" ], "text/plain": [ " trial_index arm_name trial_status generation_method accuracy lr \\\n", "0 0 0_0 COMPLETED Manual 0.841833 0.000026 \n", "1 1 1_0 COMPLETED Sobol 0.100333 0.009955 \n", "2 2 2_0 COMPLETED Sobol 0.318667 0.000005 \n", "3 3 3_0 COMPLETED Sobol 0.458500 0.000007 \n", "4 4 4_0 COMPLETED Sobol 0.926000 0.000082 \n", "5 5 5_0 COMPLETED Sobol 0.929000 0.000302 \n", "6 6 6_0 COMPLETED BoTorch 0.920000 0.000137 \n", "7 7 7_0 COMPLETED BoTorch 0.860167 0.000010 \n", "8 8 8_0 COMPLETED BoTorch 0.888833 0.000246 \n", "9 9 9_0 COMPLETED BoTorch 0.901667 0.000149 \n", "10 10 10_0 COMPLETED BoTorch 0.560333 0.000001 \n", "11 11 11_0 COMPLETED BoTorch 0.759167 0.000037 \n", "12 12 12_0 COMPLETED BoTorch 0.945667 0.000261 \n", "13 13 13_0 COMPLETED BoTorch 0.942167 0.000171 \n", "14 14 14_0 COMPLETED BoTorch 0.854000 0.000159 \n", "15 15 15_0 COMPLETED BoTorch 0.947667 0.000312 \n", "16 16 16_0 COMPLETED BoTorch 0.906833 0.000824 \n", "17 17 17_0 COMPLETED BoTorch 0.928333 0.000694 \n", "18 18 18_0 COMPLETED BoTorch 0.109500 0.000582 \n", "19 19 19_0 COMPLETED BoTorch 0.101000 0.400000 \n", "20 20 20_0 COMPLETED BoTorch 0.914000 0.000072 \n", "21 21 21_0 COMPLETED BoTorch 0.809333 0.000451 \n", "22 22 22_0 COMPLETED BoTorch 0.906833 0.000085 \n", "23 23 23_0 COMPLETED BoTorch 0.109500 0.002828 \n", "24 24 24_0 COMPLETED BoTorch 0.830833 0.000065 \n", "25 25 25_0 COMPLETED BoTorch 0.767833 0.000004 \n", "\n", " momentum \n", "0 0.580000 \n", "1 0.633423 \n", "2 0.022851 \n", "3 0.176948 \n", "4 0.908830 \n", "5 0.341904 \n", "6 0.590917 \n", "7 1.000000 \n", "8 0.000000 \n", "9 0.286357 \n", "10 1.000000 \n", "11 1.000000 \n", "12 0.913551 \n", "13 0.799543 \n", "14 1.000000 \n", "15 0.665533 \n", "16 0.000000 \n", "17 0.350825 \n", "18 0.803374 \n", "19 0.000000 \n", "20 0.703681 \n", "21 0.134976 \n", "22 0.453801 \n", "23 0.000000 \n", "24 0.000000 \n", "25 1.000000 " ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "ax_client.get_trials_data_frame()" ] }, { "cell_type": "markdown", "id": "cdb2734f", "metadata": { "customInput": null, "originalKey": "9f1ebc55-e6f2-498f-9185-569227c2f3d5", "papermill": { "duration": 0.040267, "end_time": "2024-07-23T19:43:06.881802", "exception": false, "start_time": "2024-07-23T19:43:06.841535", "status": "completed" }, "showInput": false, "tags": [] }, "source": [ "## 6. Retrieve best parameters\n", "\n", "Once it's complete, we can access the best parameters found, as well as the corresponding metric values. Note that these parameters may not necessarily be the set that yielded the highest _observed_ accuracy because Ax uses the highest model _predicted_ accuracy to choose the best parameters (see [here](https://ax.dev/api/service.html#module-ax.service.utils.best_point_mixin) for more details). Due to randomness in the data or the algorithm itself, using observed accuracy may result in choosing an outlier for the best set of parameters. Using the model predicted best will use the model to regularize the observations and reduce the likelihood of picking some outlier in the data." ] }, { "cell_type": "code", "execution_count": 12, "id": "cd2e883b", "metadata": { "customInput": null, "customOutput": null, "execution": { "iopub.execute_input": "2024-07-23T19:43:06.964552Z", "iopub.status.busy": "2024-07-23T19:43:06.964045Z", "iopub.status.idle": "2024-07-23T19:43:07.437037Z", "shell.execute_reply": "2024-07-23T19:43:07.436309Z" }, "executionStartTime": 1690415946312, "executionStopTime": 1690415949198, "originalKey": "8fdf0023-2bf5-4cdd-93ea-a8a708dc6845", "papermill": { "duration": 0.516092, "end_time": "2024-07-23T19:43:07.438456", "exception": false, "start_time": "2024-07-23T19:43:06.922364", "status": "completed" }, "requestMsgId": "c0b8c25d-c6ae-476e-be23-f1b963df296b", "showInput": true, "tags": [] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "[WARNING 07-23 19:43:07] ax.modelbridge.cross_validation: Metric accuracy was unable to be reliably fit.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[WARNING 07-23 19:43:07] ax.service.utils.best_point: Model fit is poor; falling back on raw data for best point.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[WARNING 07-23 19:43:07] ax.service.utils.best_point: Model fit is poor and data on objective metric accuracy is noisy; interpret best points results carefully.\n" ] }, { "data": { "text/plain": [ "{'lr': 0.00031243004356781596, 'momentum': 0.6655325132796462}" ] }, "execution_count": 12, "metadata": {}, "output_type": "execute_result" } ], "source": [ "best_parameters, values = ax_client.get_best_parameters()\n", "best_parameters" ] }, { "cell_type": "code", "execution_count": 13, "id": "b132d565", "metadata": { "customInput": null, "customOutput": null, "execution": { "iopub.execute_input": "2024-07-23T19:43:07.521130Z", "iopub.status.busy": "2024-07-23T19:43:07.520519Z", "iopub.status.idle": "2024-07-23T19:43:07.525312Z", "shell.execute_reply": "2024-07-23T19:43:07.524739Z" }, "executionStartTime": 1690415949308, "executionStopTime": 1690415949313, "originalKey": "f3eb18fc-be99-494a-aeac-e9b05a3bc182", "papermill": { "duration": 0.047599, "end_time": "2024-07-23T19:43:07.526601", "exception": false, "start_time": "2024-07-23T19:43:07.479002", "status": "completed" }, "requestMsgId": "ac214ea0-ea8c-46f2-a988-b42893ef6d6d", "showInput": true, "tags": [] }, "outputs": [ { "data": { "text/plain": [ "{'accuracy': 0.9476666666666667}" ] }, "execution_count": 13, "metadata": {}, "output_type": "execute_result" } ], "source": [ "mean, covariance = values\n", "mean" ] }, { "cell_type": "markdown", "id": "2c910cdc", "metadata": { "customInput": null, "originalKey": "6be3b006-d090-4c73-a64a-12901d1af817", "papermill": { "duration": 0.041114, "end_time": "2024-07-23T19:43:07.608464", "exception": false, "start_time": "2024-07-23T19:43:07.567350", "status": "completed" }, "showInput": false, "tags": [] }, "source": [ "## 7. Plot the response surface and optimization trace\n", "\n", "Contour plot showing classification accuracy as a function of the two hyperparameters.\n", "\n", "The black squares show points that we have actually run; notice how they are clustered in the optimal region." ] }, { "cell_type": "code", "execution_count": 14, "id": "8748cb35", "metadata": { "customInput": null, "customOutput": null, "execution": { "iopub.execute_input": "2024-07-23T19:43:07.691404Z", "iopub.status.busy": "2024-07-23T19:43:07.690863Z", "iopub.status.idle": "2024-07-23T19:43:08.187738Z", "shell.execute_reply": "2024-07-23T19:43:08.187012Z" }, "executionStartTime": 1690415949431, "executionStopTime": 1690415953540, "originalKey": "1beca759-2fa5-48d1-bfed-c9b13a054733", "papermill": { "duration": 0.543105, "end_time": "2024-07-23T19:43:08.192503", "exception": false, "start_time": "2024-07-23T19:43:07.649398", "status": "completed" }, "requestMsgId": "fa48963e-b43c-4079-81a4-079d347fe9ba", "showInput": true, "tags": [] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:43:07] ax.service.ax_client: Retrieving contour plot with parameter 'lr' on X-axis and 'momentum' on Y-axis, for metric 'accuracy'. Remaining parameters are affixed to the middle of their range.\n" ] }, { "data": { "application/vnd.plotly.v1+json": { "config": { "linkText": "Export to plot.ly", "plotlyServerURL": "https://plot.ly", "showLink": false }, "data": [ { "autocolorscale": false, "autocontour": true, "colorbar": { "tickfont": { "size": 8 }, "ticksuffix": "", "x": 0.45, "y": 0.5 }, "colorscale": [ [ 0.0, "rgb(247,252,253)" ], [ 0.125, "rgb(229,245,249)" ], [ 0.25, "rgb(204,236,230)" ], [ 0.375, "rgb(153,216,201)" ], [ 0.5, "rgb(102,194,164)" ], [ 0.625, "rgb(65,174,118)" ], [ 0.75, "rgb(35,139,69)" ], [ 0.875, "rgb(0,109,44)" ], [ 1.0, "rgb(0,68,27)" ] ], "contours": { "coloring": "heatmap" }, "hoverinfo": "x+y+z", "ncontours": 25, "type": "contour", "x": [ 1e-06, 1.3011511650442548e-06, 1.692994354296022e-06, 2.2028415765056147e-06, 2.866229883678204e-06, 3.729398352432554e-06, 4.852511011181743e-06, 6.3138503555892e-06, 8.215273746089953e-06, 1.0689313005882424e-05, 1.390841207112662e-05, 1.809694657026198e-05, 2.354686311364001e-05, 3.063802837345029e-05, 3.986470631277378e-05, 5.1870009063012666e-05, 6.749072272319499e-05, 8.781563250096393e-05, 0.00011426141253772724, 0.00014867137004306603, 0.00019344392634026088, 0.0002516997901283655, 0.0003274994751669172, 0.0004261263236648159, 0.0005544547624925005, 0.0007214294601814526, 0.000938688782612345, 0.0012213760031100258, 0.0015891948094037057, 0.002067782677737912, 0.0026904978401970136, 0.0035007443993213955, 0.004554997653699184, 0.005926740503884541, 0.007711585311544345, 0.010033938212454078, 0.013055670395116691, 0.01698740074503987, 0.02210317627048227, 0.028759573555516536, 0.03742055263793628, 0.04868979566145066, 0.06335278435066323, 0.0824315491666629, 0.10725590623460621, 0.13955614735503497, 0.18158364372009145, 0.23626776957937787, 0.3074200836506151, 0.4 ], "xaxis": "x", "y": [ 0.0, 0.02040816326530612, 0.04081632653061224, 0.061224489795918366, 0.08163265306122448, 0.1020408163265306, 0.12244897959183673, 0.14285714285714285, 0.16326530612244897, 0.18367346938775508, 0.2040816326530612, 0.22448979591836732, 0.24489795918367346, 0.26530612244897955, 0.2857142857142857, 0.3061224489795918, 0.32653061224489793, 0.3469387755102041, 0.36734693877551017, 0.3877551020408163, 0.4081632653061224, 0.42857142857142855, 0.44897959183673464, 0.4693877551020408, 0.4897959183673469, 0.5102040816326531, 0.5306122448979591, 0.5510204081632653, 0.5714285714285714, 0.5918367346938775, 0.6122448979591836, 0.6326530612244897, 0.6530612244897959, 0.673469387755102, 0.6938775510204082, 0.7142857142857142, 0.7346938775510203, 0.7551020408163265, 0.7755102040816326, 0.7959183673469387, 0.8163265306122448, 0.836734693877551, 0.8571428571428571, 0.8775510204081632, 0.8979591836734693, 0.9183673469387754, 0.9387755102040816, 0.9591836734693877, 0.9795918367346939, 1.0 ], "yaxis": "y", "z": [ [ 0.36902459190559583, 0.356186976128271, 0.34387894387001755, 0.3340389683445641, 0.32939910151876095, 0.3333681092438179, 0.349430545858529, 0.37955229951930786, 0.42215276301160304, 0.47330046187872044, 0.5289209340371881, 0.5855787213064039, 0.640551446252273, 0.6916501164468455, 0.7369816767588648, 0.7748376967105225, 0.804033528013207, 0.8251195794807905, 0.8408778189860281, 0.8542875150632032, 0.8668790219572444, 0.8783243853410184, 0.8868291997043514, 0.8887634588473163, 0.8760509981082709, 0.8352998300081593, 0.7549106070676777, 0.6362430240633745, 0.4947606746203898, 0.3520811075961721, 0.2293837479978441, 0.14127241447261674, 0.09035058053339018, 0.07059853088007706, 0.07339317206260598, 0.09047477049143782, 0.1150184486032998, 0.1418103087976964, 0.16707277898570938, 0.18820297509248562, 0.20353061948748075, 0.21213148080150118, 0.21370900041844637, 0.2085498364680315, 0.19755269464792813, 0.18231728105895995, 0.16525563903273366, 0.14964147748028322, 0.13942662574918974, 0.13849748068321932 ], [ 0.37124236818691597, 0.35879312952091075, 0.3469177145160886, 0.3375521372283445, 0.3334202174673831, 0.3379149364336093, 0.35449745216013506, 0.3851029258139041, 0.4280966487072824, 0.47951379758218937, 0.5352745258462153, 0.5919453561181209, 0.646813183053526, 0.6977127204619612, 0.7427928044181332, 0.780400304642446, 0.8094085581128837, 0.8304080486826061, 0.8461778994578159, 0.8596571357307273, 0.8722876041023622, 0.8836324466660893, 0.8918575568848033, 0.8934252230665636, 0.8803519231809809, 0.8390427907931766, 0.7577254240683248, 0.6378802223610903, 0.49521305413932803, 0.3515310686166574, 0.22807283897697778, 0.13940390441810013, 0.08810394591300541, 0.06813660009475853, 0.07085713311390729, 0.08798546821861186, 0.112675248938931, 0.13968702142416356, 0.16521461364272028, 0.18662771902640285, 0.20223395646050835, 0.2110942655930364, 0.2129042692742349, 0.2079482856953483, 0.19712625053900568, 0.18204085750995203, 0.16510728199034175, 0.1496009602834466, 0.13947227315035937, 0.1386005903098787 ], [ 0.37365047793230405, 0.3616396179461995, 0.35025594655394277, 0.3414317320708044, 0.33787805555777656, 0.34296177545910905, 0.3601001250422739, 0.3911673906510329, 0.4344866311927924, 0.48608588257191077, 0.5418926501883632, 0.5984802271582601, 0.6531466516644513, 0.7037525303803556, 0.7484935925834373, 0.7857818720094294, 0.814565771542941, 0.8354838447728097, 0.8512876175057312, 0.8648533590830578, 0.8775171082059738, 0.8887188200803671, 0.8965746334352971, 0.8976319822374984, 0.8840268616002656, 0.8420177759375225, 0.7597647893910375, 0.6389100654649016, 0.4953412504589151, 0.3509812871064084, 0.22703905144666747, 0.13795284699925647, 0.08630073086080425, 0.06608674138218074, 0.06867880125406134, 0.08579595115877425, 0.11058050798194752, 0.13777172566366913, 0.16353576828397964, 0.18521446938542685, 0.20109273385265625, 0.21021639194953146, 0.21227335895435995, 0.20754594510600144, 0.19693553344470505, 0.18204734786371501, 0.1652976202560874, 0.14995822816838578, 0.13996695018610816, 0.13917496678578511 ], [ 0.3762471405246524, 0.3647240767035423, 0.3538905088662724, 0.34567363076812035, 0.3427672120798347, 0.34850157609796406, 0.36622957577832205, 0.397736325143148, 0.4413139123019365, 0.4930088115579532, 0.5487683120555349, 0.605177288734861, 0.6595470349106494, 0.7097665407655291, 0.7540837746443535, 0.7909860317185446, 0.8195125974089064, 0.840355179294592, 0.8562157644087608, 0.8698851101408839, 0.8825757019373405, 0.893589438402421, 0.9009820529155613, 0.9013783520923416, 0.887057654209096, 0.8441951703285363, 0.7609892718201987, 0.639286060261758, 0.4950992089697677, 0.35038834685206505, 0.22624096382535164, 0.13688841424445597, 0.08492118444595143, 0.06443695602203181, 0.06685134129078785, 0.08390275968462213, 0.10873295047289366, 0.13606450693492422, 0.16203707821011393, 0.1839643512963397, 0.20010803590138193, 0.20949867691260016, 0.21181666019002154, 0.207342645105972, 0.19697966346612028, 0.18233495632187524, 0.16582363984434534, 0.1507086068113518, 0.14090372972622545, 0.14021159063162614 ], [ 0.37902972488535375, 0.36804293825234935, 0.3578165635372956, 0.35027128276617603, 0.34807881792134293, 0.3545223409072213, 0.372870092243242, 0.4047944269999499, 0.44856553895558143, 0.5002718618524945, 0.5558926322166083, 0.6120293465197644, 0.6660090781090253, 0.7157521272693819, 0.7595644928437869, 0.7960190114221339, 0.8242595899260073, 0.8450330589463928, 0.8609731795857349, 0.8747626401594162, 0.8874720648160213, 0.8982495404548787, 0.9050796360770856, 0.9046575220357852, 0.8894255245726443, 0.8455470680261742, 0.761362109299321, 0.6389624996364567, 0.49443861021235425, 0.34970279584138675, 0.22562835964231326, 0.13617103284419263, 0.08393917644419946, 0.06317092487515341, 0.06536505752443778, 0.08230056330154101, 0.10713008450485284, 0.13456464889210662, 0.16071881837093616, 0.1828780456633562, 0.19928052689953324, 0.20894146765059685, 0.21153397194164691, 0.20733741861597033, 0.19725664821462108, 0.18290031037465004, 0.16668008211083407, 0.15184424818420772, 0.1422713640000669, 0.1416966221521505 ], [ 0.3819947576785766, 0.37159145014541695, 0.36202760281034735, 0.35521578502419715, 0.3538007012857867, 0.3610075176595858, 0.38000042656363653, 0.4123211524925715, 0.4562246966221854, 0.5078616290423927, 0.5632548831165062, 0.6190280230748442, 0.6725269926092425, 0.7217068727199574, 0.7649379837742646, 0.8008889835307503, 0.8288195141678337, 0.8495307279459048, 0.8655723909828603, 0.8794971752850005, 0.8922148479491423, 0.9027027453538261, 0.9088639181779414, 0.9074591837254031, 0.891108624446936, 0.8460443233446673, 0.7608469440448957, 0.6378940806123741, 0.4933094595346073, 0.34887065950872387, 0.2251450176390526, 0.13575438446949406, 0.08332309401315707, 0.06226846092755334, 0.06420764502637633, 0.08098230928859418, 0.1057682931716114, 0.13327069076106202, 0.15958073869469203, 0.1819558105774035, 0.19861046493373735, 0.2085446526547584, 0.21142451466483975, 0.2075285215740531, 0.1977634181853163, 0.18373852498075782, 0.16785956624145582, 0.1533543836652771, 0.14405487179742116, 0.14361237663689308 ], [ 0.3851379389926722, 0.37536370589689627, 0.3665155091084467, 0.36049600099632845, 0.35991763580221997, 0.36793656761985594, 0.387595000538769, 0.4202916396432949, 0.46427114387365953, 0.5157622247472217, 0.5708425508180145, 0.6261637296570303, 0.6790943494467386, 0.727628363720846, 0.7702072080454038, 0.8056053849359971, 0.8332065355794891, 0.8538630399593983, 0.8700271918559976, 0.8841005308606928, 0.8968121504746759, 0.9069502644783458, 0.9123267319098781, 0.9097671019257207, 0.8920794333400647, 0.8456540478450678, 0.7594060007392743, 0.636035574086784, 0.4916608049596387, 0.3478351297838516, 0.22473114150622975, 0.135587479043624, 0.08303691884180953, 0.06170606867888562, 0.0633645001143962, 0.07993940449615788, 0.10464294503838467, 0.13218049535768206, 0.15862210548119626, 0.18119750644489174, 0.1980977185716184, 0.2083076762888516, 0.21148694844151894, 0.20791346125839916, 0.19849587525019774, 0.18484328935440208, 0.1693527514151002, 0.15522564465990374, 0.146236187978384, 0.1459381693144336 ], [ 0.3884541650390135, 0.3793526877798307, 0.371270635749949, 0.3660987163906389, 0.36641165426818, 0.3752856259227578, 0.39562498013473374, 0.42867769149147955, 0.47268174921204487, 0.52395552274378, 0.5786414188839515, 0.6334256442889485, 0.685703966237391, 0.7335139667207011, 0.775375450768261, 0.8101782698190941, 0.8374355050630756, 0.858045814135941, 0.8743521727935697, 0.8885847068956598, 0.9012710489173013, 0.9109902990192246, 0.9154540721994512, 0.9115563946844176, 0.8923024218251256, 0.8443377833296538, 0.7569988677363957, 0.6333416752071888, 0.4894415712659723, 0.3465382732868101, 0.22432545989131608, 0.13561660142267618, 0.08304139912028774, 0.0614575822346799, 0.06281907920384833, 0.07916192551747003, 0.1037485210479746, 0.13129132643768415, 0.15784174801131312, 0.18060262427879203, 0.1977417861106766, 0.20822955636541907, 0.21171939557922204, 0.20848903175764788, 0.19944895289300701, 0.18620697317331047, 0.17114853030304722, 0.15744242543734, 0.1487948035765274, 0.14865104467409063 ], [ 0.3919375572676926, 0.3835503202535252, 0.37628190534125244, 0.37200882396700263, 0.3732624052216121, 0.3830281803879492, 0.4040592110015235, 0.43744873072261703, 0.48143108339917784, 0.532421436416994, 0.5866356692774177, 0.6408016960593136, 0.6923477908781817, 0.739360593192236, 0.780445916971349, 0.8146177233094027, 0.8415213313314579, 0.8620952110605045, 0.8785622267569362, 0.8929614778376529, 0.9055971996805127, 0.9148176619953444, 0.9182254657614013, 0.9127911101505534, 0.8917325220446075, 0.8420504550521003, 0.7535819199300205, 0.6297671284735817, 0.4866014964242762, 0.3449226607482453, 0.22386703261595803, 0.13578705098960586, 0.08329524015944334, 0.06149485207790495, 0.06255329371211682, 0.07863885163993867, 0.10307875504136466, 0.1305999337821766, 0.15723810938355987, 0.18017031651388016, 0.19754181694971928, 0.20830890437159721, 0.2121194672063595, 0.20925135577892073, 0.2006166865428478, 0.1878207485175365, 0.17323424482020566, 0.15998726449644907, 0.15170835808080274, 0.1517264032837875 ], [ 0.3955814971820702, 0.38794753248951924, 0.38153692237198694, 0.378209528842102, 0.38044752966288325, 0.39113571662622354, 0.41286501521458097, 0.4465726932078692, 0.4904920147753491, 0.5411382092317428, 0.59480799505885, 0.6482785556875299, 0.69901678588383, 0.7451644634739751, 0.7854213413642175, 0.8189333449341448, 0.8454784336405629, 0.866027149216125, 0.8826720420969952, 0.8972419857627488, 0.9097945223057248, 0.9184236469351574, 0.9206140032919672, 0.913424056860683, 0.8903148908587984, 0.8387401539301473, 0.7491083859272126, 0.6252671809060992, 0.48309215514738224, 0.3429328684836173, 0.22329679555702947, 0.1360446542434307, 0.08375625403630671, 0.061788451268906974, 0.06254792797085451, 0.07835831442154662, 0.10262678472671005, 0.13010264421238782, 0.15680930046316632, 0.1798994296380767, 0.197496634600816, 0.20854394792649067, 0.2126842933299382, 0.2101959318762282, 0.20199229217665748, 0.189674723578245, 0.17559591508222938, 0.1628412247926766, 0.154953168066849, 0.15513853788931453 ], [ 0.39937866604400674, 0.3925343283052548, 0.3870220962919905, 0.3846825656744665, 0.38794303780769446, 0.39957829849054916, 0.4220088475051079, 0.4560168225763703, 0.49983625198877923, 0.5500827010200748, 0.6031397199064421, 0.6558416325570966, 0.7057008173041374, 0.7509208781655295, 0.7903036266950723, 0.8231338026267987, 0.8493202693301624, 0.8698567731768779, 0.8866955952861896, 0.9014363411899486, 0.9138649618314899, 0.9217961437534113, 0.9225870598158696, 0.9133993243143474, 0.8879860835088991, 0.8343487518189575, 0.7435290469495327, 0.6197983792999792, 0.4788680522779567, 0.34051683359024165, 0.22255887267312047, 0.13633705586374234, 0.08438242651963934, 0.062308374611540596, 0.06278306710857273, 0.07830785743527924, 0.10238530968129733, 0.12979545655384472, 0.15655315572271256, 0.17978853787519455, 0.19760476082737555, 0.20893255502154795, 0.21341055577650153, 0.21131768610716395, 0.20356825125524025, 0.1917580841303851, 0.17821847319490514, 0.16598425811040352, 0.15850468869610146, 0.158861089365273 ], [ 0.4033210886009421, 0.3972998617299674, 0.3927227713104664, 0.3914084196113783, 0.3957236697372078, 0.4083250690776936, 0.4314568118906108, 0.4657482249716307, 0.5094347884375454, 0.5592306561997407, 0.6116109202276915, 0.663475078688632, 0.7123885521635254, 0.7566240049576315, 0.7950935203698264, 0.8272264549691003, 0.8530589316999886, 0.8735979801538806, 0.8906456530570263, 0.9055532305986543, 0.9178083224692257, 0.9249199801284593, 0.9241075877560521, 0.9126555475170723, 0.8846763474861596, 0.8288133138552946, 0.7367935435165652, 0.6133196963858871, 0.4738877654889594, 0.33762706035739726, 0.22160167971516448, 0.1366148028836538, 0.08513287526018731, 0.06302470836114238, 0.06323852249667605, 0.07847469966855203, 0.10234675281832184, 0.1296741384371819, 0.1564672896689947, 0.1798359771132344, 0.19786444037347195, 0.20947225957611326, 0.214294523407686, 0.2126110270842908, 0.2053364000313661, 0.19405923891536392, 0.18108599460706165, 0.16939554366182308, 0.16233791052210678, 0.16286743198991782 ], [ 0.40740017993556304, 0.40223251641638347, 0.39862335930607984, 0.39836654384447945, 0.40376322828434347, 0.4173446683923401, 0.4411750478875103, 0.4757341440126129, 0.5192582327796169, 0.5685569448675074, 0.6202005467399978, 0.6711618004719823, 0.7190673682834838, 0.7622666876370517, 0.7997903354521421, 0.8312170383922548, 0.8567048145484006, 0.8772630098299163, 0.8945332945836787, 0.9095995248583337, 0.9216221629709085, 0.927777448412825, 0.9251357846309091, 0.9111291021647172, 0.8803126329321589, 0.822068244581035, 0.7288522537142762, 0.6057939447056249, 0.46811511235950976, 0.33422168146833214, 0.22037884037815725, 0.13683223899392705, 0.08596868545179148, 0.0639082526212118, 0.06389424345802885, 0.07884599622813915, 0.10250342169001392, 0.12973432273029561, 0.15654915249038803, 0.18003987824873646, 0.19827366574216754, 0.21016028783302687, 0.2153320889891297, 0.21406990337876264, 0.20728802131446877, 0.19656596538985682, 0.1841819210930421, 0.1730537950486949, 0.16642769656051237, 0.16713099652431296 ], [ 0.41160679453789006, 0.4073199871687267, 0.4047074725401199, 0.4055355678737122, 0.41203487676840805, 0.426605570676933, 0.4511300048994439, 0.4859421206677689, 0.5292770428637117, 0.5780357728991639, 0.628886543706292, 0.6788834793863435, 0.7257232802321385, 0.7678402829043504, 0.8043917196821706, 0.8351094159501393, 0.8602663401402072, 0.880862101965305, 0.8983674701051869, 0.9135798784888022, 0.925301748464408, 0.9303489666967173, 0.9256309302535901, 0.9087571637631389, 0.8748220409855114, 0.8140480958242943, 0.7196586902484595, 0.5971894143900671, 0.4615203118637559, 0.33026537828135233, 0.2188499321347792, 0.13694822571387755, 0.08685361786764456, 0.06493108311998086, 0.06473070535914505, 0.07940909034623977, 0.10284766602179463, 0.12997160134365338, 0.15679608352475582, 0.18039819911353983, 0.19883020148721953, 0.2109935851240054, 0.21651880709517135, 0.215687862255168, 0.2094139368843475, 0.19926555271915924, 0.18748927082349243, 0.17693753284586222, 0.17074906543152502, 0.17162553868067987 ], [ 0.41593127672613406, 0.4125493619730028, 0.4109580532836805, 0.41289349195958946, 0.4205113978629125, 0.43607634846625976, 0.4612886283678964, 0.49634011184930465, 0.5394616930250935, 0.5876408615874608, 0.6376459653343185, 0.6866206033815239, 0.732340885030026, 0.7733345296934844, 0.8088934745771986, 0.8389053844549481, 0.8637497474081143, 0.8844032256763503, 0.9021546231466712, 0.9174963061379711, 0.9288400649763705, 0.9326138185475757, 0.9255532269539878, 0.9054806153763545, 0.8681355722365997, 0.8046909731248073, 0.7091723369462574, 0.5874816507368239, 0.45408110552092706, 0.3257301627013928, 0.2169810758394361, 0.1369267058985274, 0.08775468994876279, 0.06606704320340073, 0.06572926579329419, 0.0801517511658707, 0.10337202796630618, 0.13038161413541616, 0.15720536114204609, 0.1809087541674934, 0.1995316075038831, 0.21196884255148196, 0.21784993245086443, 0.21745810876490057, 0.2117045989032213, 0.20214493938288136, 0.19099083230721803, 0.18102532230109314, 0.17527742645725286, 0.1763253597569192 ], [ 0.42036351159357477, 0.41790720307826335, 0.4173574979628979, 0.42041786461910113, 0.4291654127879082, 0.44572587247699913, 0.4716184802711727, 0.5068965783121809, 0.5497827985848034, 0.5973456002558504, 0.6464550900232874, 0.6943525110447014, 0.7389033311574794, 0.7787374549485888, 0.8132894257969732, 0.8426045370155633, 0.8671589368338916, 0.8878918825773684, 0.9058984139844269, 0.921347728635524, 0.9322279162773566, 0.9345509175377845, 0.9248655242938609, 0.9012467600041347, 0.8601920938418655, 0.7939424978654666, 0.6973618004766485, 0.5766552700551744, 0.4457837998538435, 0.32059602065939485, 0.21474538048864117, 0.13673712352087675, 0.08864263421408602, 0.06729216042881137, 0.06687248218184161, 0.08106239235785018, 0.10406938171656643, 0.13096013066307255, 0.15777424766051273, 0.18156924118092788, 0.2003752608408803, 0.2130825231621684, 0.21932045814596518, 0.21937356429635302, 0.2141501788680888, 0.20519084327689352, 0.19466934016305826, 0.18529597714532742, 0.17998877232091726, 0.18120548548565418 ], [ 0.4248929757301078, 0.42337962587487377, 0.4238877739544055, 0.4280859412936381, 0.43796956215127353, 0.4555234570818672, 0.48208781200522544, 0.5175805462725436, 0.5602112114157918, 0.6071231766091532, 0.6552895340507499, 0.7020574511087048, 0.7453923143776892, 0.7840353193155661, 0.8175713455688306, 0.8462041782635195, 0.8704953677695659, 0.8913309826614493, 0.9095995797671175, 0.9251297045390792, 0.9354541240256956, 0.9361395474038071, 0.9235348485133094, 0.8960117632972652, 0.8509424041903647, 0.781760307215881, 0.6842080819796847, 0.5647056999809262, 0.43662419027365107, 0.31485141502586317, 0.2121232523351559, 0.13635471218371198, 0.0894922403441325, 0.06858498502892563, 0.06814438567336467, 0.08213026724321915, 0.10493305930808083, 0.13170312258137196, 0.15850002797382023, 0.18237726419549982, 0.20135837560689862, 0.21433088722698068, 0.220925153200853, 0.22142692376908885, 0.21674065286020988, 0.20838988270508074, 0.198507631669969, 0.18972873144422941, 0.18485983447602383, 0.18624180848964156 ], [ 0.42950878705467926, 0.42895237453515445, 0.4305305276925285, 0.43587482342740486, 0.44689665120654604, 0.46543896082028163, 0.4926656026163674, 0.5283616464541387, 0.5707180938731884, 0.6169466896738861, 0.6641243668673227, 0.7097126602446077, 0.7517881039233579, 0.789212605936905, 0.8217289279033198, 0.8496992897737045, 0.873758003086289, 0.8947207870991996, 0.9132559421851549, 0.9288347407181972, 0.9385058294294559, 0.9373600343630004, 0.9215336838091843, 0.8897427318887144, 0.840353135307118, 0.7681190961412236, 0.6697076808670924, 0.5516407269573141, 0.426608327703506, 0.30849364444150945, 0.2091025759150862, 0.1357606635147538, 0.09028258838837999, 0.06992684976395003, 0.06953070763288882, 0.08334563672447248, 0.10595695965445717, 0.13260682457131, 0.15938004066170608, 0.18333035214263504, 0.20247802061077003, 0.21571001629101522, 0.22265859901973578, 0.22361071076098937, 0.21946588207357115, 0.21172868713290394, 0.20248878383693447, 0.1943033819523977, 0.18986820602884857, 0.19141119914404114 ], [ 0.43419975319413, 0.43461089361128286, 0.4372671832534461, 0.4437615781126496, 0.45591976317205263, 0.47544285055766733, 0.5033215713509397, 0.539210133808196, 0.581274975152698, 0.6267892495312593, 0.6729342303804321, 0.7172944623969826, 0.7580696026994129, 0.7942520554729534, 0.8257498174477214, 0.8530825433913798, 0.8769432951859502, 0.8980589069525837, 0.9168625367607952, 0.9324527980404463, 0.9413688584911013, 0.9381943140350473, 0.9188409694575975, 0.8824193311855711, 0.8284100277898261, 0.7530161080782771, 0.653875166767766, 0.5374817423680703, 0.4157530934555597, 0.3015290542802519, 0.20567877346423136, 0.13494218554044635, 0.09099718104262533, 0.07130205236789033, 0.07101905624308857, 0.08469990693313689, 0.10713563808474291, 0.13366778180286887, 0.1604117004937372, 0.18442597261581872, 0.20373113445366586, 0.21721583571447217, 0.22451522432966198, 0.22591732996733865, 0.2223156878288034, 0.21519399700546216, 0.20659623134225247, 0.19900040368094707, 0.1949924363534966, 0.19669158911391094 ], [ 0.43895441795058027, 0.4403403950145529, 0.44407903104012353, 0.4517233391785496, 0.46501234469943403, 0.48550623683090544, 0.5140261707854212, 0.5500968907400909, 0.5918537926493488, 0.6366240670613618, 0.6816934644588674, 0.7247783931311759, 0.7642144453299026, 0.7991347505784558, 0.8296196929933752, 0.8563443603652967, 0.8800452068416019, 0.9013403423439222, 0.9204118127752925, 0.9359717284591978, 0.94402809108217, 0.9386263632843193, 0.9154427873379924, 0.8740348701152338, 0.8151200063524913, 0.736475393703899, 0.6367448683397646, 0.5222646014054404, 0.4040865549024794, 0.29397309677778233, 0.20185474864589537, 0.13389246033499513, 0.09162398310132924, 0.07269796302522091, 0.07259904176482623, 0.0861857340535398, 0.10846437387008001, 0.13488288208970278, 0.16159251141921815, 0.18566154043649818, 0.20511453788266232, 0.21884413548902276, 0.2264893382723877, 0.22833911650444, 0.2252799204947904, 0.21877275230628207, 0.21081386613564163, 0.20380104145124167, 0.20021210125650435, 0.2020620313567203 ], [ 0.4437611055093935, 0.4461259200204102, 0.45094730659299315, 0.4597373911355761, 0.47414826771158197, 0.49560088681898873, 0.5247505651070521, 0.5609934163115085, 0.602426920123004, 0.6464245358479986, 0.6903762403848424, 0.7321393525168904, 0.7701991381206532, 0.8038402533027771, 0.8333224067378399, 0.8594730143379662, 0.8830552603668903, 0.9045575439151216, 0.9238938510126182, 0.9393776303018996, 0.9464677789193812, 0.938642479466282, 0.9113327363858653, 0.8645968272960515, 0.8005117154887441, 0.7185474173499166, 0.6183714824285438, 0.5060400429377281, 0.3916480836855316, 0.28585023912068835, 0.1976407204327969, 0.13261050969654442, 0.09215537618960645, 0.07410506019550767, 0.07426234982331537, 0.0877970942632963, 0.10993921343205348, 0.13624937109681146, 0.16292006937219128, 0.18703442082217037, 0.20662494331558723, 0.22059058918208785, 0.22857516138765266, 0.23086838168359541, 0.22834852194066568, 0.22245216984592614, 0.215126119802306, 0.20868738014972033, 0.20550785209669642, 0.20750273994957935 ], [ 0.44860796214935916, 0.45195239614840177, 0.4578532598889992, 0.4677812377581013, 0.48330187175344685, 0.5056992203777076, 0.5354665970788044, 0.5718718035755932, 0.6129671839930732, 0.6561643073602962, 0.6989567031264422, 0.7393517898834956, 0.7759992453151815, 0.8083467992167771, 0.8368401802443475, 0.862454776301277, 0.8859626096390821, 0.9077004779617052, 0.9272965592093037, 0.9426551206811611, 0.9486717776653822, 0.9382314042136048, 0.9065120199639933, 0.8541268519604048, 0.7846346883439357, 0.6993059529965208, 0.5988296534982736, 0.48887365825571794, 0.37848822855795866, 0.277193720757981, 0.19305395330429587, 0.1311009772903554, 0.09258803683587324, 0.07551689874462209, 0.07600276270966944, 0.08952931712665824, 0.1115569871090849, 0.1377648492292941, 0.16439205451427763, 0.1885419271568427, 0.20825896155420187, 0.22245077093157312, 0.23076685429852029, 0.23349745499072078, 0.23151158132160443, 0.22621980952249487, 0.2195180299817474, 0.21364239626763298, 0.21086144688882585, 0.2129951127147291 ], [ 0.45348299532303854, 0.4578046889506452, 0.464778215764596, 0.4758326573210192, 0.49244799079502677, 0.5157742937295999, 0.5461467465883297, 0.5827047069505944, 0.6234478687235591, 0.6658173595779879, 0.7074091221092504, 0.7463899231983753, 0.7815896263830207, 0.8126315524539226, 0.8401538574355016, 0.8652740994945833, 0.8887541318902589, 0.9107566784315666, 0.9306058195940812, 0.9457875239404583, 0.9506236791851435, 0.9373843020024677, 0.900989284584501, 0.8426603267913482, 0.7675576930307012, 0.6788450848080774, 0.5782127737689178, 0.4708454334245713, 0.3646683452922741, 0.268045164083236, 0.18811839054595259, 0.1293738355875913, 0.09292274593149141, 0.07693001484657669, 0.07781612913749703, 0.09137908107905701, 0.1133152975208861, 0.13942724918746097, 0.16600621289395168, 0.19018131357399704, 0.21001310581286792, 0.2244201704834311, 0.23305854397999032, 0.23621872210971379, 0.2347593841601614, 0.23006362999296465, 0.22397529224006563, 0.21864999313704336, 0.21625576607636138, 0.2185217392703072 ], [ 0.4583741100759339, 0.4636676489037598, 0.471703626309134, 0.4838697466278653, 0.5015619681533858, 0.5257997747160663, 0.5567640832919407, 0.5934653013502937, 0.6338427119785149, 0.6753580583867106, 0.7157080486559494, 0.7532279945875996, 0.7869447294882709, 0.8166709262229719, 0.8432432136495959, 0.8679138417974559, 0.8914145397936406, 0.9137112737530855, 0.93380557308262, 0.9487569744547761, 0.9523068456167396, 0.9360946150329682, 0.8947802558219253, 0.8302456144414839, 0.7493667969895902, 0.6572766460319589, 0.5566313152452853, 0.4520489169117283, 0.3502599952826047, 0.2584540456543138, 0.18286419827431566, 0.12744402597811222, 0.0931641376781025, 0.07834377259140113, 0.07970028320369416, 0.09334436984595151, 0.11521247772554866, 0.14123479365732916, 0.1677603279194394, 0.19194976278375453, 0.21188379329859858, 0.22649420633151957, 0.2354443475590779, 0.23902465892177993, 0.23808245481964474, 0.23397203434068908, 0.22848429883366012, 0.2236950220817136, 0.22167481534840766, 0.2240663968228288 ], [ 0.463269142866541, 0.46952615373617546, 0.4786111162287877, 0.49187095600300457, 0.5106196629002915, 0.5357499129843474, 0.5672922156323537, 0.6041272346717947, 0.6441258900281814, 0.6847612103662896, 0.7238284765031998, 0.7598405612466395, 0.7920389467264259, 0.820440973574394, 0.8460873174808567, 0.8703555223903827, 0.893926517566151, 0.9165469971561647, 0.9368778298091628, 0.9515444307026208, 0.9537043578824338, 0.9343578270114666, 0.8879072255835044, 0.8169431194522374, 0.7301634854269051, 0.634728030030983, 0.5342109474443136, 0.4325900776655843, 0.3353441317498815, 0.24847703801833843, 0.1773272287601403, 0.12533104060149247, 0.09332039626163657, 0.07976015777183953, 0.08165491352104459, 0.09542438874551162, 0.11724751754048923, 0.14318593326226492, 0.1696521825183288, 0.19384436979986563, 0.21386734468103197, 0.22866823708120293, 0.23791839365624445, 0.2419078614967362, 0.24147159257726475, 0.23793390642670542, 0.23303216579234426, 0.22876329150235086, 0.22710371759590486, 0.22961403573962152 ], [ 0.4681558929344538, 0.4753651466354047, 0.48548252228420474, 0.4998151173902673, 0.5195974508193502, 0.5455995080745455, 0.5777052383826197, 0.6146645751971476, 0.6542719937830983, 0.6940021050716129, 0.7317459999371141, 0.7662028171486347, 0.7968470371461545, 0.8239178529474449, 0.8486649385562847, 0.8725796083777023, 0.8962708849485492, 0.9192442164370084, 0.9398025970899335, 0.9541295951829065, 0.9547988998094276, 0.9321711794172173, 0.8803984507872875, 0.8028242859974202, 0.7100629794691891, 0.6113402977960415, 0.5110905954096996, 0.4125859195572288, 0.32001009744049635, 0.2381772348042882, 0.1715484126002711, 0.12305845467420662, 0.09340290870688428, 0.08118352502103232, 0.08368138369942635, 0.09761943983396304, 0.11941995666695693, 0.14527926578286565, 0.17167951337542886, 0.19586212243658652, 0.21595998188226134, 0.2309375712126719, 0.240474841333028, 0.24486107216487252, 0.24491790159340326, 0.24193863867753235, 0.23760674970497875, 0.23384156571891362, 0.2325286958507275, 0.23515075669054014 ], [ 0.4730221514400451, 0.48116967086940676, 0.49229992796441835, 0.5076814676223127, 0.5284722226850825, 0.5553238780651018, 0.5879776808193284, 0.6250517554973642, 0.664255995845212, 0.7030565446172357, 0.7394369623551168, 0.7722909350278234, 0.8013446238184518, 0.8270783712964883, 0.8509549895186226, 0.8745658258893296, 0.8984267900951174, 0.9217809749343089, 0.9425577166469954, 0.9564907315112912, 0.9555726071586231, 0.929533394015509, 0.8722875262232705, 0.7879706271215043, 0.689192784718906, 0.5872665055880034, 0.4874205057431604, 0.39216290971573176, 0.30435446047152825, 0.227623273921172, 0.1655730902171001, 0.12065341834324494, 0.0934258826293326, 0.08262030541642407, 0.08578250570203183, 0.09993075476713964, 0.12172974376450774, 0.14751343880148526, 0.17383995917406336, 0.1979998796364475, 0.21815782469155487, 0.233297475465023, 0.2431078967578036, 0.2478772018187848, 0.24841281514090968, 0.24597615209635454, 0.24219665552204428, 0.23891755520366853, 0.23793704882569, 0.24066378092582352 ], [ 0.47785572866695036, 0.48692490142446887, 0.4990456945823025, 0.515449668812381, 0.537221382361578, 0.5648988312041952, 0.5980844576372901, 0.6352635145366079, 0.6740532091060573, 0.7119008583804571, 0.7468785868096142, 0.7780824100519818, 0.8055087704136737, 0.8299006013442027, 0.8529369847561972, 0.8762934892090806, 0.9003719304969413, 0.9241330324922666, 0.9451186086991472, 0.9586043662873279, 0.9560069197444345, 0.9264444637064568, 0.8636127946861538, 0.7724728500678125, 0.6676914404286166, 0.5626701745549314, 0.4633603300526837, 0.3714552654509556, 0.2884797159405936, 0.2168883754151088, 0.15945029398452037, 0.11814611731927283, 0.09340593786216755, 0.08407868289425224, 0.08796226838749066, 0.10236028407508435, 0.12417706166270948, 0.14988703935417946, 0.17613100527290493, 0.20025434884196058, 0.2204568867640017, 0.23574318209602646, 0.24581182774003224, 0.25094934864369345, 0.25194811450682664, 0.25003690929389094, 0.2467912366057512, 0.24397989966058825, 0.24331712047142268, 0.24614141505827003 ], [ 0.48264447963821644, 0.4926161743116077, 0.5057024899696094, 0.5230998276827243, 0.5458228469597775, 0.5743006427767785, 0.6080008247719016, 0.6452748398594466, 0.6836392377007681, 0.7205119009786964, 0.754049079721033, 0.7835563777279424, 0.8093186357383447, 0.8323645503287299, 0.854591493179803, 0.877741841022964, 0.9020828022485261, 0.9262739308047813, 0.9474579417584096, 0.9604448610197107, 0.9560824838165436, 0.9229055786216391, 0.8544168565909159, 0.7564301084021727, 0.6457074057806849, 0.5377238213537956, 0.43907720157512103, 0.35060313128585724, 0.2724928807078115, 0.2060493118111124, 0.15323199294562606, 0.11556921169109868, 0.09336168112770094, 0.08556824929276563, 0.09022552534853923, 0.10491044128162519, 0.12676212103613016, 0.1523984758097291, 0.17854992767475075, 0.2026220637221815, 0.22285307159111223, 0.23826989529401765, 0.248580976312673, 0.2540708135124023, 0.2555159430105151, 0.2541119213251568, 0.25138058816574727, 0.24901814524108656, 0.24865826478550873, 0.2515730115407587 ], [ 0.48737632854945123, 0.4982290142271609, 0.5122533159183661, 0.5306125154884309, 0.5542550510458261, 0.5835060393119997, 0.6177023423821628, 0.6550609120012006, 0.692989921543107, 0.7288670323744263, 0.7609276998172729, 0.7886938724679257, 0.8127561633866601, 0.8344528110630013, 0.8559005598145166, 0.8788903971249448, 0.903534980250384, 0.9281751185126663, 0.9495453002396138, 0.9619838531724441, 0.9557791566650755, 0.9189192524151655, 0.8447462406104116, 0.7399493739947198, 0.6233979968970261, 0.5126074654241323, 0.41474376379390393, 0.32975066696827293, 0.2565040080331868, 0.195185329648484, 0.146972312568576, 0.11295726235895798, 0.09331327296359249, 0.08709964937926895, 0.09257765002360208, 0.10758379995063239, 0.12948492904541453, 0.15504585885256206, 0.18109373942031803, 0.20509936359536907, 0.22534216903263388, 0.24087279702679426, 0.251409769562526, 0.25723511230789997, 0.2591088155991071, 0.25819274909343554, 0.2559555351221244, 0.25402271703370227, 0.25395080694806615, 0.25694892587230794 ], [ 0.49203929246891975, 0.5037491612740541, 0.5186815354653558, 0.5379687900255667, 0.5624969566518039, 0.5924921921048665, 0.6271648473542415, 0.6645970535840652, 0.7020812762124977, 0.7369440799964209, 0.7674947866201981, 0.7934779950126529, 0.8158066242500441, 0.8361510614240208, 0.8568480734634604, 0.879719289850167, 0.9047034328687205, 0.9298061713072541, 0.9513470109150518, 0.9631896946626315, 0.9550761572226995, 0.9144896978438628, 0.8346512978403731, 0.7231448757835627, 0.6009282697096459, 0.4875070251676482, 0.3905361036610443, 0.3090440607680032, 0.24062464808560818, 0.18437704147940548, 0.14072674226305693, 0.11034615441662254, 0.09328199592414976, 0.08868422845185664, 0.0950241734470385, 0.11038274130739512, 0.13234504677037529, 0.15782688978034554, 0.1837591426052012, 0.2076823758424835, 0.22791985297112477, 0.24354705260776643, 0.2542927289193652, 0.26043598545342983, 0.26271962448576136, 0.2622715000469116, 0.26050761534069766, 0.2589848878261064, 0.25918600171432077, 0.26226047142548126 ], [ 0.4966215047967587, 0.5091625974613432, 0.5249709010519211, 0.5451502210304993, 0.5705280705967679, 0.6012367218916657, 0.6363644377983321, 0.6738586859403448, 0.7108894306226912, 0.7447212840801725, 0.7737317462964317, 0.7978939697594657, 0.8184587379636946, 0.8374482845197495, 0.85742006600289, 0.8802096059136407, 0.9055628734927067, 0.9311351331419163, 0.9528263413020398, 0.9640274025886122, 0.9539523765437643, 0.9096234560117339, 0.8241863859432133, 0.706137491630803, 0.5784697284619713, 0.4626125125045474, 0.3666315433947697, 0.28862948046962345, 0.22496628012302633, 0.1737053078964944, 0.13455134343865982, 0.10777252654205516, 0.09328983259151613, 0.0903336955220354, 0.09757043149551226, 0.11330904956900034, 0.13534135970410643, 0.16073876487595862, 0.1865424890231394, 0.21036700247981288, 0.23058168059495754, 0.24628781624249158, 0.25722447811733395, 0.26366740493770763, 0.266341641290482, 0.26634082084997623, 0.2650290590909002, 0.26389674401140967, 0.2643559898668472, 0.26749987266320485 ], [ 0.5011112390150279, 0.5144555737041417, 0.5311055845158911, 0.55213892008328, 0.5783284703722673, 0.6097177163637826, 0.6452774721131009, 0.6828212965261405, 0.7193905656506067, 0.752177228929324, 0.7796209969241187, 0.8019290939695981, 0.820704404988608, 0.8383367198942725, 0.8576049409462704, 0.8803437158515093, 0.9060881488524413, 0.932128987521393, 0.9539441708528439, 0.9644595242038219, 0.95238681288769, 0.9043301846175672, 0.8134104168429631, 0.6890539043796322, 0.5561987288828832, 0.4381159337892106, 0.3432062527833527, 0.2686509771660105, 0.2096387425530588, 0.16325012922084947, 0.1285019707223587, 0.10527321501569942, 0.09335906112165993, 0.09205981415573739, 0.10022126016796418, 0.11636345824570882, 0.13847189512526847, 0.16377810393556846, 0.18943975196664842, 0.21314891185697887, 0.23332309373404014, 0.24909023679037712, 0.2601997500370048, 0.2669235791229006, 0.26996851612927447, 0.27039388666275116, 0.26951276548580494, 0.268751149396526, 0.26945375341659156, 0.27266021740244795 ], [ 0.5054969332990403, 0.5196146380471091, 0.5370702097887035, 0.5589175759213969, 0.5858788395745776, 0.6179137620127451, 0.6538805852617338, 0.6914604208382555, 0.727560857717012, 0.7592907643708022, 0.7851458794963291, 0.80557260100042, 0.8225383278257359, 0.8388116778712408, 0.8573936404828368, 0.8801055934776014, 0.9062546607143882, 0.9327542475070223, 0.9546600175607242, 0.9644474420150704, 0.9503590484551339, 0.8986233414251837, 0.8023878219465799, 0.6720252625314866, 0.5342944414475073, 0.4142088057128244, 0.320432658952094, 0.24924836371241232, 0.19474868807368984, 0.15308956631870574, 0.1226335185625369, 0.10288472037972585, 0.09351187493657842, 0.09387413051717497, 0.10298077822089102, 0.11954532208254842, 0.14173371960365133, 0.16694090893205937, 0.19244651099299825, 0.21602353618138392, 0.2361394225702692, 0.25194946393856305, 0.2632133926247597, 0.27019895561619584, 0.27359427407694153, 0.2744243876111124, 0.2739522765778352, 0.2735417075675549, 0.27447307014101546, 0.2777354086842289 ], [ 0.5097672166002185, 0.5246266658351955, 0.5428498890803725, 0.5654694958496008, 0.59316051356889, 0.6258039915543441, 0.6621507249162857, 0.6997516430035369, 0.7353764321617522, 0.7660409232077061, 0.7902905446027094, 0.8088154709047731, 0.8239576710780505, 0.8388713354768187, 0.8567797671950037, 0.8794811243531926, 0.9060388144101592, 0.9329776358573688, 0.954933207319662, 0.9639526325203729, 0.9478496668806233, 0.8925203321720854, 0.7911894094101665, 0.6551850677244604, 0.5129362453326206, 0.39107919896916443, 0.29847665395568235, 0.23055509960549253, 0.18039809242284088, 0.14329870959147417, 0.1169992048322227, 0.10064270399729636, 0.09377003183143173, 0.09578774427297254, 0.10585227665142727, 0.12285272814268156, 0.14512293228965412, 0.1702225555094814, 0.19555795056127778, 0.2189860752613696, 0.23902589192769802, 0.2548606549400682, 0.2662603740689267, 0.2734882224734579, 0.2772133094035166, 0.27842651297712045, 0.2783417497033263, 0.2782627233750086, 0.27940846795847885, 0.28272011672280584 ], [ 0.5139109368536574, 0.5294788925582012, 0.5484302632383019, 0.5717786536950129, 0.600155535743268, 0.63336814786639, 0.6700652100526373, 0.7076706196476088, 0.742813332121763, 0.7724068418764777, 0.7950398272587724, 0.8116502220971659, 0.8249617874315078, 0.8385165662903034, 0.8557596775321077, 0.8784584021174513, 0.9054184845911597, 0.9327668169477314, 0.954724034352554, 0.9629377746510323, 0.9448405362395716, 0.8860417417208533, 0.7798906878757432, 0.6386661009753463, 0.49230043710586824, 0.36890823314138005, 0.27749463378621836, 0.21269622579077513, 0.16668284653820264, 0.13394871444413026, 0.111649902207278, 0.09858152089405514, 0.09415453630978043, 0.09781112355999766, 0.10883820263609789, 0.12628276412156958, 0.14863474248042707, 0.17361781623127714, 0.19876887247182506, 0.22203150653258885, 0.24197763022286006, 0.25781898201877995, 0.26933578739068503, 0.2767863079867579, 0.2808203779555673, 0.28239493358756423, 0.2826759285945791, 0.2829091640193363, 0.28425517955913204, 0.28760973133049805 ], [ 0.517917192011667, 0.5341589501034374, 0.553797546862074, 0.5778297445024547, 0.6068467243492183, 0.6405866649796376, 0.6776018143812038, 0.7151931320347628, 0.7498475094818802, 0.7783676927351852, 0.7993791236823949, 0.8140707111338258, 0.8255520042935872, 0.8377508143108445, 0.8543325600174507, 0.8770280107862195, 0.9043734823934454, 0.9320911408547207, 0.9539948477934092, 0.961367724132161, 0.9413149411314364, 0.879209652422587, 0.7685672281247846, 0.622596403506382, 0.47255615626912595, 0.3478659710665391, 0.25763044586170813, 0.19578640471895703, 0.1536914626584952, 0.12510592041213908, 0.1066335270432347, 0.0967337942962867, 0.09468535752634577, 0.09995396122959888, 0.11194020418075523, 0.12983174702518185, 0.15226360112345894, 0.17712091113527195, 0.20207372111360333, 0.22515460111185548, 0.24498968103123914, 0.26081964049249734, 0.27243485458349886, 0.28008837928761915, 0.28441058802266517, 0.28632478282737117, 0.2869501137095327, 0.28747662014383285, 0.28900909764300997, 0.29240031514729364 ], [ 0.5217753646598656, 0.5386549071655202, 0.5589385786420847, 0.5836082458910179, 0.6132177495309172, 0.6474407661741552, 0.684738876637511, 0.7222951717560177, 0.7564548453141898, 0.7839026374354633, 0.803294284144508, 0.8160719590162961, 0.8257314600698943, 0.8365800046783826, 0.8525005044763059, 0.8751832900128287, 0.9028860010851294, 0.930922366652455, 0.9527110488621773, 0.9592103889536859, 0.9372576142995614, 0.8720454879537962, 0.7572898368819805, 0.6070945770199474, 0.4538604467389399, 0.32810671857520063, 0.23901237335126824, 0.17992813071119074, 0.1415039245806613, 0.11683106955247236, 0.10199449421902085, 0.09513003625528549, 0.09538118392747508, 0.10222506687357347, 0.11515920112166755, 0.1334954093432964, 0.15600335413848065, 0.18072557890025842, 0.2054666197572349, 0.22834994533502218, 0.24805701710893885, 0.2638578576146805, 0.2755529304090443, 0.2833898399756704, 0.28797938999720607, 0.290211636655611, 0.2911601321665963, 0.291961267276852, 0.2936667310548224, 0.2970885579453058 ], [ 0.5254751610370338, 0.5429553145856082, 0.5638408772594659, 0.5891004856982139, 0.619253219719983, 0.6539125786485278, 0.6914554391866156, 0.7289530653625236, 0.7626112080447089, 0.7889908116860661, 0.8067715357034486, 0.8176500149622427, 0.8255049778703786, 0.8350124777245775, 0.8502685613907128, 0.8729205787833014, 0.9009410319698935, 0.9292353441335497, 0.9508420019558058, 0.9564375426671887, 0.9326547697043736, 0.8645679482699958, 0.7461206468239191, 0.5922648808722628, 0.4363523836785525, 0.3097638670419443, 0.2217503362824525, 0.16521018100623597, 0.13019071048707131, 0.10917863763612967, 0.09777324497593887, 0.09379831769764135, 0.09625921462075815, 0.1046322880090007, 0.11849545936494876, 0.13726904887480496, 0.1598473945076343, 0.18442516108274198, 0.2089414155965469, 0.23161196700059772, 0.2511745566056379, 0.26692890208827935, 0.2786855059315515, 0.286686326959045, 0.2915225651034218, 0.29405149295532607, 0.29530230761537407, 0.2963598279075183, 0.2982251620511849, 0.30167173222635785 ], [ 0.529006655355893, 0.5470492564242497, 0.5684927030282545, 0.5942937152222509, 0.6249387761247028, 0.6599852635481216, 0.6977314155691554, 0.7351436432026588, 0.7682925583818061, 0.7936113524506312, 0.8097974480387711, 0.8188018632804426, 0.8248789640197964, 0.8330589301312988, 0.8476447840007204, 0.8702394307053146, 0.8985267570798737, 0.9270086452918618, 0.9483618668010314, 0.9530256068235062, 0.9274942628169137, 0.8567914616258844, 0.7351103133269753, 0.5781927498027273, 0.42014620164222305, 0.2929446865791957, 0.20593352737825454, 0.1517063766123229, 0.11981201318430401, 0.10219628911005707, 0.09400585318277999, 0.09276399019485992, 0.0973349867135449, 0.10718245402335158, 0.12194865687930823, 0.14114764771996446, 0.16378880055786837, 0.1882126922684748, 0.21249173096978713, 0.23493496537222902, 0.25433718112097825, 0.27002809416309914, 0.2818282118472727, 0.2899737066691302, 0.2950362134395195, 0.29784075050477976, 0.2993734303227387, 0.30066953442819505, 0.3026820048889886, 0.3061476502857304 ], [ 0.5323603404113755, 0.5509264076102336, 0.5728831252635341, 0.5991761870376996, 0.6302611935843874, 0.6656431593724552, 0.7035477864921506, 0.740844457189639, 0.7734751108001371, 0.7977434792975902, 0.8123589549966486, 0.8195253755886264, 0.8238613185490262, 0.830732345454571, 0.8446402403968236, 0.8671427905206405, 0.8956349203911035, 0.9242251449268366, 0.9452503551892057, 0.9489564237345478, 0.9217660002833737, 0.8487253981502959, 0.7242964327251901, 0.564941421758798, 0.405323535336476, 0.2777259022094181, 0.1916287069175795, 0.1394747129081222, 0.1104171771795367, 0.09592446376410224, 0.09072371373905175, 0.0920494607259491, 0.09862223732489472, 0.10988133754665774, 0.1255179375308657, 0.14512596535720257, 0.16782045530625556, 0.19208099017031932, 0.21611101816962058, 0.23831314389642022, 0.25753975519492284, 0.27315081619326054, 0.28497682164239263, 0.29324807078892, 0.2985167415460576, 0.30157618781718415, 0.30337072770765194, 0.3048880931321219, 0.3070353658825375, 0.3105146228757127 ], [ 0.5355271855677416, 0.5545770990451214, 0.5770020951004627, 0.6037372360092306, 0.6352084856084131, 0.6708719359535893, 0.7088868223153691, 0.7460340511152346, 0.7781355620703384, 0.8013666422968839, 0.8144434439428125, 0.8198193084589361, 0.8224613449009445, 0.8280478951946312, 0.8412689786093575, 0.8636371166607236, 0.8922611771206451, 0.9208725546425924, 0.941493407358566, 0.9442180259673202, 0.9154626935981536, 0.8403741698140581, 0.7137032597719754, 0.5525502861423182, 0.3919270973828993, 0.2641512155791428, 0.1788793408564564, 0.12855690278181786, 0.1020443649471583, 0.0903960996326224, 0.08795331503264592, 0.09167401973150224, 0.10013279865199987, 0.11273362929882214, 0.12920195275745183, 0.14919861016765845, 0.1719351459434073, 0.19602274122563368, 0.21979261543282408, 0.24174064456505934, 0.26077714678477787, 0.27629252350450767, 0.28812725459188343, 0.2965057316118665, 0.3019608496843389, 0.3052549420586761, 0.3072918355170113, 0.3090136494153405, 0.3112838050413687, 0.31477141956840005 ], [ 0.538498703338097, 0.557992391062361, 0.5808405231418097, 0.6079673617705295, 0.639770010997701, 0.6756587553444934, 0.7137323283022388, 0.7506922851846556, 0.7822513978870602, 0.8044607495875146, 0.8160389244774686, 0.8196833449886822, 0.8206896466224098, 0.8250227908024031, 0.8375479245334848, 0.8597324290607731, 0.8884054208916138, 0.916943918608525, 0.9370837706799406, 0.9388053813465445, 0.9085809899327089, 0.8317382781763216, 0.7033427323769916, 0.5410352666510092, 0.37996176091057415, 0.25223163373233026, 0.1677056791371585, 0.11897835152970304, 0.09472045644337768, 0.08563649301580811, 0.08571609554820214, 0.09165372184085663, 0.1018765242999421, 0.11574292377954443, 0.13299889277780486, 0.15336009325714983, 0.17612564457997715, 0.20003057874884056, 0.22352980202391498, 0.245211582887368, 0.26404424826712763, 0.2794487554029475, 0.291275578594103, 0.2997432171272384, 0.305365518981147, 0.30887448822027996, 0.3111347697982416, 0.3130447542976845, 0.31542629936972, 0.31891723088762974 ], [ 0.5412670259114089, 0.5611641561033605, 0.5843903608455404, 0.6118583105849534, 0.6439365790801765, 0.6799924351306782, 0.7180699068364482, 0.754800713372036, 0.785801288899968, 0.8070064885590104, 0.8171342871157448, 0.8191181772035182, 0.8185580001871481, 0.821676066583339, 0.8334966904196954, 0.8554422562267104, 0.8840720837554995, 0.9124380834685021, 0.9320214479421944, 0.9327210517745732, 0.9011229097873752, 0.8228163384561433, 0.6932166993196673, 0.5303910979968925, 0.3694015916985778, 0.2419484517142081, 0.15810575553394146, 0.1107485557785245, 0.08846117704659962, 0.08166329279390672, 0.0840283829352757, 0.09200131780585974, 0.10386124500206129, 0.11891171422869817, 0.13690650960014306, 0.15760486795074335, 0.1803847722436943, 0.2040971519778947, 0.2273158507276694, 0.24872008252074757, 0.2673359975100539, 0.28261514614730276, 0.2944180128217036, 0.3029572659089608, 0.30872799857078675, 0.31243261868881905, 0.314897899792052, 0.31698033234776407, 0.3194622078819166, 0.32295163225452267 ], [ 0.5438249931449155, 0.5640851713003604, 0.5876446839524113, 0.6154031541818719, 0.6477005503095715, 0.6838636089453097, 0.7218872285596883, 0.7583430076661619, 0.78876558697478, 0.8089857555215039, 0.817719660396775, 0.8181256241707004, 0.8160791957586665, 0.818028273647878, 0.8291372706555138, 0.8507834528540814, 0.8792703850109393, 0.9073601566850505, 0.9263139669812478, 0.9259756760710366, 0.8930973997610763, 0.8136080909494322, 0.6833200767696551, 0.5205949518742091, 0.3601971969676139, 0.23325785960125645, 0.15005717925420847, 0.10386189218793629, 0.08327144037957668, 0.07848662261792139, 0.08290141215017977, 0.09272623540457303, 0.10609275181786482, 0.12224139602999162, 0.14092213413304133, 0.1619273578979693, 0.18470544837880454, 0.20821518534430705, 0.2311440764883002, 0.2522603087305005, 0.27064739858661524, 0.2857874357054347, 0.2975509301587838, 0.30614482186795516, 0.31204579284229056, 0.31592742333411966, 0.31857992184038286, 0.3208196510710277, 0.3233912383650465, 0.32687454976933805 ], [ 0.5461662537257687, 0.5667492211809606, 0.5905977754891065, 0.6185963628968774, 0.6510559288075775, 0.6872648784028574, 0.7251743021521933, 0.7613054170771808, 0.7911269303209222, 0.8103822097941168, 0.8177868701516783, 0.8167087779716145, 0.813266842128684, 0.8141010664651623, 0.8244936017040548, 0.8457758623384146, 0.8740144610001443, 0.9017219443455864, 0.9199764139001935, 0.9185881827949077, 0.884521694582505, 0.8041183921430229, 0.6736444269713976, 0.5116106967606245, 0.3522820732291506, 0.22609605971845909, 0.14351951218401038, 0.09829873934886446, 0.07914588464277683, 0.07610932128216918, 0.08234141773907389, 0.0938346063939155, 0.10857480486254156, 0.12573227818556265, 0.14504268950548915, 0.16632197632525825, 0.18908072807836585, 0.21237752800210102, 0.23500788033664377, 0.25582649999041274, 0.2739735417409739, 0.2889614801219554, 0.3006708593871459, 0.3093030289132758, 0.31531664887888294, 0.3193572702052851, 0.3221798343797587, 0.32456229179775103, 0.32721341590074743, 0.3306862278338461 ], [ 0.5482853814167111, 0.569151209604217, 0.593245205001512, 0.621433870270946, 0.653998443398653, 0.6901909503467093, 0.727923730530691, 0.7636772415187336, 0.7928709599653079, 0.8111819693230464, 0.8173299950342039, 0.8148721662634951, 0.8101351385328966, 0.8099166676878681, 0.8195909643019019, 0.8404418109009231, 0.8683232550017055, 0.8955421836146593, 0.9130311783842318, 0.9105856511807218, 0.8754221195735004, 0.7943617236278294, 0.664181235218648, 0.5033931453366692, 0.3455779572173864, 0.2203841814940456, 0.13843699782166852, 0.0940268608322885, 0.07606957477685128, 0.07452728878518478, 0.08234979398341735, 0.09532933599948856, 0.11130916557892168, 0.12938360271349292, 0.14926470241348133, 0.17078313860501015, 0.1935038291286596, 0.2165771941160226, 0.2389007881046385, 0.25941299718691874, 0.27730962226962425, 0.2921332613360503, 0.30377448708080085, 0.312429225554882, 0.3185385441583699, 0.32272078690715256, 0.32569691406874635, 0.32820812208878025, 0.33092905314138776, 0.3343871986027917 ], [ 0.5501780085384503, 0.5712872777337666, 0.5955838997854265, 0.6239131262225768, 0.6565256138142354, 0.692638753302212, 0.7301309399187764, 0.7654512921297207, 0.7939871351120734, 0.81137446624741, 0.816345994753179, 0.812621916450102, 0.8066986242687595, 0.8054972074741387, 0.8144552082127623, 0.8348054440220444, 0.8622200597862252, 0.8888461140189776, 0.9055073893818394, 0.9020027639642861, 0.8658340309035841, 0.7843641001365431, 0.6549241807312022, 0.49589183883871624, 0.3399992585957609, 0.2160326712348421, 0.1347414230636631, 0.09100296994826884, 0.07401883839473511, 0.07372992339688345, 0.08292331552366561, 0.09721021094041127, 0.11429565050408341, 0.1331935718794004, 0.15358431399417594, 0.17530526997631446, 0.19796815074063945, 0.2208073946881582, 0.2428164837253013, 0.2630142700384427, 0.2806509580379315, 0.29529889630593065, 0.30685865916718813, 0.3155209394706383, 0.321709674566996, 0.3260168427090994, 0.32913069307965565, 0.33175726966070557, 0.334538722322011, 0.3379782532399473 ], [ 0.5518409791080526, 0.5731549206312141, 0.5976122031917006, 0.6260331360347122, 0.6586367990388606, 0.6946075273892498, 0.7317943679086949, 0.766624301973079, 0.7944696027429645, 0.8109534822318055, 0.8148353593186031, 0.8099659033438721, 0.8029719257261748, 0.8008639590873677, 0.8091117832881035, 0.8288919497570837, 0.8557317348131409, 0.8816643853376648, 0.8974400690921481, 0.8928808326977807, 0.8558008112547271, 0.7741605421071241, 0.6458701067321987, 0.4890541302057662, 0.33545665702703636, 0.21294505785620488, 0.13235493548222688, 0.08917439709501729, 0.07296220115878971, 0.07370063355339374, 0.08405441024225457, 0.09947404160273954, 0.11753220441255785, 0.13715938312491982, 0.1579972914173764, 0.1798828099494172, 0.20246728562340188, 0.22506156184568943, 0.24674883715275642, 0.26662494048065255, 0.2839930054128736, 0.29845464531727145, 0.3099203821155252, 0.3185758820520248, 0.32482844276539957, 0.32924453142198057, 0.33248093756878383, 0.33521009781855476, 0.33804322897786127, 0.34146041494341417 ], [ 0.5532725243501155, 0.5747530888218023, 0.5993299148663468, 0.6277944827196549, 0.6603332252322107, 0.6960988826972719, 0.7329155975702538, 0.7671972469933236, 0.794318019601542, 0.8099183838153483, 0.8128026922321221, 0.8069138608529673, 0.7989695270803461, 0.7960365465673547, 0.8035845730845665, 0.8227267492629361, 0.848887752347823, 0.874031803337716, 0.8888690847852098, 0.8832664218123436, 0.8453721178224787, 0.7637916890198169, 0.6370189351070774, 0.48282750130490404, 0.33185994958510195, 0.21102109769524607, 0.1311926891843106, 0.08848078711590857, 0.07286138706730971, 0.07441740762542737, 0.08573147563760586, 0.10211483370068364, 0.12101499064490928, 0.14127727144539415, 0.16249904109298408, 0.1845102146531833, 0.20699502683736104, 0.22933336656240888, 0.2506919271136502, 0.27023980289241417, 0.2873313734533264, 0.30159691937624655, 0.31295682371681494, 0.32159194293684823, 0.32789344693346634, 0.332403155064925, 0.3357476273260746, 0.3385671823727724, 0.341443587327804, 0.3448349136959775 ] ], "zauto": true, "zmax": 0.9644595242038219, "zmin": -0.9644595242038219 }, { "autocolorscale": false, "autocontour": true, "colorbar": { "tickfont": { "size": 8 }, "ticksuffix": "", "x": 1, "y": 0.5 }, "colorscale": [ [ 0.0, "rgb(255,247,251)" ], [ 0.14285714285714285, "rgb(236,231,242)" ], [ 0.2857142857142857, "rgb(208,209,230)" ], [ 0.42857142857142855, "rgb(166,189,219)" ], [ 0.5714285714285714, "rgb(116,169,207)" ], [ 0.7142857142857143, "rgb(54,144,192)" ], [ 0.8571428571428571, "rgb(5,112,176)" ], [ 1.0, "rgb(3,78,123)" ] ], "contours": { "coloring": "heatmap" }, "hoverinfo": "x+y+z", "ncontours": 25, "type": "contour", "x": [ 1e-06, 1.3011511650442548e-06, 1.692994354296022e-06, 2.2028415765056147e-06, 2.866229883678204e-06, 3.729398352432554e-06, 4.852511011181743e-06, 6.3138503555892e-06, 8.215273746089953e-06, 1.0689313005882424e-05, 1.390841207112662e-05, 1.809694657026198e-05, 2.354686311364001e-05, 3.063802837345029e-05, 3.986470631277378e-05, 5.1870009063012666e-05, 6.749072272319499e-05, 8.781563250096393e-05, 0.00011426141253772724, 0.00014867137004306603, 0.00019344392634026088, 0.0002516997901283655, 0.0003274994751669172, 0.0004261263236648159, 0.0005544547624925005, 0.0007214294601814526, 0.000938688782612345, 0.0012213760031100258, 0.0015891948094037057, 0.002067782677737912, 0.0026904978401970136, 0.0035007443993213955, 0.004554997653699184, 0.005926740503884541, 0.007711585311544345, 0.010033938212454078, 0.013055670395116691, 0.01698740074503987, 0.02210317627048227, 0.028759573555516536, 0.03742055263793628, 0.04868979566145066, 0.06335278435066323, 0.0824315491666629, 0.10725590623460621, 0.13955614735503497, 0.18158364372009145, 0.23626776957937787, 0.3074200836506151, 0.4 ], "xaxis": "x2", "y": [ 0.0, 0.02040816326530612, 0.04081632653061224, 0.061224489795918366, 0.08163265306122448, 0.1020408163265306, 0.12244897959183673, 0.14285714285714285, 0.16326530612244897, 0.18367346938775508, 0.2040816326530612, 0.22448979591836732, 0.24489795918367346, 0.26530612244897955, 0.2857142857142857, 0.3061224489795918, 0.32653061224489793, 0.3469387755102041, 0.36734693877551017, 0.3877551020408163, 0.4081632653061224, 0.42857142857142855, 0.44897959183673464, 0.4693877551020408, 0.4897959183673469, 0.5102040816326531, 0.5306122448979591, 0.5510204081632653, 0.5714285714285714, 0.5918367346938775, 0.6122448979591836, 0.6326530612244897, 0.6530612244897959, 0.673469387755102, 0.6938775510204082, 0.7142857142857142, 0.7346938775510203, 0.7551020408163265, 0.7755102040816326, 0.7959183673469387, 0.8163265306122448, 0.836734693877551, 0.8571428571428571, 0.8775510204081632, 0.8979591836734693, 0.9183673469387754, 0.9387755102040816, 0.9591836734693877, 0.9795918367346939, 1.0 ], "yaxis": "y2", "z": [ [ 0.301686975782336, 0.2830702552071149, 0.25788816027744, 0.22521214246951213, 0.18557281259934805, 0.1432065350923429, 0.11085848188714839, 0.10873315123247457, 0.13397077742801566, 0.164450941985803, 0.18712216013949182, 0.1971172837464094, 0.1933894444997929, 0.1770800321797276, 0.15174835548609877, 0.1251993887057385, 0.10981570047920267, 0.11027982391936526, 0.11463007614375677, 0.1127152439252237, 0.10453312333174353, 0.0970571774192001, 0.09546968287981362, 0.09618267622628221, 0.09575856057639942, 0.09642323270605996, 0.1033634105381207, 0.11369640401135846, 0.11816432472680527, 0.11460850850366708, 0.11352509170013436, 0.12975696726093144, 0.16103721416833297, 0.19563405163187034, 0.2267075092586636, 0.2522834519058513, 0.2726137213609257, 0.2885012628245793, 0.3005289846669156, 0.30885945623761457, 0.3132898086579935, 0.31334230494494103, 0.30830942084031654, 0.29727632657895603, 0.2791964160562305, 0.2531330940544766, 0.21886780664394861, 0.1783591430990721, 0.1393487647140043, 0.12105971363603539 ], [ 0.30088288695626153, 0.2822689189471937, 0.25707035570830644, 0.22427903495067908, 0.18426561685673387, 0.14094846463156624, 0.10676386494710564, 0.1032482542698107, 0.12886419169147997, 0.1601803555957614, 0.18350941957202052, 0.19396074633546273, 0.1905983686453805, 0.1746472820565994, 0.14964507414034595, 0.12323862812776647, 0.10760075125005428, 0.10750367941088677, 0.11133553196037384, 0.1091507639260221, 0.1009331000244321, 0.0933747379130726, 0.09146455096566086, 0.0918992002516158, 0.09168308541551724, 0.0931059405311191, 0.10107044499074772, 0.11231541508077542, 0.11747584008157524, 0.11439082361079655, 0.11339381505617861, 0.1291641672725245, 0.159739525920669, 0.19368271308528423, 0.22427181016129047, 0.24959942958166073, 0.26994131465658594, 0.2860655484956183, 0.29847411780889815, 0.30723780864632244, 0.3120820246197235, 0.31248928633695555, 0.30774064178967425, 0.2969285569130865, 0.2790238341325302, 0.2531124841723139, 0.2190028385485487, 0.17868570013133184, 0.13992222963121198, 0.12180358151867417 ], [ 0.3000861676434585, 0.28153059976699757, 0.256406335395406, 0.22364146949097197, 0.18347073661737037, 0.1395125071905703, 0.103754815032813, 0.09864172436351919, 0.12429307121143668, 0.1562202721602478, 0.18006659680583997, 0.19090421634129398, 0.18791365199956853, 0.17241442052409384, 0.14792497470874588, 0.12189216456846137, 0.10609839689675947, 0.10527824325556541, 0.10841894867734975, 0.1059414000478943, 0.09781250223173488, 0.09030008865467476, 0.08805568164655384, 0.08817251967590238, 0.08821510761596299, 0.09051785859831747, 0.09955890055850766, 0.11170747533724237, 0.11763890643994354, 0.11521936948722779, 0.11444818836857201, 0.1295485334790324, 0.1590336283535256, 0.19200357575921193, 0.22189550400295852, 0.24685322946499558, 0.2671551244119458, 0.2835118298634738, 0.2963229414994985, 0.3055515969579757, 0.3108434221572503, 0.31164000211492654, 0.30721463260581944, 0.29667444975501434, 0.279019078459039, 0.2533746954329286, 0.21960437379835676, 0.17977243513072252, 0.1416761008496877, 0.12400833636677545 ], [ 0.29928924503656357, 0.2808470243787902, 0.25588709377312674, 0.22328965453524768, 0.18317941229985085, 0.1389053711835846, 0.10191099319039004, 0.0950463853220774, 0.12033563409922438, 0.15260588033429753, 0.1768033239162089, 0.18794051104823772, 0.18531532697731623, 0.17034905467267542, 0.1465383499090638, 0.12108509395753343, 0.10522268116043589, 0.10354240730550608, 0.10583803913387783, 0.10304981997011042, 0.09513002559492789, 0.08779167458500481, 0.08522864591413491, 0.08501146770173025, 0.08535391435971276, 0.08862147138041143, 0.09876641805097157, 0.11181042678681742, 0.11858712014909532, 0.11700566634057863, 0.11658046555451021, 0.13083810488166306, 0.15888356591188268, 0.19057377033921344, 0.21955934676423833, 0.24402793847254134, 0.2642418808503931, 0.2808313967396931, 0.2940708775784138, 0.3037990972212911, 0.3095737915355076, 0.3107947501221944, 0.30673152026677225, 0.29651340521798836, 0.27918006785664484, 0.25391463566755473, 0.220660564055791, 0.18159056753809572, 0.14453983545686827, 0.12755921666334932 ], [ 0.2984835324195194, 0.2802083466833362, 0.25550095604106526, 0.2232087371591795, 0.18337206453733046, 0.13910805080325409, 0.10126304201831447, 0.09257408079606584, 0.11706688862134078, 0.14937054777321843, 0.1737274564436854, 0.18506034426187087, 0.18278031862548844, 0.16841335776416363, 0.14542477789273245, 0.1207220381040472, 0.10486140631729929, 0.10221729226931497, 0.10354266943907595, 0.10043277979099716, 0.09283208802477533, 0.08578766471444267, 0.08294999696363214, 0.08241142963507689, 0.08308058760219811, 0.08735168837106937, 0.09860429789864494, 0.11254106279910193, 0.12023126608975328, 0.1196285153365723, 0.11964150409366808, 0.1329282614783722, 0.1592374861003274, 0.1893637651080185, 0.21724114683545184, 0.2411052762110343, 0.2611878744370483, 0.2780157191583066, 0.29171383040701554, 0.30197908897063713, 0.30827327056233084, 0.30995392108258746, 0.3062911745330966, 0.29644403308281003, 0.27950299108583904, 0.25472362029386075, 0.22215191411219176, 0.18409425221016096, 0.14840629116676352, 0.13228992306649237 ], [ 0.29765946896418194, 0.279603236933252, 0.2552337787346738, 0.22337926564954355, 0.1840193978929669, 0.14007795062502307, 0.10178896408410443, 0.09130098844647837, 0.11455371043701684, 0.1465443808391402, 0.17084481455738426, 0.1822526085215088, 0.18028316578515807, 0.16656557039028866, 0.1445166090715016, 0.12069551043590562, 0.10488806373602388, 0.10121313674004036, 0.10147886899823895, 0.09804528722747632, 0.09085910309371968, 0.08421400498572208, 0.0811707510097697, 0.08035412661864301, 0.08135938401238345, 0.08662428485646072, 0.09896877224346062, 0.11380370940757059, 0.12246807451603875, 0.1229481597570618, 0.12346097818526428, 0.13569335855848677, 0.1600311270371086, 0.1883385417854844, 0.2149163198465203, 0.238065927009747, 0.2579792090402391, 0.2750566636183684, 0.2892483564150165, 0.30009096954903774, 0.30694241275368933, 0.3091180385724601, 0.3058932391407051, 0.29646419863552903, 0.27998241190121587, 0.2557896611125164, 0.22405219465613316, 0.1872239498633964, 0.15314452515672042, 0.1380078028617611 ], [ 0.2968065713428194, 0.27901899438087635, 0.2550692030297304, 0.22377779027917064, 0.18508396427999174, 0.14175309458376725, 0.10341820919041755, 0.09125489829920823, 0.11284879461410512, 0.1441525073242222, 0.16815892942567204, 0.17950473726857077, 0.17779685608514526, 0.1647616717376845, 0.14374272328584878, 0.12089432222470366, 0.10517319521016456, 0.1004366145440827, 0.09959299198271998, 0.09584485150068399, 0.08915200650005528, 0.08299288885859195, 0.07983139865468243, 0.07880813860433218, 0.08014075628448154, 0.08634597980465171, 0.09975250375125005, 0.11549895673199011, 0.12518902111460084, 0.12681978169110228, 0.1278653169923149, 0.1389986154953962, 0.1611919454718045, 0.18745902534781422, 0.21255854945839542, 0.2348899281833159, 0.25460209413618157, 0.27194673788271456, 0.2866718501412883, 0.2981348748994215, 0.3055822570081021, 0.30828780025139146, 0.30553716806209597, 0.2965710831576326, 0.28061141134933215, 0.25709784974291483, 0.2263296034232081, 0.19091038369787916, 0.15861267449069316, 0.14451571648702258 ], [ 0.29591349573642584, 0.27844167911219303, 0.2549889474682098, 0.22437755586942987, 0.18652199207917625, 0.14405747555414866, 0.10604214706261583, 0.09240853422138944, 0.11198417945386434, 0.14221320296298146, 0.1656708185425438, 0.17680314225490126, 0.17529374202453746, 0.1629571116861997, 0.1430321705512364, 0.1212108462576253, 0.10559367778324358, 0.0997974608348414, 0.0978355288393653, 0.0937952866445938, 0.08765798926827353, 0.08205038475566985, 0.0788676082132099, 0.07773009612949014, 0.07936567053040783, 0.0864242249279219, 0.10085432040959143, 0.11753113884951975, 0.12828781687624205, 0.1311040959792513, 0.1326907130481672, 0.14271021423882213, 0.16264335398310015, 0.18668366585906282, 0.21014053198443436, 0.23155711010927255, 0.2510431765544623, 0.268679365129133, 0.2839827461059926, 0.29611180530877307, 0.3041943973435547, 0.30746411923395545, 0.30522226572601857, 0.2967612569506481, 0.28138176096829154, 0.25863081228699814, 0.22894807174747284, 0.19507855922638767, 0.16466869142921245, 0.15162730833678678 ], [ 0.29496810870425305, 0.2778562584713341, 0.2549731260809458, 0.22514923814806356, 0.18828528351806018, 0.1469065542168283, 0.10952774204869989, 0.09468141954987422, 0.11196552930679653, 0.1407360653629723, 0.16337882009579474, 0.1741337198842131, 0.17274650582760423, 0.16110850653619085, 0.1423174046005815, 0.12154655252176319, 0.1060392973995385, 0.09921370345244111, 0.09616416692940195, 0.09186962022249404, 0.0863347122036441, 0.08132230968041693, 0.07821580421854145, 0.07706680077792034, 0.07897078210461929, 0.08677528413740727, 0.10218616100760174, 0.11981379180746297, 0.13166589561309078, 0.13567429931588862, 0.13779082420839034, 0.14670266051573122, 0.16430863012431826, 0.18597007143598882, 0.20763478234898014, 0.22804758409708137, 0.2472899127347438, 0.2652491882598378, 0.2811807354441762, 0.29402375418561794, 0.30278105101987207, 0.3066481643627786, 0.3049477299323846, 0.29703076238346193, 0.2822841194749272, 0.2603692092947139, 0.23186861374385417, 0.19965142985244663, 0.1711779802300874, 0.1591755397205441 ], [ 0.2939575653640122, 0.27724676349645183, 0.2550005783516975, 0.22606167983990297, 0.1903230134246454, 0.15021214570982433, 0.11373086855670769, 0.09794941196991755, 0.11276861731994849, 0.13972049791631022, 0.16127851884244446, 0.17148241864492905, 0.17012914418232697, 0.15917522301487538, 0.14153694010512177, 0.12181572846010806, 0.10641670163538869, 0.09861524802336956, 0.0945458420745235, 0.09005178276972534, 0.08515262135852664, 0.080757913663111, 0.07781795930914837, 0.0767594698119905, 0.07889392999790082, 0.08732988130996555, 0.10367706002087163, 0.12227292600573247, 0.13523580621575387, 0.14041971415760707, 0.1430401553664221, 0.15086336859856475, 0.166114214414747, 0.18527660482309452, 0.2050144809483388, 0.22434227518291155, 0.24333098322674204, 0.26165240517288346, 0.27826699571436503, 0.2918738374926226, 0.3013451231407979, 0.30584139804514815, 0.3047126960906659, 0.2973752041852322, 0.2833082449637617, 0.26229225499029035, 0.2350506280530162, 0.20455294697830897, 0.17801799606840152, 0.167016020766476 ], [ 0.29286839341089493, 0.2765964510882966, 0.2550491989274734, 0.22708259043434567, 0.19258331307364568, 0.15388628092906734, 0.1185068942708669, 0.10205855795409446, 0.11433922805166102, 0.13915477171634477, 0.15936279267705103, 0.1688358580831516, 0.16741794946067873, 0.15712080143034324, 0.14063737702979465, 0.12194760443417146, 0.10665122910564417, 0.09794590967606073, 0.09295766815328622, 0.08833687939562627, 0.08409527028309359, 0.08032134834339145, 0.07762512205026209, 0.07674970141829388, 0.07907918780219976, 0.08803629594825042, 0.1052745299207819, 0.12484835332277107, 0.1389227955494162, 0.14524695176722424, 0.14833451499325082, 0.15509494654419545, 0.1679922728905084, 0.18456387714662986, 0.20225434503198333, 0.2204234988993626, 0.23915675221700408, 0.25788713573456284, 0.27524443163321194, 0.2896664209400451, 0.29989026561012333, 0.3050456102471527, 0.3045162813505059, 0.29778984407704157, 0.28444321458656785, 0.26437823169254915, 0.23845308275455962, 0.20971038774725997, 0.18508040386636157, 0.17502702637795906 ], [ 0.29168658166761824, 0.2758879681678424, 0.2550962571514418, 0.22817918380208416, 0.19501457812763337, 0.15784395030391565, 0.12371779325022211, 0.1068407136262845, 0.11659688311332937, 0.1390158685695478, 0.15762199950273498, 0.16618198763302572, 0.16459247041847397, 0.15491419449046911, 0.139574819040604, 0.1218872334156704, 0.1066872285031388, 0.09716417655481123, 0.09138676005783114, 0.08672996101488238, 0.08315776719242639, 0.07999114848735245, 0.07759937682382918, 0.07698461843796726, 0.07948059312878653, 0.08886113726233115, 0.10694389890342244, 0.12749349143325814, 0.14266502236485704, 0.15007946395125818, 0.15358978336343143, 0.1593158253263672, 0.16988252641352028, 0.18379609626518623, 0.19933151367536103, 0.21627558370981398, 0.23475977619063484, 0.25395382098876507, 0.27211792363557563, 0.2874072414726932, 0.298420928114497, 0.3042629472060322, 0.3043576271849843, 0.298269696839264, 0.28567764408214735, 0.26660497889601403, 0.2420355377931321, 0.21505597306130844, 0.19227153437403144, 0.18310774148002082 ], [ 0.2903976720939381, 0.27510351478629086, 0.2551186990624519, 0.2293187384702785, 0.19756648771291455, 0.16200485289121097, 0.12923619201383726, 0.1121281847069781, 0.11944160201593336, 0.1392701816400661, 0.1560443089550051, 0.1635107708315802, 0.16163644150940515, 0.15253081980034291, 0.13831576196514542, 0.12159547782344592, 0.10648741441095494, 0.0962430523757615, 0.08982907824080896, 0.08524332902435454, 0.08234363713266962, 0.0797580721551475, 0.07771410362429806, 0.07741985919324425, 0.08006391145887361, 0.08978814645161837, 0.10866615086426644, 0.13017409372816746, 0.14641284480201341, 0.1548561875490062, 0.15873987758096866, 0.16345982838623047, 0.1717334290704079, 0.18294224949508675, 0.19622644161294497, 0.21188554411904564, 0.2301353673917489, 0.2498556546873748, 0.2688945800695452, 0.2851035189588844, 0.2969423986342155, 0.30349593342830644, 0.3042359390298943, 0.29880962503854946, 0.2869999002987743, 0.2689503409504854, 0.24575898195863838, 0.22052786256126658, 0.19951179026294333, 0.19117576977189807 ], [ 0.28898685447722017, 0.2742250039631871, 0.2550934271688488, 0.23046907497255215, 0.20019075610880535, 0.16629437218015383, 0.13494718177192247, 0.11776390334656905, 0.12276201564500239, 0.13987499756467842, 0.15461616519792273, 0.1608148767795653, 0.15853867472340538, 0.14995343936856376, 0.13683754747851642, 0.12104839524557501, 0.10603166585736352, 0.09516930204303978, 0.0882875348511719, 0.08389254630816041, 0.08166056725178489, 0.07962168629258917, 0.07795258058547304, 0.0780202210827222, 0.08080623760873512, 0.09081536934403815, 0.11043472356852273, 0.13286630254800397, 0.1501275528881975, 0.1595297721602633, 0.16373446241769132, 0.16747514331038488, 0.17350281489480962, 0.18197711846737744, 0.19292380349770102, 0.20724381342630638, 0.22528221945011673, 0.24559904636920624, 0.2655839874560361, 0.2827640533287648, 0.29546083085754854, 0.30274748559325804, 0.30415052167447865, 0.29940442987112376, 0.2883983008927592, 0.2713925620812005, 0.2495864804207901, 0.22607064660300452, 0.20673449052690956, 0.19916450493267154 ], [ 0.2874390633600175, 0.2732342168831772, 0.25499755581574096, 0.23159895236941577, 0.2028416581995228, 0.1706440114916261, 0.14074870013009555, 0.12360674534528934, 0.12644309350292707, 0.14078055227334627, 0.1533228485154292, 0.15809035597095383, 0.1552939102753173, 0.14717288782561244, 0.13512847493531221, 0.12023623254531837, 0.10531552377613411, 0.09394235559322008, 0.08676971723497358, 0.0826915049404412, 0.08111570398041673, 0.07958611103416194, 0.07830513902593787, 0.07875827136089167, 0.0816936426763941, 0.09195101126800416, 0.11225164040653199, 0.13555435267621335, 0.15377982751202002, 0.16406469794373968, 0.16853670773053625, 0.17132301259544055, 0.17515813721438617, 0.18088213599287792, 0.18941341594817698, 0.2023450496130702, 0.22020310424109804, 0.24119411373887867, 0.26219845152389826, 0.280399301736829, 0.29398325581241, 0.30202091708290346, 0.30410080923645083, 0.3000489358959974, 0.28986129658622667, 0.27391062212589523, 0.25348364281307223, 0.23163545956986004, 0.21388447906396515, 0.20702066571475672 ], [ 0.2857390771217721, 0.27211295292582116, 0.25480864214927657, 0.23267839142827448, 0.20547637701282895, 0.1749914858779084, 0.14655109121773122, 0.12953331556713105, 0.13037235594483604, 0.14193237903302364, 0.15214908795872437, 0.15533727162398003, 0.1519036215732166, 0.14418867245033687, 0.13318764906045674, 0.11916216001479181, 0.10434851535466724, 0.09257303672049444, 0.08528567087633697, 0.08164714925702125, 0.08071133692642078, 0.0796553847449435, 0.07876523028382408, 0.07961153934121075, 0.08271735417916826, 0.09320829595520062, 0.11412332221457275, 0.13822818874528126, 0.15734812619731148, 0.16843545819102917, 0.17312123434457954, 0.1749763405748827, 0.17667641084496669, 0.17964610071402712, 0.18569118943996799, 0.19718903284926506, 0.21490565047030755, 0.23665519971265475, 0.2587532196380388, 0.2780214296802162, 0.2925175750528521, 0.3013199320143892, 0.3040863887315132, 0.300738067808665, 0.2913776326226004, 0.2764845104681062, 0.25741893083620176, 0.23717982628690218, 0.22091669520737148, 0.21470211842292142 ], [ 0.2838716195079083, 0.27084317479594733, 0.2545048944418996, 0.23367893513508659, 0.20805522012047015, 0.17928061586207422, 0.15227625367353662, 0.1354376162063889, 0.13444418049580456, 0.14327366284906148, 0.15107966740666004, 0.15256025027974462, 0.14837676467807034, 0.14100946264815178, 0.13102461952475836, 0.11784081544687107, 0.10315233517010383, 0.09108218656336417, 0.08384616018795732, 0.08075575470198335, 0.08044179974668547, 0.07982899723683227, 0.07932489547103763, 0.08055906497831355, 0.08386915188582561, 0.0945997153796966, 0.11605645107853911, 0.14088121037355036, 0.16081713238554016, 0.1726248960807524, 0.17747229862132677, 0.17841832690270232, 0.17804394441800117, 0.17826576532988597, 0.1817601249993776, 0.1917816783296659, 0.2094032151504413, 0.2320014058300913, 0.25526667276232595, 0.2756443294495229, 0.29107253287056994, 0.3006486078612402, 0.304107016458454, 0.30146691781865537, 0.2929364872816816, 0.27909543900750133, 0.26136382898723604, 0.24266733336515725, 0.22779481478511746, 0.22217601461425543 ], [ 0.2818214642825039, 0.26940714973945096, 0.25406536085228276, 0.23457385853497636, 0.21054174520334867, 0.1834611199897232, 0.15785663048108897, 0.14122965550121683, 0.13856233291558132, 0.1447473668390019, 0.15009996442045162, 0.14976890670728651, 0.14473045176254543, 0.13765347409735676, 0.1286588443450705, 0.11629668605685663, 0.1017588450485424, 0.08949916066066575, 0.08246163363506064, 0.08000337985618634, 0.08029213705607878, 0.08009826048969729, 0.0799702750482165, 0.08157811345055087, 0.08513681320932316, 0.09613115371175486, 0.1180543093310464, 0.1435083170397821, 0.16417635802204636, 0.17662273401872514, 0.18158221790381612, 0.18164117974765034, 0.17925592520154582, 0.1767463088990356, 0.1776313707916939, 0.18613619310805496, 0.2037158582583147, 0.2272571283128819, 0.25176047228468246, 0.2732835989031154, 0.2896576652680085, 0.3000113660168781, 0.3041626266579127, 0.3022308026201127, 0.29452758644027976, 0.28172599755050715, 0.26529290333071653, 0.24806719614993497, 0.23449001334586056, 0.22941722606541884 ], [ 0.27957354406650126, 0.2677875884612782, 0.2534701025807058, 0.2353383398460297, 0.212902826715019, 0.18748836569737268, 0.16323418651857693, 0.1468336607825913, 0.14264109906083886, 0.14629797697344904, 0.14919636538797942, 0.14697808840874943, 0.1409905105409606, 0.13414873276045292, 0.12611898470294708, 0.11456233449273838, 0.10020781685647506, 0.08786009801323827, 0.0811418312378714, 0.07936853111243762, 0.0802386133582532, 0.08044428215207602, 0.08067817627624567, 0.08264176404000055, 0.0865005256663144, 0.09779651538869692, 0.12011403678416115, 0.14610438244045376, 0.16741895482157684, 0.18042430523850603, 0.18545001675631037, 0.18464492673664198, 0.18031589574094795, 0.17510169378526136, 0.173325349555906, 0.18027440920491433, 0.19787142708518382, 0.22245257535034807, 0.24825964400483932, 0.27095647344235607, 0.28828322385314953, 0.29941292997576985, 0.30425333215803646, 0.3030253093634185, 0.29614129414334334, 0.284360256752134, 0.26918377226113027, 0.25335377286903316, 0.24097986938119198, 0.23640704089992678 ], [ 0.2771130648127983, 0.26596778391169007, 0.25270035590888545, 0.2359496038460462, 0.21510868731226457, 0.19132311234325552, 0.1683594498771762, 0.15218626076402259, 0.14660543980669194, 0.1478727955412503, 0.14835651089728813, 0.14420787521628575, 0.13719186592382931, 0.1305331753196508, 0.12344201521391769, 0.11267647283750834, 0.09854433970820309, 0.08620581119297296, 0.07989575920511094, 0.07882532565205555, 0.08025071342120192, 0.0808381406382292, 0.08141532023487633, 0.08371784064599275, 0.08793107392537326, 0.09957446211089496, 0.12222516701284623, 0.14866323961464445, 0.17054076338718155, 0.18402948187858625, 0.18908026571098066, 0.1874363217355567, 0.18123514233210614, 0.17335489379896407, 0.16887295703841887, 0.17422832537622676, 0.19190674796940158, 0.21762423366840328, 0.24479257859275677, 0.2686817043473062, 0.2869600734159781, 0.29885827118003816, 0.3043794169851338, 0.3038463304312315, 0.2977686799594104, 0.2869838247831059, 0.27301701087404057, 0.2585060619006674, 0.2472474067630564, 0.24313208071432293 ], [ 0.27442562776824064, 0.26393175258781115, 0.25173868786461295, 0.23638704711567934, 0.21713291063393964, 0.1949312642674817, 0.17319065098431805, 0.15723481172736595, 0.15039053499017094, 0.14942278339608564, 0.14756934080875472, 0.14148326159102095, 0.13337864546204956, 0.12685450444693516, 0.12067211186816827, 0.11068190355009004, 0.09681585657660267, 0.08457914378940562, 0.07873172124252723, 0.07834668012146359, 0.08029406192820678, 0.08124322313457381, 0.08214076377784411, 0.08476931621010307, 0.08939017117214991, 0.1014296501607944, 0.12436958743735202, 0.1511772033896697, 0.1735396083439572, 0.1874417856705136, 0.19248208312665793, 0.1900278354273458, 0.18203200042321893, 0.1715379626767385, 0.16431681048894203, 0.16804188248628005, 0.18586890591391644, 0.21281523871220545, 0.24139092527806794, 0.2664793774471684, 0.285699562734393, 0.2983525429944085, 0.30454132118343186, 0.3046900871815601, 0.29940156452526906, 0.2895838643602749, 0.27677600757109216, 0.2635072059827921, 0.25328026636748435, 0.2495834009739848 ], [ 0.2714973611911695, 0.26166438143191806, 0.2505691503109976, 0.23663235314959524, 0.21895244648483506, 0.19828364259578113, 0.17769297016458394, 0.16193593481131804, 0.15394099308043163, 0.15090299931575346, 0.1468249277486304, 0.1388334466946371, 0.12960386749280287, 0.12316967016512582, 0.11785926181317395, 0.10862337458526469, 0.09506890131765111, 0.08302170375849072, 0.07765720683310269, 0.07790717831916265, 0.08033366127816063, 0.08161895582413968, 0.08280992044184624, 0.08575599571941278, 0.09083266684649784, 0.10331692272398862, 0.12652278210036447, 0.1536371012104637, 0.17641483023829338, 0.19066766315008404, 0.1956682728779536, 0.19243671367462772, 0.1827310717031535, 0.16969189233213194, 0.1597124897635403, 0.16177297516586853, 0.1798165622470751, 0.20807558590116812, 0.2380893540370991, 0.26437066758335775, 0.2845133691321843, 0.2979010037180609, 0.30473961834648633, 0.3055531431347353, 0.3010325451438408, 0.29214907679757496, 0.28044678836392267, 0.26834401827447335, 0.25906999306460776, 0.25575574185337163 ], [ 0.26831506454772747, 0.2591515838236675, 0.24917743720492255, 0.2366696038278473, 0.22054761547869725, 0.20135577967496, 0.18183789246239435, 0.16625427707627666, 0.15720991808673562, 0.15227271219159796, 0.14611410940826564, 0.1362906652457442, 0.12592852522344858, 0.1195437966105487, 0.11505752904530839, 0.10654542998580943, 0.09334578568503105, 0.0815700186816703, 0.07667860463695059, 0.07748536810765655, 0.08033693110155014, 0.08192492703921078, 0.08337879752270978, 0.08663706657833986, 0.09220991764781339, 0.1051860601072877, 0.12865599155333526, 0.15603273552479693, 0.17916703066988798, 0.19371590503091657, 0.19865457391281596, 0.1946840874939436, 0.1833623419591998, 0.1678661903597358, 0.1551296552265087, 0.15549565317741293, 0.1738212106854936, 0.2034621013741048, 0.23492516192289034, 0.26237752658977487, 0.2834133184562557, 0.2975089300022091, 0.3049749866051001, 0.3064324073450252, 0.3026550036017497, 0.2946696594370156, 0.2840178213238481, 0.2730065388623386, 0.26461142346048644, 0.26164690209773367 ], [ 0.2648663684221172, 0.2563804685918674, 0.24755104966245012, 0.23648539238441454, 0.2219021174565792, 0.2041277376997413, 0.18560266346300167, 0.17016148484223148, 0.16015795807916336, 0.1534952727489616, 0.14542795276738674, 0.13388851724029593, 0.12241984318609318, 0.11604832815160605, 0.1123229150723233, 0.10449036631684991, 0.09168170262938721, 0.08025141340683825, 0.07580085002362079, 0.07706533168154713, 0.08027616890245219, 0.08212464453413862, 0.08380784504486237, 0.08737404334585713, 0.09347356312028679, 0.1069863252231754, 0.130738847562868, 0.15835366791591374, 0.18179799610435604, 0.1965971881371357, 0.20145900128446165, 0.19679412068932836, 0.18396018791590962, 0.16611808974629255, 0.15065284032798776, 0.14930237092218274, 0.16796819601042268, 0.19903807199575713, 0.2319377024226257, 0.26052230562670003, 0.28241118341657073, 0.297181522496951, 0.3052481740345042, 0.30732512890868363, 0.3042630985395837, 0.29713724231502864, 0.2874798109182872, 0.27748762597728965, 0.26990216050209737, 0.2672572135052624 ], [ 0.26113991397320074, 0.2533395264459367, 0.2456794733550085, 0.23606894185477004, 0.2230030461940273, 0.20658395205913874, 0.18896983835063044, 0.17363536959036235, 0.16275241225925238, 0.1545378314825347, 0.14475710558364807, 0.1316598037550658, 0.11914847634166295, 0.11275815249147965, 0.10971078348614324, 0.10249641808251012, 0.09010283111299666, 0.07908092650959207, 0.0750271583472609, 0.07663744471447295, 0.08013019961660589, 0.08218850858073161, 0.08406514465898253, 0.08793369528592662, 0.09457915616076051, 0.108670482218552, 0.1327420946248175, 0.1605902003993114, 0.18431075834049823, 0.1993237177219116, 0.20410126168500967, 0.19879318492331685, 0.18456226536792825, 0.1645112964585233, 0.14638160085780721, 0.1433059779117062, 0.16235721262570182, 0.19487242106734612, 0.2291676238561396, 0.2588273166363952, 0.28151846457943397, 0.2969238059766118, 0.30555995962927196, 0.3082288837184555, 0.3058517447617973, 0.29954480929828015, 0.2908254896542147, 0.2817825833229564, 0.27494212245391775, 0.27258909795020003 ], [ 0.2571255564873985, 0.25001883878274195, 0.24355437266462612, 0.2354122320116619, 0.2238409117255682, 0.20871309946827135, 0.1919269166491859, 0.17665924616232556, 0.16496644240652636, 0.15537098308176062, 0.14409111013895223, 0.12963395380024453, 0.11618447841854179, 0.10974750643360126, 0.10727288189995299, 0.10059628449606213, 0.08862586257724082, 0.07806162014912495, 0.07435890830131323, 0.07619828757119587, 0.07988512022394685, 0.08209584030769213, 0.08412882897149478, 0.08829066547544419, 0.09548934995982282, 0.11019815008030229, 0.13464012389576288, 0.16273443130086349, 0.18670974578858238, 0.20190894802532014, 0.20660222966679836, 0.20070905667600797, 0.18520828000317918, 0.16311419053276238, 0.1424295690269282, 0.13764088287775203, 0.15710186745701177, 0.19103831600347485, 0.22665591468223384, 0.2573143422916776, 0.2807461596388653, 0.2967405265637415, 0.30591111114103586, 0.30914155468179777, 0.3074165818378189, 0.30188660825294955, 0.2940494125236708, 0.28588882309637764, 0.27973315535773, 0.2776466924359551 ], [ 0.2528145984595471, 0.24641031449615136, 0.2411698059539934, 0.23451013695432416, 0.22440967077903815, 0.21050799064413775, 0.19446605592581434, 0.17922142559405083, 0.16677841526549092, 0.15596840804965037, 0.14341776869236106, 0.12783423117120638, 0.11359201953203821, 0.10708462600634104, 0.10505409756182671, 0.09881607650734499, 0.08725893740128456, 0.07718733211165558, 0.07379557976298348, 0.07574970930982919, 0.07953416602951996, 0.0818359619680413, 0.08398871471935596, 0.08842961339122417, 0.09617653644474387, 0.11153848278411756, 0.13641315789190675, 0.1647812771405327, 0.1890009793594204, 0.204367359583169, 0.20898347363550807, 0.2025701357145576, 0.18593865594513156, 0.16199743245393744, 0.13892184709678032, 0.1324624696267361, 0.1523277596107194, 0.18761111760599058, 0.22444277043767155, 0.25600410878546526, 0.2801045278086852, 0.2966360489618194, 0.3063023411756983, 0.31006130667950976, 0.30895393424301604, 0.30415805414106944, 0.29714775819074957, 0.2898055631829521, 0.28427869965760005, 0.28243553054560055 ], [ 0.2482000587336416, 0.24250796123917306, 0.23852246624107842, 0.23336057472545818, 0.22470676511692272, 0.21196548690852363, 0.1965838586165283, 0.18131484840927495, 0.16817139108970955, 0.1563065714289505, 0.14272265647134602, 0.12627502551997336, 0.11142310405006146, 0.10482540382100267, 0.1030892130914054, 0.09717471582362797, 0.08600358080790478, 0.07644585450088891, 0.07333454761961661, 0.07529709319991346, 0.07907683157950757, 0.08140840745173006, 0.08364717383758129, 0.0883468084956657, 0.09662495146131901, 0.11267224076045908, 0.1380490082920343, 0.1667293721217877, 0.19119226939140313, 0.20671427313666796, 0.2112668233717727, 0.20440468994006933, 0.1867931340396435, 0.16123099581212288, 0.13599016143009351, 0.127943467434841, 0.1481684622311445, 0.18466563827350055, 0.22256631820341646, 0.25491574070596773, 0.27960285716424216, 0.29661425679090264, 0.30673426299910234, 0.31098655755723803, 0.31046076513698356, 0.3063556282992473, 0.3001181396298339, 0.2935335563974097, 0.28858350309921105, 0.2869622709478325 ], [ 0.24327698562082903, 0.23830819859389818, 0.2356119514672231, 0.23196466950306463, 0.22473316687316672, 0.21308643954260015, 0.19828122675042045, 0.18293684749777744, 0.16913276718541237, 0.1563645275872095, 0.1419888738564666, 0.12495962650639421, 0.10971092061391807, 0.10300677266660531, 0.10140005324257786, 0.09568376982009955, 0.08485708544273468, 0.07582189846362346, 0.07297059181313813, 0.0748469509556134, 0.07851747726591993, 0.08082237569254014, 0.08311927005036662, 0.08805116276943935, 0.09683232059476375, 0.11359334196769627, 0.13954437946967127, 0.1685817781867825, 0.19329337501080887, 0.20896568202565385, 0.2134739733096044, 0.20624013633608512, 0.18780935012778255, 0.16088074904721233, 0.1337654023131076, 0.12426580147801428, 0.14475890082761048, 0.18227277487376617, 0.22106125794236683, 0.2540662214754648, 0.2792492434015988, 0.2966784591781808, 0.30720734750046264, 0.31191594641913145, 0.31193462569344493, 0.3084767765685399, 0.30295942597126113, 0.2970748493535511, 0.29265337325740814, 0.29123446542671044 ], [ 0.23804282366301263, 0.23381022181749048, 0.23244106834049602, 0.23032692595135526, 0.22449342914176362, 0.21387564991649738, 0.1995632795448638, 0.18408903234946217, 0.16965408190273795, 0.15612386943417259, 0.14119711358461257, 0.12387890763760963, 0.10846397354826098, 0.10164109323891547, 0.09999348035765385, 0.09434766961411649, 0.08381484945292546, 0.0752995610507656, 0.07269521871639231, 0.07440411239850052, 0.07786373772731275, 0.08009555258319766, 0.08243217295451201, 0.08756472169195367, 0.09681113144380542, 0.11430996658654194, 0.1409057122992375, 0.17034645800362616, 0.1953160939612464, 0.21113808763806718, 0.2156261180676861, 0.20810237171962379, 0.1890214599100102, 0.16100483019704256, 0.1323677121573067, 0.12160686661676232, 0.1422260405549525, 0.18049571580401003, 0.21995750230430025, 0.2534698857528966, 0.2790503886714374, 0.2968313066723741, 0.30772188271073747, 0.3128483004358877, 0.31337360169418743, 0.310519808419468, 0.3056715765841412, 0.30043256846181976, 0.2964949641048517, 0.29526036032981406 ], [ 0.2324978459200355, 0.2290164262615283, 0.2290161733111031, 0.2284554141090664, 0.22399573904069608, 0.21434184734031073, 0.20043932852589819, 0.1847772882222209, 0.1697309828442013, 0.15556885063567324, 0.14032609252138636, 0.12301126424159234, 0.10766290853869913, 0.10071322829565475, 0.09886065421879449, 0.09316424115286609, 0.08287231039793429, 0.07486415236307574, 0.07249621315185269, 0.07396914091120046, 0.07712509698288295, 0.07925244222759709, 0.08162384525312774, 0.08692264804278838, 0.09658959347729154, 0.11484524832988184, 0.14214956478402133, 0.17203647953674234, 0.1972742586679929, 0.213248325745001, 0.21774361876028542, 0.2100151696098643, 0.19045888792249624, 0.16165016999410714, 0.13189515833468576, 0.1201206019169026, 0.14067759135722402, 0.17938606742284754, 0.21927891297902283, 0.2531379712024681, 0.27901142881235613, 0.2970747193212996, 0.3082779371703173, 0.3137826012938123, 0.3147762588893377, 0.31248379875636345, 0.30825548788218526, 0.30361073059621163, 0.3001155919225711, 0.299048726453309 ], [ 0.22664566642741527, 0.2239329041815969, 0.22534755342615545, 0.22636196162840877, 0.22325196918835383, 0.21449768020908455, 0.20092290392427023, 0.1850118846983383, 0.16936336232802604, 0.15468670073005478, 0.13935336468191817, 0.1223239509249969, 0.10726201734901972, 0.1001816094083806, 0.09797780411187712, 0.0921254850704814, 0.08202625105504983, 0.07450336212663004, 0.0723580075240823, 0.07353736935607083, 0.07631197502448621, 0.0783223891659073, 0.0807410009305197, 0.08617274304986325, 0.09621228945050177, 0.1152375261024412, 0.1433025169466648, 0.17366993441050732, 0.19918362267717746, 0.21531337523661379, 0.21984570035186565, 0.2119996603837918, 0.19214528026852426, 0.16284958068736755, 0.13241299555355418, 0.11991616158073226, 0.14019047054774555, 0.17898036011851295, 0.21904223766874972, 0.25307825566619724, 0.2791357964246715, 0.29740982937995736, 0.3088753282910409, 0.31471795230267025, 0.31614158841586554, 0.3143684936892323, 0.3107128529386789, 0.3066140760877649, 0.303523076591086, 0.3026087132697586 ], [ 0.22049385089222806, 0.21857002737244793, 0.2214498483748664, 0.22406234813057294, 0.22227772195666395, 0.21435971432887482, 0.20103182453488938, 0.18480768793043775, 0.16855566287420823, 0.1534681459535619, 0.13825649853610725, 0.12177570184909998, 0.10719512166030247, 0.09998320255999023, 0.09730850718739054, 0.09121857121198977, 0.08127538017835359, 0.07420783610918824, 0.07226317122285826, 0.07310053848440234, 0.07543556683710542, 0.07733756209719744, 0.07983637229172441, 0.0853745595602183, 0.09574043286879282, 0.11554005977809942, 0.14440057574373158, 0.17526956445946087, 0.20106163031254579, 0.21735014471401887, 0.22195018169605965, 0.21407391075395243, 0.19409773197690297, 0.16461980460577222, 0.1339460684918955, 0.12103940244854855, 0.14080156678539338, 0.17929742851366606, 0.21925634372954864, 0.2532948018846971, 0.2794251258157692, 0.29783694061688887, 0.3095135966680031, 0.31565354705339194, 0.3174689533613536, 0.31617422122502215, 0.3130460337091167, 0.3094479218561891, 0.30672560491123224, 0.30594972410777616 ], [ 0.2140546478146918, 0.21294313066714193, 0.21734251262769277, 0.2215764937888605, 0.22109235899450852, 0.21394843030937447, 0.20078830092085226, 0.18418446951536518, 0.167317355288652, 0.15190814338358027, 0.13701457338991951, 0.1213202891888909, 0.1073835953089553, 0.10004095760756267, 0.09680721380780175, 0.09042706432330179, 0.08062021143889953, 0.07397131049684065, 0.07219476651712387, 0.07264999890778369, 0.07450851399812003, 0.07633130865775212, 0.07896543661299629, 0.08459816009131882, 0.09525152600847306, 0.11582004335707961, 0.14548804989513978, 0.17686210333788802, 0.2029270711823308, 0.21937523638102743, 0.22407324085157743, 0.216252615491325, 0.19632634020961512, 0.16696079499882127, 0.13647647599085924, 0.12346446776060466, 0.14250329992515723, 0.18033707821211478, 0.21992182159745244, 0.2537878256875732, 0.2798792039923114, 0.298355505584187, 0.31019198707529094, 0.3165886403773468, 0.3187580373649042, 0.3179018075480712, 0.3152579454583818, 0.3121180326619201, 0.309731613107991, 0.3090813094526671 ], [ 0.20734586696884896, 0.20707331239356713, 0.21305031271761055, 0.21892863087367406, 0.21971900636443073, 0.2132882096056135, 0.20021905870029214, 0.18316730205844828, 0.16566359134292633, 0.15000683281902885, 0.1356099312978216, 0.12091056142061979, 0.10774465361845038, 0.10027204734338808, 0.09642359424006254, 0.08973245546359293, 0.08006238443245699, 0.07379055458791509, 0.07213904563507734, 0.07218019269092255, 0.07354632392919573, 0.07533740227673036, 0.07818327944901023, 0.08392252712110863, 0.09483807623067046, 0.11615669491020665, 0.14661587262407036, 0.17847735425174321, 0.20479962980016916, 0.221404690492032, 0.22622921874863222, 0.21854690950753797, 0.1988341064978785, 0.16985630561699525, 0.1399471326749159, 0.1271010538852408, 0.14524526819335692, 0.18208025610128098, 0.22103099580922636, 0.25455369559641666, 0.2804959697049231, 0.29896412154457497, 0.3109094366366658, 0.31752252221204014, 0.32000879596441795, 0.3195524993254673, 0.3173519528529322, 0.3146305086342976, 0.3125496860872017, 0.31201307598587724 ], [ 0.20039193720113188, 0.20098836757821256, 0.20860384877467544, 0.2161474428168351, 0.21818452326620943, 0.21240729623749782, 0.19935546459336292, 0.18178702663416366, 0.16361603135175748, 0.14777070926608385, 0.13403011385029656, 0.12050251948889822, 0.10819882408849243, 0.10059562458633226, 0.09610724246895459, 0.08911611201287291, 0.0796037540990933, 0.07366552319103165, 0.07208808460200625, 0.07169184206396718, 0.07256932173127734, 0.07439059953068725, 0.07754358183630482, 0.0834335181091672, 0.09460491368155798, 0.11663819291920119, 0.14783937943161787, 0.1801470414219811, 0.20669934873453835, 0.22345371701420677, 0.22843046427285332, 0.22096430243597323, 0.20161717929335043, 0.17327564976477017, 0.14427002391109944, 0.13181211721931627, 0.14894139738702444, 0.18449068204384717, 0.22256833949504853, 0.25558506292432415, 0.2812715602363167, 0.2996605450463977, 0.311664570408293, 0.31845449482858446, 0.32122141123114445, 0.32112789228080874, 0.3193317770908913, 0.31699168739944367, 0.3151884713599548, 0.3147546093428839 ], [ 0.1932251799040701, 0.1947238665865892, 0.2040400806098299, 0.21326615037829352, 0.2165194198578966, 0.21133771852052827, 0.19823363289348925, 0.18008076986590799, 0.16120384290249073, 0.14521401783107019, 0.13226991494841187, 0.12005909270561067, 0.10867605298270791, 0.1009393893928735, 0.09581234339509051, 0.08856174022217946, 0.07924616479561053, 0.07360023559207098, 0.07204216305425411, 0.07119456475418887, 0.0716048309777983, 0.07352851050903998, 0.07710013786072605, 0.08322111098211989, 0.09466463497694248, 0.11735729917372026, 0.14921560192128308, 0.18190349126930558, 0.20864603092752895, 0.22553642392567128, 0.2306872233595103, 0.22350873141429872, 0.2046653978607455, 0.1771763173075345, 0.1493367767549763, 0.13743455793064477, 0.15348047065715484, 0.18751765489045616, 0.22451124721307783, 0.2568711127461323, 0.28220040335029306, 0.30044172444548406, 0.31245570435730624, 0.31938385372871575, 0.32239625007944683, 0.322629866123631, 0.32120141339160346, 0.3192080592893312, 0.31765660583594485, 0.3173154088630441 ], [ 0.18588733630214202, 0.1883243835503958, 0.19940282604108117, 0.21032251887761144, 0.21475770734431912, 0.21011515250090834, 0.1968944841369625, 0.1780924774364656, 0.15846485856348771, 0.14236036987893388, 0.13033348584848126, 0.11955340594776093, 0.10912030050687403, 0.10124473183852117, 0.09550203366922759, 0.08805835330186021, 0.07899347470916283, 0.07360472619458278, 0.07201182271747446, 0.07070874060773652, 0.07068922255830763, 0.07279428614071497, 0.07690878367592409, 0.08337563317955644, 0.09513087445642296, 0.11840569464642393, 0.1508002144599293, 0.18377821765121347, 0.21065861256735893, 0.227665553494727, 0.23300757380802897, 0.2261807216318847, 0.20796307487982146, 0.18150705253874408, 0.15502913066392782, 0.14379770982676648, 0.1587374396961155, 0.1910995861878895, 0.2268310881861818, 0.2583979185534863, 0.28327534976705, 0.3013038490317108, 0.3132808554783102, 0.32030987237762004, 0.32353382650262424, 0.3240605257961663, 0.32296505814139814, 0.3212861942592744, 0.31996265392605194, 0.31970483284547585 ], [ 0.1784313818580414, 0.18184486140087874, 0.19474318190109807, 0.20735875458061184, 0.2129366616195319, 0.20877870652027303, 0.1953837212550383, 0.17587341565836984, 0.155446864431617, 0.13924457031356907, 0.12823642972679183, 0.11897143798168998, 0.10949273250879832, 0.10147052576330678, 0.09515230510700455, 0.08760357355923351, 0.07885561606844976, 0.0736978486622815, 0.07201955564915677, 0.07026652533361076, 0.0698694431050524, 0.07223935276127033, 0.0770285448856438, 0.08398284887786012, 0.09610955172373574, 0.11986737126819441, 0.1526443554381283, 0.18580050070840204, 0.21275454154880485, 0.22985223881672837, 0.23539740632876208, 0.22897763905265886, 0.21148994129089807, 0.1862110044813953, 0.16122766403455058, 0.15073688343708258, 0.16458344344139797, 0.1951677739121473, 0.22949444240723918, 0.2601488777922023, 0.28448783985239173, 0.30224241286299397, 0.31413775856992165, 0.3212317908070647, 0.3246347678645295, 0.3254221488994505, 0.32462704498666795, 0.32323267927397875, 0.32211505559473325, 0.3219320530235264 ], [ 0.170923642193775, 0.17535206757789293, 0.19011979580150187, 0.20442125296413297, 0.2110964807992929, 0.20737060490537054, 0.19375168200482562, 0.17348257266425704, 0.15220896327037126, 0.13591462392798545, 0.12600781094877134, 0.11831403432969628, 0.10977372526842206, 0.1015957869317817, 0.09475537250192131, 0.0872069291694971, 0.07885277175713001, 0.07391015280265988, 0.07210101690928394, 0.06991200344723825, 0.06920367161247226, 0.07192538607888925, 0.0775214605368158, 0.08511821934882631, 0.0976889223277003, 0.1218118352478371, 0.1547916160895781, 0.1879960573080869, 0.2149491975615649, 0.23210579274566748, 0.23786045093977107, 0.23189401606400245, 0.21522217378480582, 0.1912286401948918, 0.16781809898842878, 0.1581015603174821, 0.17089343559181167, 0.19964999338890138, 0.2324644173572907, 0.2621052022686989, 0.2858280970126525, 0.30325229096802514, 0.3150238890035941, 0.32214880800436535, 0.32569978527140564, 0.3267171390829782, 0.32619178917339203, 0.32505406504255097, 0.32412208317164626, 0.3240060171354964 ], [ 0.16344617878424464, 0.16892603809085252, 0.1855988898675428, 0.2015601579501944, 0.20927981758247294, 0.2059357488137249, 0.19205302105641156, 0.17098686596318988, 0.14882290966729786, 0.13243384348257586, 0.12369196981624019, 0.11759825319481136, 0.10996389288403194, 0.1016214066105059, 0.09432243606794358, 0.0868926898621364, 0.07901840295459554, 0.07428587336053406, 0.0723055541500591, 0.06970067744567525, 0.06876095545618523, 0.07192476655263687, 0.07845076967781543, 0.08684118680773063, 0.09992999754486699, 0.12428826584836125, 0.15727551897069117, 0.19038589941350476, 0.2172553878590563, 0.23443354015988405, 0.2403983463726235, 0.23492192889448701, 0.21913343159930668, 0.1965002229002921, 0.1746952423923361, 0.16575922393982775, 0.17755121320641132, 0.2044736103585307, 0.23570195159251217, 0.2642464366931088, 0.28728533962702146, 0.3043278252693831, 0.3159364906636638, 0.3230600778989797, 0.32672964795846654, 0.32794798512265544, 0.3276637394498172, 0.32675682109116866, 0.3259918058750634, 0.32593541860780045 ], [ 0.15609931801017582, 0.16266131747496881, 0.1812539090408513, 0.19882869107658177, 0.2075311702662747, 0.20452113453475018, 0.19034617241679502, 0.16846103533296816, 0.1453742433954403, 0.1288828989281147, 0.12134996824876312, 0.11685799725261929, 0.1100842678513066, 0.101571073407564, 0.0938857097337232, 0.08670173273508276, 0.07940040369018073, 0.07488317993715851, 0.07269573033449672, 0.06969873941593918, 0.06862003078029402, 0.0723188511943021, 0.07987745796222406, 0.08919065808158967, 0.102859989372074, 0.12732188879109385, 0.16011777297739027, 0.19298546320807275, 0.21968294775887748, 0.23684070235848295, 0.24301074875277118, 0.2380514054877841, 0.2231958421846929, 0.2019677657402627, 0.18176500576860116, 0.17359622245988413, 0.184452193072412, 0.20956806334170502, 0.2391670292975308, 0.2665509801428434, 0.28884800322265924, 0.3054629174083464, 0.3168726081249336, 0.32396470867173355, 0.32772516155069975, 0.3291172253677623, 0.3290473368731257, 0.32834729825942216, 0.3277320611217539, 0.32772867248389753 ], [ 0.14900402340168095, 0.15666766999051485, 0.17716464244837105, 0.19628221519449468, 0.2058961221532607, 0.2031751148719633, 0.18869254516058392, 0.16598707395666729, 0.14196294645724775, 0.12536150906549415, 0.1190603910755666, 0.11614383189722112, 0.11017565460895508, 0.10149134271562142, 0.09349946786836973, 0.08669192369577278, 0.08005986010808008, 0.07577205233747496, 0.07334543916583036, 0.0699823052998787, 0.06886740118771967, 0.0731936190976314, 0.08185556299399839, 0.09218271857098775, 0.1064721994806538, 0.13091338440067493, 0.16332748908244424, 0.19580406713553947, 0.22223846729437727, 0.23933033950492197, 0.24569547460650404, 0.2412708438962626, 0.22738089157948993, 0.20757645759712903, 0.1889450503608944, 0.18151689949873676, 0.1915045008271593, 0.2148666834254768, 0.2428197523353236, 0.2689965884794721, 0.29050396495497505, 0.306651125619266, 0.3178291220638601, 0.32486176504282893, 0.3286871499964686, 0.3302274172065366, 0.3303469798893586, 0.329831697793693, 0.3293504318028392, 0.3293938968322467 ], [ 0.14230352387451314, 0.15106976556153104, 0.17341565785615107, 0.19397701104682893, 0.20442042861280024, 0.20194649833407954, 0.18715541540428918, 0.16365303529778508, 0.13870322314201175, 0.1219892721917033, 0.1169190982325558, 0.1155218320627262, 0.11029706631317919, 0.1014506391078803, 0.09323970699736582, 0.08693654332940991, 0.08106716041511232, 0.07702947001037504, 0.07433626228550717, 0.07063587224246176, 0.06959391369914956, 0.07463271941199792, 0.0844270315965418, 0.0958109963044073, 0.11073159223305729, 0.13504123204518947, 0.16690139341191085, 0.19884472449525648, 0.2249251559505104, 0.24190335369890403, 0.248448672301219, 0.24456742379561774, 0.23166019203428356, 0.2132756125738541, 0.19616454860247484, 0.1894418791047385, 0.1986289343511505, 0.2203079028738265, 0.24662123904562444, 0.27156084010900167, 0.2922407632086296, 0.30788576289333297, 0.31880278687095204, 0.32575027314418165, 0.3296164409249295, 0.3312811111797587, 0.3315669950870387, 0.3312160462894068, 0.33085422879572024, 0.33093889895243706 ], [ 0.13616319615806446, 0.14600516298965202, 0.1700939124410251, 0.1919687688590881, 0.2031489643678154, 0.20088349409756068, 0.18579849769525, 0.16155106234013633, 0.1357218748004899, 0.11890487340099332, 0.11503739646480957, 0.11507126325118112, 0.11052307876754719, 0.10153682345704117, 0.09320187587775394, 0.08752042167567552, 0.08249567575604595, 0.0787321194876355, 0.07575198166724235, 0.07174886702282833, 0.07088898489160557, 0.0767086816737192, 0.08761717159089454, 0.10004933850215743, 0.11558256118775143, 0.139666073678235, 0.17082492298984947, 0.20210430112350639, 0.22774284714037668, 0.24455855179006528, 0.2512650154046115, 0.24792749698519298, 0.23600611437097116, 0.21901922226255863, 0.2033634388154758, 0.19730605768155376, 0.20575825683582738, 0.22583594896511178, 0.25053433896056976, 0.27422155231258494, 0.2940458061757393, 0.3091599938706396, 0.31979026944266914, 0.3266292275488838, 0.33051385414312356, 0.3322808293606055, 0.33271161305768776, 0.332506175804595, 0.3322504780609666, 0.33237116577307063 ], [ 0.13076722092745824, 0.14161980625637244, 0.16728547621255357, 0.19031083120396736, 0.20212456013274804, 0.20003252774393707, 0.18468421101525592, 0.15977453690822957, 0.13315467071847664, 0.11626266446440425, 0.11353804656886156, 0.11488092470619661, 0.110939928047902, 0.10185288523485016, 0.09349609030645199, 0.08853374143494022, 0.08441392803262032, 0.08094757192403541, 0.07767165101969174, 0.07340975640169965, 0.07283271788783177, 0.0794738186593581, 0.09143172932688831, 0.10485594440194478, 0.12095679199094857, 0.1447359192463799, 0.1750739758815141, 0.20557397684973513, 0.23068813418517928, 0.24729276376991818, 0.254137911172636, 0.2513369463007552, 0.24039228453643924, 0.2247661970340384, 0.21049143338857193, 0.20505660926288677, 0.21283614579757495, 0.2314011343297268, 0.25452416805230305, 0.2769571401501302, 0.29590656448153063, 0.3104669281826517, 0.3207881881765867, 0.32749760001474354, 0.33138019296484106, 0.33322904761558203, 0.3337849488299659, 0.3337077085308533, 0.3335459117420171, 0.3336978578999292 ], [ 0.12631022706520825, 0.13806037019377143, 0.16507143996770757, 0.1890522654876245, 0.2013867760301257, 0.1994369723039495, 0.18387169757545443, 0.1584143552335474, 0.1311402060325822, 0.11422555778369455, 0.11254866003840878, 0.11504410267347887, 0.1116402769065073, 0.10251041872643402, 0.09423944940092972, 0.09006395612347073, 0.08687785639557874, 0.08372599654512966, 0.08016220131295765, 0.07569816816239687, 0.07548737691729962, 0.08295295990776357, 0.09585630701586279, 0.11017798829859664, 0.12678004636772097, 0.15019122504398685, 0.1796170354468019, 0.20923994571672855, 0.23375462112700085, 0.2501010097746698, 0.2570597174348749, 0.2547815058654161, 0.24479395240199128, 0.2304803770924887, 0.2175069458296676, 0.21265115215872757, 0.219816011111633, 0.23695985043195586, 0.258558479676414, 0.2797469142511818, 0.2978107442060198, 0.3117997083095169, 0.3217931512683166, 0.32835434949844855, 0.3322162380498975, 0.33412818136427386, 0.3347909863778815, 0.3348260454653464, 0.33474696274629595, 0.334925806830057 ], [ 0.12298249223544691, 0.13546333130380478, 0.16352327197807728, 0.18823588846365036, 0.20097067671672708, 0.1991358586875675, 0.18341470199259216, 0.1575545045655314, 0.12981111455723124, 0.1129535771680305, 0.11219249355012939, 0.1156523212203452, 0.11271680101713848, 0.10362089863038129, 0.0955456179454784, 0.0921868713334706, 0.08992496644336075, 0.08709533551305494, 0.08327192806415316, 0.0786763546778862, 0.07889028528987378, 0.08714110368758499, 0.10085824086569418, 0.11595597456094854, 0.13297732677609048, 0.15596924858965483, 0.1844173978523996, 0.21308427714126307, 0.23693326553089822, 0.25297670661045935, 0.2600219615022311, 0.2582470387731622, 0.2491882462632028, 0.23613038230641978, 0.22437603373613563, 0.22005613290239395, 0.22665981004929964, 0.24247435605077092, 0.26260789245623195, 0.2825713174076751, 0.29974643790069055, 0.31315159039300233, 0.3228017935062964, 0.32919843301008167, 0.3330227434281496, 0.33498057446390367, 0.335733566737039, 0.3358663585797061, 0.33585976234016784, 0.33606151489789876 ], [ 0.12094886955413618, 0.13394169805313733, 0.16269810013894223, 0.18789640179288047, 0.2009056865027756, 0.19916264567235614, 0.18335946556315794, 0.15726732169077204, 0.12928324625194895, 0.11258850228957121, 0.1125774559213634, 0.11678841204291861, 0.1142550762751128, 0.10528541270380914, 0.09751270154577664, 0.09495843075375175, 0.09357145537290426, 0.09106084047594945, 0.0870261353899896, 0.08238177650644349, 0.08304987664800421, 0.09200701640990021, 0.10639036006644453, 0.12212739195424356, 0.13947638360434922, 0.1620074258686662, 0.18943528484934272, 0.21708585850071235, 0.2402127869327714, 0.2559119033913865, 0.26301555532008114, 0.2617197709732383, 0.2535543292360628, 0.24168935617519527, 0.23107140930216802, 0.22724543454492835, 0.23333692919862145, 0.24791243430657775, 0.2666459988452418, 0.28541210266419903, 0.3017022523503716, 0.31451601683317987, 0.32381081087447805, 0.3300288169023777, 0.3338004343862194, 0.33578849085728085, 0.3366163792959118, 0.33683358602917735, 0.33689014033812914, 0.3371111575645872 ], [ 0.12032455665110761, 0.13357169572570415, 0.1626345598276972, 0.18805881579293213, 0.20121460841628616, 0.1995441383665955, 0.18374282328245245, 0.1576090023372048, 0.12964441862240736, 0.11323673486359269, 0.11378510057760569, 0.11851975794638718, 0.11632658987780281, 0.10758431530156642, 0.10021134240451049, 0.09840884264607454, 0.09781232927847884, 0.09560742593697794, 0.0914256782251799, 0.08682235250661242, 0.08794559385869434, 0.09749957041555116, 0.11239558877649858, 0.12862952123748098, 0.14620981786785395, 0.16824575932313954, 0.1946296959407847, 0.221221347185371, 0.24358011376627958, 0.2588975353804395, 0.26603100186463013, 0.265186482229988, 0.25787347447631653, 0.2471346462350128, 0.2375715406602676, 0.23419919317890864, 0.2398231662151598, 0.25324697392875567, 0.2706493780158423, 0.2882524575663551, 0.30366741283414983, 0.3158866798753281, 0.32481699239664324, 0.33084448822295115, 0.3345500069052861, 0.33655410863885743, 0.3374429558614441, 0.33773242998870734, 0.3378436275096008, 0.33808058770170296 ] ] }, { "hoverinfo": "text", "legendgroup": "In-sample", "marker": { "color": "black", "opacity": 0.5, "symbol": 1 }, "mode": "markers", "name": "In-sample", "text": [ "Arm 0_0
accuracy: 0.841833 (SEM: None)
lr: 2.6e-05
momentum: 0.58", "Arm 10_0
accuracy: 0.560333 (SEM: None)
lr: 1e-06
momentum: 1", "Arm 11_0
accuracy: 0.759167 (SEM: None)
lr: 3.73218e-05
momentum: 1", "Arm 12_0
accuracy: 0.945667 (SEM: None)
lr: 0.000261045
momentum: 0.913551", "Arm 13_0
accuracy: 0.942167 (SEM: None)
lr: 0.000170536
momentum: 0.799543", "Arm 14_0
accuracy: 0.854 (SEM: None)
lr: 0.000159238
momentum: 1", "Arm 15_0
accuracy: 0.947667 (SEM: None)
lr: 0.00031243
momentum: 0.665533", "Arm 16_0
accuracy: 0.906833 (SEM: None)
lr: 0.000823622
momentum: 0", "Arm 17_0
accuracy: 0.928333 (SEM: None)
lr: 0.000694305
momentum: 0.350825", "Arm 18_0
accuracy: 0.1095 (SEM: None)
lr: 0.000582225
momentum: 0.803374", "Arm 19_0
accuracy: 0.101 (SEM: None)
lr: 0.4
momentum: 0", "Arm 1_0
accuracy: 0.100333 (SEM: None)
lr: 0.00995509
momentum: 0.633423", "Arm 20_0
accuracy: 0.914 (SEM: None)
lr: 7.24369e-05
momentum: 0.703681", "Arm 21_0
accuracy: 0.809333 (SEM: None)
lr: 0.000450821
momentum: 0.134976", "Arm 22_0
accuracy: 0.906833 (SEM: None)
lr: 8.45341e-05
momentum: 0.453801", "Arm 23_0
accuracy: 0.1095 (SEM: None)
lr: 0.00282781
momentum: 0", "Arm 24_0
accuracy: 0.830833 (SEM: None)
lr: 6.46166e-05
momentum: 0", "Arm 2_0
accuracy: 0.318667 (SEM: None)
lr: 5.18336e-06
momentum: 0.0228515", "Arm 3_0
accuracy: 0.4585 (SEM: None)
lr: 6.63497e-06
momentum: 0.176948", "Arm 4_0
accuracy: 0.926 (SEM: None)
lr: 8.15482e-05
momentum: 0.90883", "Arm 5_0
accuracy: 0.929 (SEM: None)
lr: 0.000302077
momentum: 0.341904", "Arm 6_0
accuracy: 0.92 (SEM: None)
lr: 0.000137311
momentum: 0.590917", "Arm 7_0
accuracy: 0.860167 (SEM: None)
lr: 1.03262e-05
momentum: 1", "Arm 8_0
accuracy: 0.888833 (SEM: None)
lr: 0.000246295
momentum: 0", "Arm 9_0
accuracy: 0.901667 (SEM: None)
lr: 0.000149439
momentum: 0.286357" ], "type": "scatter", "x": [ 2.6e-05, 1e-06, 3.732184409504214e-05, 0.0002610449283938493, 0.00017053570203455322, 0.0001592376912143492, 0.00031243004356781596, 0.0008236218541279722, 0.0006943048179834304, 0.0005822252247415435, 0.4, 0.00995509203921593, 7.243686845468841e-05, 0.00045082139108132483, 8.453410095178891e-05, 0.0028278053483573595, 6.461659285089037e-05, 5.183362665905935e-06, 6.6349677287433636e-06, 8.154817047148868e-05, 0.000302076728574865, 0.00013731096665846605, 1.0326238819187764e-05, 0.00024629479479540775, 0.0001494385653033887 ], "xaxis": "x", "y": [ 0.58, 1.0, 1.0, 0.9135505379162181, 0.7995431686056417, 1.0, 0.6655325132796462, 0.0, 0.350824525958222, 0.8033736548767413, 0.0, 0.6334228515625, 0.703680958091337, 0.134976004192787, 0.4538006087975372, 0.0, 0.0, 0.022851496934890747, 0.1769482558593154, 0.9088304992765188, 0.34190391190350056, 0.5909173150416861, 1.0, 0.0, 0.2863569208604894 ], "yaxis": "y" }, { "hoverinfo": "text", "legendgroup": "In-sample", "marker": { "color": "black", "opacity": 0.5, "symbol": 1 }, "mode": "markers", "name": "In-sample", "showlegend": false, "text": [ "Arm 0_0
accuracy: 0.841833 (SEM: None)
lr: 2.6e-05
momentum: 0.58", "Arm 10_0
accuracy: 0.560333 (SEM: None)
lr: 1e-06
momentum: 1", "Arm 11_0
accuracy: 0.759167 (SEM: None)
lr: 3.73218e-05
momentum: 1", "Arm 12_0
accuracy: 0.945667 (SEM: None)
lr: 0.000261045
momentum: 0.913551", "Arm 13_0
accuracy: 0.942167 (SEM: None)
lr: 0.000170536
momentum: 0.799543", "Arm 14_0
accuracy: 0.854 (SEM: None)
lr: 0.000159238
momentum: 1", "Arm 15_0
accuracy: 0.947667 (SEM: None)
lr: 0.00031243
momentum: 0.665533", "Arm 16_0
accuracy: 0.906833 (SEM: None)
lr: 0.000823622
momentum: 0", "Arm 17_0
accuracy: 0.928333 (SEM: None)
lr: 0.000694305
momentum: 0.350825", "Arm 18_0
accuracy: 0.1095 (SEM: None)
lr: 0.000582225
momentum: 0.803374", "Arm 19_0
accuracy: 0.101 (SEM: None)
lr: 0.4
momentum: 0", "Arm 1_0
accuracy: 0.100333 (SEM: None)
lr: 0.00995509
momentum: 0.633423", "Arm 20_0
accuracy: 0.914 (SEM: None)
lr: 7.24369e-05
momentum: 0.703681", "Arm 21_0
accuracy: 0.809333 (SEM: None)
lr: 0.000450821
momentum: 0.134976", "Arm 22_0
accuracy: 0.906833 (SEM: None)
lr: 8.45341e-05
momentum: 0.453801", "Arm 23_0
accuracy: 0.1095 (SEM: None)
lr: 0.00282781
momentum: 0", "Arm 24_0
accuracy: 0.830833 (SEM: None)
lr: 6.46166e-05
momentum: 0", "Arm 2_0
accuracy: 0.318667 (SEM: None)
lr: 5.18336e-06
momentum: 0.0228515", "Arm 3_0
accuracy: 0.4585 (SEM: None)
lr: 6.63497e-06
momentum: 0.176948", "Arm 4_0
accuracy: 0.926 (SEM: None)
lr: 8.15482e-05
momentum: 0.90883", "Arm 5_0
accuracy: 0.929 (SEM: None)
lr: 0.000302077
momentum: 0.341904", "Arm 6_0
accuracy: 0.92 (SEM: None)
lr: 0.000137311
momentum: 0.590917", "Arm 7_0
accuracy: 0.860167 (SEM: None)
lr: 1.03262e-05
momentum: 1", "Arm 8_0
accuracy: 0.888833 (SEM: None)
lr: 0.000246295
momentum: 0", "Arm 9_0
accuracy: 0.901667 (SEM: None)
lr: 0.000149439
momentum: 0.286357" ], "type": "scatter", "x": [ 2.6e-05, 1e-06, 3.732184409504214e-05, 0.0002610449283938493, 0.00017053570203455322, 0.0001592376912143492, 0.00031243004356781596, 0.0008236218541279722, 0.0006943048179834304, 0.0005822252247415435, 0.4, 0.00995509203921593, 7.243686845468841e-05, 0.00045082139108132483, 8.453410095178891e-05, 0.0028278053483573595, 6.461659285089037e-05, 5.183362665905935e-06, 6.6349677287433636e-06, 8.154817047148868e-05, 0.000302076728574865, 0.00013731096665846605, 1.0326238819187764e-05, 0.00024629479479540775, 0.0001494385653033887 ], "xaxis": "x2", "y": [ 0.58, 1.0, 1.0, 0.9135505379162181, 0.7995431686056417, 1.0, 0.6655325132796462, 0.0, 0.350824525958222, 0.8033736548767413, 0.0, 0.6334228515625, 0.703680958091337, 0.134976004192787, 0.4538006087975372, 0.0, 0.0, 0.022851496934890747, 0.1769482558593154, 0.9088304992765188, 0.34190391190350056, 0.5909173150416861, 1.0, 0.0, 0.2863569208604894 ], "yaxis": "y2" } ], "layout": { "annotations": [ { "font": { "size": 14 }, "showarrow": false, "text": "Mean", "x": 0.25, "xanchor": "center", "xref": "paper", "y": 1, "yanchor": "bottom", "yref": "paper" }, { "font": { "size": 14 }, "showarrow": false, "text": "Standard Error", "x": 0.8, "xanchor": "center", "xref": "paper", "y": 1, "yanchor": "bottom", "yref": "paper" } ], "autosize": false, "height": 450, "hovermode": "closest", "legend": { "orientation": "h", "x": 0, "y": -0.25 }, "margin": { "b": 100, "l": 35, "pad": 0, "r": 35, "t": 35 }, "template": { "data": { "bar": [ { "error_x": { "color": "#2a3f5f" }, "error_y": { "color": "#2a3f5f" }, "marker": { "line": { "color": "#E5ECF6", "width": 0.5 }, "pattern": { "fillmode": "overlay", "size": 10, "solidity": 0.2 } }, "type": "bar" } ], "barpolar": [ { "marker": { "line": { "color": "#E5ECF6", "width": 0.5 }, "pattern": { "fillmode": "overlay", "size": 10, "solidity": 0.2 } }, "type": "barpolar" } ], "carpet": [ { "aaxis": { "endlinecolor": "#2a3f5f", "gridcolor": "white", "linecolor": "white", "minorgridcolor": "white", "startlinecolor": "#2a3f5f" }, "baxis": { "endlinecolor": "#2a3f5f", "gridcolor": "white", "linecolor": "white", "minorgridcolor": "white", "startlinecolor": "#2a3f5f" }, "type": "carpet" } ], "choropleth": [ { "colorbar": { "outlinewidth": 0, "ticks": "" }, "type": "choropleth" } ], "contour": [ { "colorbar": { "outlinewidth": 0, "ticks": "" }, "colorscale": [ [ 0.0, "#0d0887" ], [ 0.1111111111111111, "#46039f" ], [ 0.2222222222222222, "#7201a8" ], [ 0.3333333333333333, "#9c179e" ], [ 0.4444444444444444, "#bd3786" ], [ 0.5555555555555556, "#d8576b" ], [ 0.6666666666666666, "#ed7953" ], [ 0.7777777777777778, "#fb9f3a" ], [ 0.8888888888888888, "#fdca26" ], [ 1.0, "#f0f921" ] ], "type": "contour" } ], "contourcarpet": [ { "colorbar": { "outlinewidth": 0, "ticks": "" }, "type": "contourcarpet" } ], "heatmap": [ { "colorbar": { "outlinewidth": 0, "ticks": "" }, "colorscale": [ [ 0.0, "#0d0887" ], [ 0.1111111111111111, "#46039f" ], [ 0.2222222222222222, "#7201a8" ], [ 0.3333333333333333, "#9c179e" ], [ 0.4444444444444444, "#bd3786" ], [ 0.5555555555555556, "#d8576b" ], [ 0.6666666666666666, "#ed7953" ], [ 0.7777777777777778, "#fb9f3a" ], [ 0.8888888888888888, "#fdca26" ], [ 1.0, "#f0f921" ] ], "type": "heatmap" } ], "heatmapgl": [ { "colorbar": { "outlinewidth": 0, "ticks": "" }, "colorscale": [ [ 0.0, "#0d0887" ], [ 0.1111111111111111, "#46039f" ], [ 0.2222222222222222, "#7201a8" ], [ 0.3333333333333333, "#9c179e" ], [ 0.4444444444444444, "#bd3786" ], [ 0.5555555555555556, "#d8576b" ], [ 0.6666666666666666, "#ed7953" ], [ 0.7777777777777778, "#fb9f3a" ], [ 0.8888888888888888, "#fdca26" ], [ 1.0, "#f0f921" ] ], "type": "heatmapgl" } ], "histogram": [ { "marker": { "pattern": { "fillmode": "overlay", "size": 10, "solidity": 0.2 } }, "type": "histogram" } ], "histogram2d": [ { "colorbar": { "outlinewidth": 0, "ticks": "" }, "colorscale": [ [ 0.0, "#0d0887" ], [ 0.1111111111111111, "#46039f" ], [ 0.2222222222222222, "#7201a8" ], [ 0.3333333333333333, "#9c179e" ], [ 0.4444444444444444, "#bd3786" ], [ 0.5555555555555556, "#d8576b" ], [ 0.6666666666666666, "#ed7953" ], [ 0.7777777777777778, "#fb9f3a" ], [ 0.8888888888888888, "#fdca26" ], [ 1.0, "#f0f921" ] ], "type": "histogram2d" } ], "histogram2dcontour": [ { "colorbar": { "outlinewidth": 0, "ticks": "" }, "colorscale": [ [ 0.0, "#0d0887" ], [ 0.1111111111111111, "#46039f" ], [ 0.2222222222222222, "#7201a8" ], [ 0.3333333333333333, "#9c179e" ], [ 0.4444444444444444, "#bd3786" ], [ 0.5555555555555556, "#d8576b" ], [ 0.6666666666666666, "#ed7953" ], [ 0.7777777777777778, "#fb9f3a" ], [ 0.8888888888888888, "#fdca26" ], [ 1.0, "#f0f921" ] ], "type": "histogram2dcontour" } ], "mesh3d": [ { "colorbar": { "outlinewidth": 0, "ticks": "" }, "type": "mesh3d" } ], "parcoords": [ { "line": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "type": "parcoords" } ], "pie": [ { "automargin": true, "type": "pie" } ], "scatter": [ { "fillpattern": { "fillmode": "overlay", "size": 10, "solidity": 0.2 }, "type": "scatter" } ], "scatter3d": [ { "line": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "marker": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "type": "scatter3d" } ], "scattercarpet": [ { "marker": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "type": "scattercarpet" } ], "scattergeo": [ { "marker": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "type": "scattergeo" } ], "scattergl": [ { "marker": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "type": "scattergl" } ], "scattermapbox": [ { "marker": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "type": "scattermapbox" } ], "scatterpolar": [ { "marker": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "type": "scatterpolar" } ], "scatterpolargl": [ { "marker": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "type": "scatterpolargl" } ], "scatterternary": [ { "marker": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "type": "scatterternary" } ], "surface": [ { "colorbar": { "outlinewidth": 0, "ticks": "" }, "colorscale": [ [ 0.0, "#0d0887" ], [ 0.1111111111111111, "#46039f" ], [ 0.2222222222222222, "#7201a8" ], [ 0.3333333333333333, "#9c179e" ], [ 0.4444444444444444, "#bd3786" ], [ 0.5555555555555556, "#d8576b" ], [ 0.6666666666666666, "#ed7953" ], [ 0.7777777777777778, "#fb9f3a" ], [ 0.8888888888888888, "#fdca26" ], [ 1.0, "#f0f921" ] ], "type": "surface" } ], "table": [ { "cells": { "fill": { "color": "#EBF0F8" }, "line": { "color": "white" } }, "header": { "fill": { "color": "#C8D4E3" }, "line": { "color": "white" } }, "type": "table" } ] }, "layout": { "annotationdefaults": { "arrowcolor": "#2a3f5f", "arrowhead": 0, "arrowwidth": 1 }, "autotypenumbers": "strict", "coloraxis": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "colorscale": { "diverging": [ [ 0, "#8e0152" ], [ 0.1, "#c51b7d" ], [ 0.2, "#de77ae" ], [ 0.3, "#f1b6da" ], [ 0.4, "#fde0ef" ], [ 0.5, "#f7f7f7" ], [ 0.6, "#e6f5d0" ], [ 0.7, "#b8e186" ], [ 0.8, "#7fbc41" ], [ 0.9, "#4d9221" ], [ 1, "#276419" ] ], "sequential": [ [ 0.0, "#0d0887" ], [ 0.1111111111111111, "#46039f" ], [ 0.2222222222222222, "#7201a8" ], [ 0.3333333333333333, "#9c179e" ], [ 0.4444444444444444, "#bd3786" ], [ 0.5555555555555556, "#d8576b" ], [ 0.6666666666666666, "#ed7953" ], [ 0.7777777777777778, "#fb9f3a" ], [ 0.8888888888888888, "#fdca26" ], [ 1.0, "#f0f921" ] ], "sequentialminus": [ [ 0.0, "#0d0887" ], [ 0.1111111111111111, "#46039f" ], [ 0.2222222222222222, "#7201a8" ], [ 0.3333333333333333, "#9c179e" ], [ 0.4444444444444444, "#bd3786" ], [ 0.5555555555555556, "#d8576b" ], [ 0.6666666666666666, "#ed7953" ], [ 0.7777777777777778, "#fb9f3a" ], [ 0.8888888888888888, "#fdca26" ], [ 1.0, "#f0f921" ] ] }, "colorway": [ "#636efa", "#EF553B", "#00cc96", "#ab63fa", "#FFA15A", "#19d3f3", "#FF6692", "#B6E880", "#FF97FF", "#FECB52" ], "font": { "color": "#2a3f5f" }, "geo": { "bgcolor": "white", "lakecolor": "white", "landcolor": "#E5ECF6", "showlakes": true, "showland": true, "subunitcolor": "white" }, "hoverlabel": { "align": "left" }, "hovermode": "closest", "mapbox": { "style": "light" }, "paper_bgcolor": "white", "plot_bgcolor": "#E5ECF6", "polar": { "angularaxis": { "gridcolor": "white", "linecolor": "white", "ticks": "" }, "bgcolor": "#E5ECF6", "radialaxis": { "gridcolor": "white", "linecolor": "white", "ticks": "" } }, "scene": { "xaxis": { "backgroundcolor": "#E5ECF6", "gridcolor": "white", "gridwidth": 2, "linecolor": "white", "showbackground": true, "ticks": "", "zerolinecolor": "white" }, "yaxis": { "backgroundcolor": "#E5ECF6", "gridcolor": "white", "gridwidth": 2, "linecolor": "white", "showbackground": true, "ticks": "", "zerolinecolor": "white" }, "zaxis": { "backgroundcolor": "#E5ECF6", "gridcolor": "white", "gridwidth": 2, "linecolor": "white", "showbackground": true, "ticks": "", "zerolinecolor": "white" } }, "shapedefaults": { "line": { "color": "#2a3f5f" } }, "ternary": { "aaxis": { "gridcolor": "white", "linecolor": "white", "ticks": "" }, "baxis": { "gridcolor": "white", "linecolor": "white", "ticks": "" }, "bgcolor": "#E5ECF6", "caxis": { "gridcolor": "white", "linecolor": "white", "ticks": "" } }, "title": { "x": 0.05 }, "xaxis": { "automargin": true, "gridcolor": "white", "linecolor": "white", "ticks": "", "title": { "standoff": 15 }, "zerolinecolor": "white", "zerolinewidth": 2 }, "yaxis": { "automargin": true, "gridcolor": "white", "linecolor": "white", "ticks": "", "title": { "standoff": 15 }, "zerolinecolor": "white", "zerolinewidth": 2 } } }, "title": { "text": "accuracy" }, "width": 950, "xaxis": { "anchor": "y", "autorange": false, "domain": [ 0.05, 0.45 ], "exponentformat": "e", "range": [ -6.0, -0.3979400086720376 ], "tickfont": { "size": 11 }, "tickmode": "auto", "title": { "text": "lr" }, "type": "log" }, "xaxis2": { "anchor": "y2", "autorange": false, "domain": [ 0.6, 1 ], "exponentformat": "e", "range": [ -6.0, -0.3979400086720376 ], "tickfont": { "size": 11 }, "tickmode": "auto", "title": { "text": "lr" }, "type": "log" }, "yaxis": { "anchor": "x", "autorange": false, "domain": [ 0, 1 ], "exponentformat": "e", "range": [ 0.0, 1.0 ], "tickfont": { "size": 11 }, "tickmode": "auto", "title": { "text": "momentum" }, "type": "linear" }, "yaxis2": { "anchor": "x2", "autorange": false, "domain": [ 0, 1 ], "exponentformat": "e", "range": [ 0.0, 1.0 ], "tickfont": { "size": 11 }, "tickmode": "auto", "type": "linear" } } }, "text/html": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "render(ax_client.get_contour_plot(param_x=\"lr\", param_y=\"momentum\", metric_name=\"accuracy\"))" ] }, { "cell_type": "markdown", "id": "022a1319", "metadata": { "customInput": null, "originalKey": "5c91d83a-9a90-4ea0-8df9-9d242d998cb3", "papermill": { "duration": 0.056453, "end_time": "2024-07-23T19:43:08.305528", "exception": false, "start_time": "2024-07-23T19:43:08.249075", "status": "completed" }, "showInput": false, "tags": [] }, "source": [ "Here we plot the optimization trace, showing the progression of finding the point with the optimal objective:\n", "\n" ] }, { "cell_type": "code", "execution_count": 15, "id": "673b4eb3", "metadata": { "customInput": null, "customOutput": null, "execution": { "iopub.execute_input": "2024-07-23T19:43:08.420244Z", "iopub.status.busy": "2024-07-23T19:43:08.419964Z", "iopub.status.idle": "2024-07-23T19:43:08.471384Z", "shell.execute_reply": "2024-07-23T19:43:08.470661Z" }, "executionStartTime": 1690415953760, "executionStopTime": 1690415954260, "originalKey": "3a767bdf-7ef3-48e7-b853-6fae5e9e02ff", "papermill": { "duration": 0.110579, "end_time": "2024-07-23T19:43:08.472960", "exception": false, "start_time": "2024-07-23T19:43:08.362381", "status": "completed" }, "requestMsgId": "043de459-6a28-4796-b237-808385c9e54c", "showInput": true, "tags": [] }, "outputs": [ { "data": { "application/vnd.plotly.v1+json": { "config": { "linkText": "Export to plot.ly", "plotlyServerURL": "https://plot.ly", "showLink": false }, "data": [ { "hoverinfo": "none", "legendgroup": "", "line": { "width": 0 }, "mode": "lines", "showlegend": false, "type": "scatter", "x": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26 ], "y": [ 0.8418333333333333, 0.8418333333333333, 0.8418333333333333, 0.8418333333333333, 0.926, 0.929, 0.929, 0.929, 0.929, 0.929, 0.929, 0.929, 0.9456666666666667, 0.9456666666666667, 0.9456666666666667, 0.9476666666666667, 0.9476666666666667, 0.9476666666666667, 0.9476666666666667, 0.9476666666666667, 0.9476666666666667, 0.9476666666666667, 0.9476666666666667, 0.9476666666666667, 0.9476666666666667, 0.9476666666666667 ] }, { "fill": "tonexty", "fillcolor": "rgba(128,177,211,0.3)", "legendgroup": "objective value", "line": { "color": "rgba(128,177,211,1)" }, "mode": "lines", "name": "objective value", "text": [ "
Parameterization:
lr: 2.6e-05
momentum: 0.58", "
Parameterization:
lr: 0.00995509203921593
momentum: 0.6334228515625", "
Parameterization:
lr: 5.183362665905935e-06
momentum: 0.022851496934890747", "
Parameterization:
lr: 6.6349677287433636e-06
momentum: 0.1769482558593154", "
Parameterization:
lr: 8.154817047148868e-05
momentum: 0.9088304992765188", "
Parameterization:
lr: 0.000302076728574865
momentum: 0.34190391190350056", "
Parameterization:
lr: 0.00013731096665846605
momentum: 0.5909173150416861", "
Parameterization:
lr: 1.0326238819187764e-05
momentum: 1.0", "
Parameterization:
lr: 0.00024629479479540775
momentum: 0.0", "
Parameterization:
lr: 0.0001494385653033887
momentum: 0.2863569208604894", "
Parameterization:
lr: 1e-06
momentum: 1.0", "
Parameterization:
lr: 3.732184409504214e-05
momentum: 1.0", "
Parameterization:
lr: 0.0002610449283938493
momentum: 0.9135505379162181", "
Parameterization:
lr: 0.00017053570203455322
momentum: 0.7995431686056417", "
Parameterization:
lr: 0.0001592376912143492
momentum: 1.0", "
Parameterization:
lr: 0.00031243004356781596
momentum: 0.6655325132796462", "
Parameterization:
lr: 0.0008236218541279722
momentum: 0.0", "
Parameterization:
lr: 0.0006943048179834304
momentum: 0.350824525958222", "
Parameterization:
lr: 0.0005822252247415435
momentum: 0.8033736548767413", "
Parameterization:
lr: 0.4
momentum: 0.0", "
Parameterization:
lr: 7.243686845468841e-05
momentum: 0.703680958091337", "
Parameterization:
lr: 0.00045082139108132483
momentum: 0.134976004192787", "
Parameterization:
lr: 8.453410095178891e-05
momentum: 0.4538006087975372", "
Parameterization:
lr: 0.0028278053483573595
momentum: 0.0", "
Parameterization:
lr: 6.461659285089037e-05
momentum: 0.0", "
Parameterization:
lr: 4.406690932652801e-06
momentum: 1.0" ], "type": "scatter", "x": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26 ], "y": [ 0.8418333333333333, 0.8418333333333333, 0.8418333333333333, 0.8418333333333333, 0.926, 0.929, 0.929, 0.929, 0.929, 0.929, 0.929, 0.929, 0.9456666666666667, 0.9456666666666667, 0.9456666666666667, 0.9476666666666667, 0.9476666666666667, 0.9476666666666667, 0.9476666666666667, 0.9476666666666667, 0.9476666666666667, 0.9476666666666667, 0.9476666666666667, 0.9476666666666667, 0.9476666666666667, 0.9476666666666667 ] }, { "fill": "tonexty", "fillcolor": "rgba(128,177,211,0.3)", "hoverinfo": "none", "legendgroup": "", "line": { "width": 0 }, "mode": "lines", "showlegend": false, "type": "scatter", "x": [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26 ], "y": [ 0.8418333333333333, 0.8418333333333333, 0.8418333333333333, 0.8418333333333333, 0.926, 0.929, 0.929, 0.929, 0.929, 0.929, 0.929, 0.929, 0.9456666666666667, 0.9456666666666667, 0.9456666666666667, 0.9476666666666667, 0.9476666666666667, 0.9476666666666667, 0.9476666666666667, 0.9476666666666667, 0.9476666666666667, 0.9476666666666667, 0.9476666666666667, 0.9476666666666667, 0.9476666666666667, 0.9476666666666667 ] } ], "layout": { "showlegend": true, "template": { "data": { "bar": [ { "error_x": { "color": "#2a3f5f" }, "error_y": { "color": "#2a3f5f" }, "marker": { "line": { "color": "#E5ECF6", "width": 0.5 }, "pattern": { "fillmode": "overlay", "size": 10, "solidity": 0.2 } }, "type": "bar" } ], "barpolar": [ { "marker": { "line": { "color": "#E5ECF6", "width": 0.5 }, "pattern": { "fillmode": "overlay", "size": 10, "solidity": 0.2 } }, "type": "barpolar" } ], "carpet": [ { "aaxis": { "endlinecolor": "#2a3f5f", "gridcolor": "white", "linecolor": "white", "minorgridcolor": "white", "startlinecolor": "#2a3f5f" }, "baxis": { "endlinecolor": "#2a3f5f", "gridcolor": "white", "linecolor": "white", "minorgridcolor": "white", "startlinecolor": "#2a3f5f" }, "type": "carpet" } ], "choropleth": [ { "colorbar": { "outlinewidth": 0, "ticks": "" }, "type": "choropleth" } ], "contour": [ { "colorbar": { "outlinewidth": 0, "ticks": "" }, "colorscale": [ [ 0.0, "#0d0887" ], [ 0.1111111111111111, "#46039f" ], [ 0.2222222222222222, "#7201a8" ], [ 0.3333333333333333, "#9c179e" ], [ 0.4444444444444444, "#bd3786" ], [ 0.5555555555555556, "#d8576b" ], [ 0.6666666666666666, "#ed7953" ], [ 0.7777777777777778, "#fb9f3a" ], [ 0.8888888888888888, "#fdca26" ], [ 1.0, "#f0f921" ] ], "type": "contour" } ], "contourcarpet": [ { "colorbar": { "outlinewidth": 0, "ticks": "" }, "type": "contourcarpet" } ], "heatmap": [ { "colorbar": { "outlinewidth": 0, "ticks": "" }, "colorscale": [ [ 0.0, "#0d0887" ], [ 0.1111111111111111, "#46039f" ], [ 0.2222222222222222, "#7201a8" ], [ 0.3333333333333333, "#9c179e" ], [ 0.4444444444444444, "#bd3786" ], [ 0.5555555555555556, "#d8576b" ], [ 0.6666666666666666, "#ed7953" ], [ 0.7777777777777778, "#fb9f3a" ], [ 0.8888888888888888, "#fdca26" ], [ 1.0, "#f0f921" ] ], "type": "heatmap" } ], "heatmapgl": [ { "colorbar": { "outlinewidth": 0, "ticks": "" }, "colorscale": [ [ 0.0, "#0d0887" ], [ 0.1111111111111111, "#46039f" ], [ 0.2222222222222222, "#7201a8" ], [ 0.3333333333333333, "#9c179e" ], [ 0.4444444444444444, "#bd3786" ], [ 0.5555555555555556, "#d8576b" ], [ 0.6666666666666666, "#ed7953" ], [ 0.7777777777777778, "#fb9f3a" ], [ 0.8888888888888888, "#fdca26" ], [ 1.0, "#f0f921" ] ], "type": "heatmapgl" } ], "histogram": [ { "marker": { "pattern": { "fillmode": "overlay", "size": 10, "solidity": 0.2 } }, "type": "histogram" } ], "histogram2d": [ { "colorbar": { "outlinewidth": 0, "ticks": "" }, "colorscale": [ [ 0.0, "#0d0887" ], [ 0.1111111111111111, "#46039f" ], [ 0.2222222222222222, "#7201a8" ], [ 0.3333333333333333, "#9c179e" ], [ 0.4444444444444444, "#bd3786" ], [ 0.5555555555555556, "#d8576b" ], [ 0.6666666666666666, "#ed7953" ], [ 0.7777777777777778, "#fb9f3a" ], [ 0.8888888888888888, "#fdca26" ], [ 1.0, "#f0f921" ] ], "type": "histogram2d" } ], "histogram2dcontour": [ { "colorbar": { "outlinewidth": 0, "ticks": "" }, "colorscale": [ [ 0.0, "#0d0887" ], [ 0.1111111111111111, "#46039f" ], [ 0.2222222222222222, "#7201a8" ], [ 0.3333333333333333, "#9c179e" ], [ 0.4444444444444444, "#bd3786" ], [ 0.5555555555555556, "#d8576b" ], [ 0.6666666666666666, "#ed7953" ], [ 0.7777777777777778, "#fb9f3a" ], [ 0.8888888888888888, "#fdca26" ], [ 1.0, "#f0f921" ] ], "type": "histogram2dcontour" } ], "mesh3d": [ { "colorbar": { "outlinewidth": 0, "ticks": "" }, "type": "mesh3d" } ], "parcoords": [ { "line": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "type": "parcoords" } ], "pie": [ { "automargin": true, "type": "pie" } ], "scatter": [ { "fillpattern": { "fillmode": "overlay", "size": 10, "solidity": 0.2 }, "type": "scatter" } ], "scatter3d": [ { "line": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "marker": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "type": "scatter3d" } ], "scattercarpet": [ { "marker": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "type": "scattercarpet" } ], "scattergeo": [ { "marker": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "type": "scattergeo" } ], "scattergl": [ { "marker": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "type": "scattergl" } ], "scattermapbox": [ { "marker": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "type": "scattermapbox" } ], "scatterpolar": [ { "marker": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "type": "scatterpolar" } ], "scatterpolargl": [ { "marker": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "type": "scatterpolargl" } ], "scatterternary": [ { "marker": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "type": "scatterternary" } ], "surface": [ { "colorbar": { "outlinewidth": 0, "ticks": "" }, "colorscale": [ [ 0.0, "#0d0887" ], [ 0.1111111111111111, "#46039f" ], [ 0.2222222222222222, "#7201a8" ], [ 0.3333333333333333, "#9c179e" ], [ 0.4444444444444444, "#bd3786" ], [ 0.5555555555555556, "#d8576b" ], [ 0.6666666666666666, "#ed7953" ], [ 0.7777777777777778, "#fb9f3a" ], [ 0.8888888888888888, "#fdca26" ], [ 1.0, "#f0f921" ] ], "type": "surface" } ], "table": [ { "cells": { "fill": { "color": "#EBF0F8" }, "line": { "color": "white" } }, "header": { "fill": { "color": "#C8D4E3" }, "line": { "color": "white" } }, "type": "table" } ] }, "layout": { "annotationdefaults": { "arrowcolor": "#2a3f5f", "arrowhead": 0, "arrowwidth": 1 }, "autotypenumbers": "strict", "coloraxis": { "colorbar": { "outlinewidth": 0, "ticks": "" } }, "colorscale": { "diverging": [ [ 0, "#8e0152" ], [ 0.1, "#c51b7d" ], [ 0.2, "#de77ae" ], [ 0.3, "#f1b6da" ], [ 0.4, "#fde0ef" ], [ 0.5, "#f7f7f7" ], [ 0.6, "#e6f5d0" ], [ 0.7, "#b8e186" ], [ 0.8, "#7fbc41" ], [ 0.9, "#4d9221" ], [ 1, "#276419" ] ], "sequential": [ [ 0.0, "#0d0887" ], [ 0.1111111111111111, "#46039f" ], [ 0.2222222222222222, "#7201a8" ], [ 0.3333333333333333, "#9c179e" ], [ 0.4444444444444444, "#bd3786" ], [ 0.5555555555555556, "#d8576b" ], [ 0.6666666666666666, "#ed7953" ], [ 0.7777777777777778, "#fb9f3a" ], [ 0.8888888888888888, "#fdca26" ], [ 1.0, "#f0f921" ] ], "sequentialminus": [ [ 0.0, "#0d0887" ], [ 0.1111111111111111, "#46039f" ], [ 0.2222222222222222, "#7201a8" ], [ 0.3333333333333333, "#9c179e" ], [ 0.4444444444444444, "#bd3786" ], [ 0.5555555555555556, "#d8576b" ], [ 0.6666666666666666, "#ed7953" ], [ 0.7777777777777778, "#fb9f3a" ], [ 0.8888888888888888, "#fdca26" ], [ 1.0, "#f0f921" ] ] }, "colorway": [ "#636efa", "#EF553B", "#00cc96", "#ab63fa", "#FFA15A", "#19d3f3", "#FF6692", "#B6E880", "#FF97FF", "#FECB52" ], "font": { "color": "#2a3f5f" }, "geo": { "bgcolor": "white", "lakecolor": "white", "landcolor": "#E5ECF6", "showlakes": true, "showland": true, "subunitcolor": "white" }, "hoverlabel": { "align": "left" }, "hovermode": "closest", "mapbox": { "style": "light" }, "paper_bgcolor": "white", "plot_bgcolor": "#E5ECF6", "polar": { "angularaxis": { "gridcolor": "white", "linecolor": "white", "ticks": "" }, "bgcolor": "#E5ECF6", "radialaxis": { "gridcolor": "white", "linecolor": "white", "ticks": "" } }, "scene": { "xaxis": { "backgroundcolor": "#E5ECF6", "gridcolor": "white", "gridwidth": 2, "linecolor": "white", "showbackground": true, "ticks": "", "zerolinecolor": "white" }, "yaxis": { "backgroundcolor": "#E5ECF6", "gridcolor": "white", "gridwidth": 2, "linecolor": "white", "showbackground": true, "ticks": "", "zerolinecolor": "white" }, "zaxis": { "backgroundcolor": "#E5ECF6", "gridcolor": "white", "gridwidth": 2, "linecolor": "white", "showbackground": true, "ticks": "", "zerolinecolor": "white" } }, "shapedefaults": { "line": { "color": "#2a3f5f" } }, "ternary": { "aaxis": { "gridcolor": "white", "linecolor": "white", "ticks": "" }, "baxis": { "gridcolor": "white", "linecolor": "white", "ticks": "" }, "bgcolor": "#E5ECF6", "caxis": { "gridcolor": "white", "linecolor": "white", "ticks": "" } }, "title": { "x": 0.05 }, "xaxis": { "automargin": true, "gridcolor": "white", "linecolor": "white", "ticks": "", "title": { "standoff": 15 }, "zerolinecolor": "white", "zerolinewidth": 2 }, "yaxis": { "automargin": true, "gridcolor": "white", "linecolor": "white", "ticks": "", "title": { "standoff": 15 }, "zerolinecolor": "white", "zerolinewidth": 2 } } }, "title": { "text": "Best objective found vs. # of iterations" }, "xaxis": { "title": { "text": "Iteration" } }, "yaxis": { "title": { "text": "Accuracy" } } } }, "text/html": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "render(\n", " ax_client.get_optimization_trace()\n", ") " ] }, { "cell_type": "markdown", "id": "41a86b6f", "metadata": { "customInput": null, "executionStartTime": 1689617061294, "executionStopTime": 1689617061325, "originalKey": "09aaec9d-c178-42e2-b549-663cd17f8c3d", "papermill": { "duration": 0.059743, "end_time": "2024-07-23T19:43:08.593271", "exception": false, "start_time": "2024-07-23T19:43:08.533528", "status": "completed" }, "requestMsgId": "09aaec9d-c178-42e2-b549-663cd17f8c3d", "showInput": false, "tags": [] }, "source": [ "## 8. Train CNN with best hyperparameters and evaluate on test set\n", "Note that the resulting accuracy on the test set generally won't be the same as the maximum accuracy achieved on the evaluation set throughout optimization. " ] }, { "cell_type": "code", "execution_count": 16, "id": "cf5db231", "metadata": { "customInput": null, "customOutput": null, "execution": { "iopub.execute_input": "2024-07-23T19:43:08.720178Z", "iopub.status.busy": "2024-07-23T19:43:08.719904Z", "iopub.status.idle": "2024-07-23T19:43:08.743199Z", "shell.execute_reply": "2024-07-23T19:43:08.742529Z" }, "executionStartTime": 1690415954397, "executionStopTime": 1690415954452, "originalKey": "27f92d16-93c4-43bb-a37f-e7a1aeecd856", "papermill": { "duration": 0.085708, "end_time": "2024-07-23T19:43:08.744495", "exception": false, "start_time": "2024-07-23T19:43:08.658787", "status": "completed" }, "requestMsgId": "07eba5ce-bebe-4588-8dbb-07553efeb2b0", "showInput": true, "tags": [] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "[WARNING 07-23 19:43:08] ax.service.utils.report_utils: Column reason missing for all trials. Not appending column.\n" ] }, { "data": { "text/plain": [ "{'lr': 0.00031243004356781596, 'momentum': 0.6655325132796462}" ] }, "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ "df = ax_client.get_trials_data_frame()\n", "best_arm_idx = df.trial_index[df[\"accuracy\"] == df[\"accuracy\"].max()].values[0]\n", "best_arm = ax_client.get_trial_parameters(best_arm_idx)\n", "best_arm" ] }, { "cell_type": "code", "execution_count": 17, "id": "e1d13733", "metadata": { "customInput": null, "customOutput": null, "execution": { "iopub.execute_input": "2024-07-23T19:43:08.866059Z", "iopub.status.busy": "2024-07-23T19:43:08.865399Z", "iopub.status.idle": "2024-07-23T19:43:08.869596Z", "shell.execute_reply": "2024-07-23T19:43:08.868926Z" }, "executionStartTime": 1690415954677, "executionStopTime": 1690415954681, "originalKey": "d0c7c645-c230-4654-a3b5-a01c61a09393", "papermill": { "duration": 0.066381, "end_time": "2024-07-23T19:43:08.870975", "exception": false, "start_time": "2024-07-23T19:43:08.804594", "status": "completed" }, "requestMsgId": "0a962cef-65a1-4f95-9410-37a9a8e5c5ac", "showInput": true, "tags": [] }, "outputs": [], "source": [ "combined_train_valid_set = torch.utils.data.ConcatDataset(\n", " [\n", " train_loader.dataset.dataset,\n", " valid_loader.dataset.dataset,\n", " ]\n", ")\n", "combined_train_valid_loader = torch.utils.data.DataLoader(\n", " combined_train_valid_set,\n", " batch_size=BATCH_SIZE,\n", " shuffle=True,\n", ")" ] }, { "cell_type": "code", "execution_count": 18, "id": "3c6858fb", "metadata": { "customInput": null, "customOutput": null, "execution": { "iopub.execute_input": "2024-07-23T19:43:08.992423Z", "iopub.status.busy": "2024-07-23T19:43:08.992133Z", "iopub.status.idle": "2024-07-23T19:43:38.712901Z", "shell.execute_reply": "2024-07-23T19:43:38.712047Z" }, "executionStartTime": 1690415954791, "executionStopTime": 1690416061340, "originalKey": "5695c78b-4c6e-4d35-ab08-6c60781bd8f1", "papermill": { "duration": 29.783557, "end_time": "2024-07-23T19:43:38.714907", "exception": false, "start_time": "2024-07-23T19:43:08.931350", "status": "completed" }, "requestMsgId": "e22fa0c7-88cc-4d8a-bb7d-4f96fbae9a42", "showInput": true, "tags": [] }, "outputs": [], "source": [ "net = train(\n", " net=CNN(),\n", " train_loader=combined_train_valid_loader,\n", " parameters=best_arm,\n", " dtype=dtype,\n", " device=device,\n", ")\n", "test_accuracy = evaluate(\n", " net=net,\n", " data_loader=test_loader,\n", " dtype=dtype,\n", " device=device,\n", ")" ] }, { "cell_type": "code", "execution_count": 19, "id": "6376bee9", "metadata": { "customInput": null, "customOutput": null, "execution": { "iopub.execute_input": "2024-07-23T19:43:38.839464Z", "iopub.status.busy": "2024-07-23T19:43:38.838794Z", "iopub.status.idle": "2024-07-23T19:43:38.842880Z", "shell.execute_reply": "2024-07-23T19:43:38.842214Z" }, "executionStartTime": 1690416061460, "executionStopTime": 1690416061467, "originalKey": "7522e229-9641-4383-a892-12c3f0a8011c", "papermill": { "duration": 0.06731, "end_time": "2024-07-23T19:43:38.844201", "exception": false, "start_time": "2024-07-23T19:43:38.776891", "status": "completed" }, "requestMsgId": "5552d77d-9c9d-4712-9256-2cb3da836f2c", "showInput": true, "tags": [] }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Classification Accuracy (test set): 97.96%\n" ] } ], "source": [ "print(f\"Classification Accuracy (test set): {round(test_accuracy*100, 2)}%\")" ] }, { "cell_type": "markdown", "id": "accaf010", "metadata": { "customInput": null, "originalKey": "c8232211-4837-4677-b86c-bce730635fff", "papermill": { "duration": 0.06064, "end_time": "2024-07-23T19:43:38.965646", "exception": false, "start_time": "2024-07-23T19:43:38.905006", "status": "completed" }, "showInput": false, "tags": [] }, "source": [ "## 9. Save / reload optimization to JSON / SQL\n", "We can serialize the state of optimization to JSON and save it to a `.json` file or save it to the SQL backend. For the former:" ] }, { "cell_type": "code", "execution_count": 20, "id": "6ea01dc4", "metadata": { "customInput": null, "customOutput": null, "execution": { "iopub.execute_input": "2024-07-23T19:43:39.094034Z", "iopub.status.busy": "2024-07-23T19:43:39.093349Z", "iopub.status.idle": "2024-07-23T19:43:39.121538Z", "shell.execute_reply": "2024-07-23T19:43:39.120921Z" }, "executionStartTime": 1690416061571, "executionStopTime": 1690416061657, "originalKey": "6afddb45-c980-4b14-b5e9-927747ea98ea", "papermill": { "duration": 0.096175, "end_time": "2024-07-23T19:43:39.122865", "exception": false, "start_time": "2024-07-23T19:43:39.026690", "status": "completed" }, "requestMsgId": "bab02be8-706c-4422-b97b-c222b5084bba", "showInput": true, "tags": [] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:43:39] ax.service.ax_client: Saved JSON-serialized state of optimization to `ax_client_snapshot.json`.\n" ] } ], "source": [ "ax_client.save_to_json_file() # For custom filepath, pass `filepath` argument." ] }, { "cell_type": "code", "execution_count": 21, "id": "99a2088d", "metadata": { "customInput": null, "customOutput": null, "execution": { "iopub.execute_input": "2024-07-23T19:43:39.247397Z", "iopub.status.busy": "2024-07-23T19:43:39.246681Z", "iopub.status.idle": "2024-07-23T19:43:39.380804Z", "shell.execute_reply": "2024-07-23T19:43:39.380175Z" }, "executionStartTime": 1690416061758, "executionStopTime": 1690416062132, "originalKey": "31e6f7b4-cf6b-4967-95ff-f76d03657fb2", "papermill": { "duration": 0.197745, "end_time": "2024-07-23T19:43:39.382186", "exception": false, "start_time": "2024-07-23T19:43:39.184441", "status": "completed" }, "requestMsgId": "f2d10848-f995-420d-88e7-9036894d7b1b", "showInput": true, "tags": [] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:43:39] ax.service.ax_client: Starting optimization with verbose logging. To disable logging, set the `verbose_logging` argument to `False`. Note that float values in the logs are rounded to 6 decimal points.\n" ] } ], "source": [ "restored_ax_client = (\n", " AxClient.load_from_json_file()\n", ") # For custom filepath, pass `filepath` argument." ] }, { "cell_type": "markdown", "id": "7fdda0be", "metadata": { "customInput": null, "originalKey": "122510f5-5b9e-4b1c-9f5e-8c8ea2e08848", "papermill": { "duration": 0.060874, "end_time": "2024-07-23T19:43:39.503967", "exception": false, "start_time": "2024-07-23T19:43:39.443093", "status": "completed" }, "showInput": false, "tags": [] }, "source": [ "To store state of optimization to an SQL backend, first follow [setup instructions](https://ax.dev/docs/storage.html#sql) on Ax website." ] }, { "cell_type": "markdown", "id": "36c2564f", "metadata": { "customInput": null, "originalKey": "bd80e639-aa0f-4dc1-8542-0caf0d674fda", "papermill": { "duration": 0.06038, "end_time": "2024-07-23T19:43:39.625235", "exception": false, "start_time": "2024-07-23T19:43:39.564855", "status": "completed" }, "showInput": false, "tags": [] }, "source": [ "Having set up the SQL backend, pass `DBSettings` to `AxClient` on instantiation (note that `SQLAlchemy` dependency will have to be installed – for installation, refer to [optional dependencies](https://ax.dev/docs/installation.html#optional-dependencies) on Ax website):" ] }, { "cell_type": "code", "execution_count": 22, "id": "693722fb", "metadata": { "customInput": null, "customOutput": null, "execution": { "iopub.execute_input": "2024-07-23T19:43:39.747786Z", "iopub.status.busy": "2024-07-23T19:43:39.747490Z", "iopub.status.idle": "2024-07-23T19:43:39.758519Z", "shell.execute_reply": "2024-07-23T19:43:39.757954Z" }, "executionStartTime": 1690416062222, "executionStopTime": 1690416062314, "originalKey": "80eb6a2e-6564-405e-b5d4-d448e32dbf60", "papermill": { "duration": 0.074183, "end_time": "2024-07-23T19:43:39.759878", "exception": false, "start_time": "2024-07-23T19:43:39.685695", "status": "completed" }, "requestMsgId": "65f2307f-b800-4415-b9e7-11734a2a6889", "showInput": true, "tags": [] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:43:39] ax.service.ax_client: Starting optimization with verbose logging. To disable logging, set the `verbose_logging` argument to `False`. Note that float values in the logs are rounded to 6 decimal points.\n" ] } ], "source": [ "from ax.storage.sqa_store.structs import DBSettings\n", "\n", "# URL is of the form \"dialect+driver://username:password@host:port/database\".\n", "db_settings = DBSettings(url=\"sqlite:///foo.db\")\n", "# Instead of URL, can provide a `creator function`; can specify custom encoders/decoders if necessary.\n", "new_ax = AxClient(db_settings=db_settings)" ] }, { "cell_type": "markdown", "id": "9a24f046", "metadata": { "customInput": null, "originalKey": "adafd3aa-b84e-4e86-9694-a29f94c6d5f3", "papermill": { "duration": 0.060884, "end_time": "2024-07-23T19:43:39.882049", "exception": false, "start_time": "2024-07-23T19:43:39.821165", "status": "completed" }, "showInput": false, "tags": [] }, "source": [ "When valid `DBSettings` are passed into `AxClient`, a unique experiment name is a required argument (`name`) to `ax_client.create_experiment`. The **state of the optimization is auto-saved** any time it changes (i.e. a new trial is added or completed, etc). \n", "\n", "To reload an optimization state later, instantiate `AxClient` with the same `DBSettings` and use `ax_client.load_experiment_from_database(experiment_name=\"my_experiment\")`." ] }, { "cell_type": "markdown", "id": "4e435a66", "metadata": { "customInput": null, "originalKey": "2f4a875b-1e18-4352-955d-576d6b01c5ed", "papermill": { "duration": 0.061172, "end_time": "2024-07-23T19:43:40.004799", "exception": false, "start_time": "2024-07-23T19:43:39.943627", "status": "completed" }, "showInput": false, "tags": [] }, "source": [ "# Special Cases" ] }, { "cell_type": "markdown", "id": "f4babbd7", "metadata": { "customInput": null, "originalKey": "0d49e448-4768-401d-ac1d-810aee633c9a", "papermill": { "duration": 0.061182, "end_time": "2024-07-23T19:43:40.128739", "exception": false, "start_time": "2024-07-23T19:43:40.067557", "status": "completed" }, "showInput": false, "tags": [] }, "source": [ "**Evaluation failure**: should any optimization iterations fail during evaluation, `log_trial_failure` will ensure that the same trial is not proposed again." ] }, { "cell_type": "code", "execution_count": 23, "id": "ee17414e", "metadata": { "customInput": null, "customOutput": null, "execution": { "iopub.execute_input": "2024-07-23T19:43:40.252216Z", "iopub.status.busy": "2024-07-23T19:43:40.251681Z", "iopub.status.idle": "2024-07-23T19:43:41.267558Z", "shell.execute_reply": "2024-07-23T19:43:41.266814Z" }, "executionStartTime": 1690416062420, "executionStopTime": 1690416064316, "originalKey": "faa83f1d-31da-481a-96e4-ccbc12f30b91", "papermill": { "duration": 1.079395, "end_time": "2024-07-23T19:43:41.269142", "exception": false, "start_time": "2024-07-23T19:43:40.189747", "status": "completed" }, "requestMsgId": "80a40c3a-76ed-4e1d-aa77-3652fadbe69f", "showInput": true, "tags": [] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:43:41] ax.service.ax_client: Generated new trial 26 with parameters {'lr': 0.000204, 'momentum': 0.53482} using model BoTorch.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:43:41] ax.service.ax_client: Registered failure of trial 26.\n" ] } ], "source": [ "_, trial_index = ax_client.get_next_trial()\n", "ax_client.log_trial_failure(trial_index=trial_index)" ] }, { "cell_type": "markdown", "id": "f760cb8f", "metadata": { "customInput": null, "originalKey": "c826a96e-9431-49bd-87d7-62b517537a15", "papermill": { "duration": 0.060838, "end_time": "2024-07-23T19:43:41.392157", "exception": false, "start_time": "2024-07-23T19:43:41.331319", "status": "completed" }, "showInput": false, "tags": [] }, "source": [ "**Need to run many trials in parallel**: for optimal results and optimization efficiency, we strongly recommend sequential optimization (generating a few trials, then waiting for them to be completed with evaluation data). However, if your use case needs to dispatch many trials in parallel before they are updated with data and you are running into the *\"All trials for current model have been generated, but not enough data has been observed to fit next model\"* error, instantiate `AxClient` as `AxClient(enforce_sequential_optimization=False)`." ] }, { "cell_type": "markdown", "id": "094f1603", "metadata": { "customInput": null, "originalKey": "683378e0-893b-49a1-b090-084dc394da1a", "papermill": { "duration": 0.060666, "end_time": "2024-07-23T19:43:41.514076", "exception": false, "start_time": "2024-07-23T19:43:41.453410", "status": "completed" }, "showInput": false, "tags": [] }, "source": [ "# Service API Exceptions Meaning and Handling\n", "[**`DataRequiredError`**](https://ax.dev/api/exceptions.html#ax.exceptions.core.DataRequiredError): Ax generation strategy needs to be updated with more data to proceed to the next optimization model. When the optimization moves from initialization stage to the Bayesian optimization stage, the underlying BayesOpt model needs sufficient data to train. For optimal results and optimization efficiency (finding the optimal point in the least number of trials), we recommend sequential optimization (generating a few trials, then waiting for them to be completed with evaluation data). Therefore, the correct way to handle this exception is to wait until more trial evaluations complete and log their data via `ax_client.complete_trial(...)`. \n", "\n", "However, if there is strong need to generate more trials before more data is available, instantiate `AxClient` as `AxClient(enforce_sequential_optimization=False)`. With this setting, as many trials will be generated from the initialization stage as requested, and the optimization will move to the BayesOpt stage whenever enough trials are completed." ] }, { "cell_type": "markdown", "id": "9d471810", "metadata": { "customInput": null, "originalKey": "4602d41d-43aa-46d2-9ca6-392c414d0b5f", "papermill": { "duration": 0.060257, "end_time": "2024-07-23T19:43:41.634998", "exception": false, "start_time": "2024-07-23T19:43:41.574741", "status": "completed" }, "showInput": false, "tags": [] }, "source": [ "[**`MaxParallelismReachedException`**](https://ax.dev/api/modelbridge.html#ax.modelbridge.generation_strategy.MaxParallelismReachedException): generation strategy restricts the number of trials that can be run simultaneously (to encourage sequential optimization), and the parallelism limit has been reached. The correct way to handle this exception is the same as `DataRequiredError` – to wait until more trial evluations complete and log their data via `ax_client.complete_trial(...)`.\n", " \n", "In some cases higher parallelism is important, so `enforce_sequential_optimization=False` kwarg to AxClient allows the user to suppress limiting of parallelism. It's also possible to override the default parallelism setting for all stages of the optimization by passing `choose_generation_strategy_kwargs` to `ax_client.create_experiment`:" ] }, { "cell_type": "code", "execution_count": 24, "id": "cdc51d65", "metadata": { "customInput": null, "customOutput": null, "execution": { "iopub.execute_input": "2024-07-23T19:43:41.756462Z", "iopub.status.busy": "2024-07-23T19:43:41.755972Z", "iopub.status.idle": "2024-07-23T19:43:41.768652Z", "shell.execute_reply": "2024-07-23T19:43:41.768076Z" }, "executionStartTime": 1690416064534, "executionStopTime": 1690416064564, "originalKey": "d62e6cfd-5127-450e-80b7-d0edcaf97d6c", "papermill": { "duration": 0.074914, "end_time": "2024-07-23T19:43:41.769891", "exception": false, "start_time": "2024-07-23T19:43:41.694977", "status": "completed" }, "requestMsgId": "cb9a17f9-5734-41c6-9018-c0635c61d8b3", "showInput": true, "tags": [] }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:43:41] ax.service.ax_client: Starting optimization with verbose logging. To disable logging, set the `verbose_logging` argument to `False`. Note that float values in the logs are rounded to 6 decimal points.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:43:41] ax.service.utils.instantiation: Inferred value type of ParameterType.FLOAT for parameter x. If that is not the expected value type, you can explicitly specify 'value_type' ('int', 'float', 'bool' or 'str') in parameter dict.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:43:41] ax.service.utils.instantiation: Inferred value type of ParameterType.FLOAT for parameter y. If that is not the expected value type, you can explicitly specify 'value_type' ('int', 'float', 'bool' or 'str') in parameter dict.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:43:41] ax.service.utils.instantiation: Created search space: SearchSpace(parameters=[RangeParameter(name='x', parameter_type=FLOAT, range=[-5.0, 10.0]), RangeParameter(name='y', parameter_type=FLOAT, range=[0.0, 15.0])], parameter_constraints=[]).\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:43:41] ax.modelbridge.dispatch_utils: Using Models.BOTORCH_MODULAR since there is at least one ordered parameter and there are no unordered categorical parameters.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:43:41] ax.modelbridge.dispatch_utils: Calculating the number of remaining initialization trials based on num_initialization_trials=None max_initialization_trials=None num_tunable_parameters=2 num_trials=None use_batch_trials=False\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:43:41] ax.modelbridge.dispatch_utils: calculated num_initialization_trials=5\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:43:41] ax.modelbridge.dispatch_utils: num_completed_initialization_trials=0 num_remaining_initialization_trials=5\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:43:41] ax.modelbridge.dispatch_utils: `verbose`, `disable_progbar`, and `jit_compile` are not yet supported when using `choose_generation_strategy` with ModularBoTorchModel, dropping these arguments.\n" ] }, { "name": "stderr", "output_type": "stream", "text": [ "[INFO 07-23 19:43:41] ax.modelbridge.dispatch_utils: Using Bayesian Optimization generation strategy: GenerationStrategy(name='Sobol+BoTorch', steps=[Sobol for 5 trials, BoTorch for subsequent trials]). Iterations after 5 will take longer to generate due to model-fitting.\n" ] } ], "source": [ "ax_client = AxClient()\n", "ax_client.create_experiment(\n", " parameters=[\n", " {\"name\": \"x\", \"type\": \"range\", \"bounds\": [-5.0, 10.0]},\n", " {\"name\": \"y\", \"type\": \"range\", \"bounds\": [0.0, 15.0]},\n", " ],\n", " # Sets max parallelism to 10 for all steps of the generation strategy.\n", " choose_generation_strategy_kwargs={\"max_parallelism_override\": 10},\n", ")" ] }, { "cell_type": "code", "execution_count": 25, "id": "2d7d1400", "metadata": { "customInput": null, "customOutput": null, "execution": { "iopub.execute_input": "2024-07-23T19:43:41.893562Z", "iopub.status.busy": "2024-07-23T19:43:41.893052Z", "iopub.status.idle": "2024-07-23T19:43:41.897792Z", "shell.execute_reply": "2024-07-23T19:43:41.897134Z" }, "executionStartTime": 1690416064679, "executionStopTime": 1690416064702, "originalKey": "bc15d2cf-8ddc-4d66-83b6-7469cd15aa4d", "papermill": { "duration": 0.068024, "end_time": "2024-07-23T19:43:41.899141", "exception": false, "start_time": "2024-07-23T19:43:41.831117", "status": "completed" }, "requestMsgId": "996c4bd3-b296-4cf9-8f95-cbf488639c2f", "showInput": true, "tags": [] }, "outputs": [ { "data": { "text/plain": [ "[(5, 10), (-1, 10)]" ] }, "execution_count": 25, "metadata": {}, "output_type": "execute_result" } ], "source": [ "ax_client.get_max_parallelism() # Max parallelism is now 10 for all stages of the optimization." ] } ], "metadata": { "fileHeader": "", "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.10.14" }, "papermill": { "default_parameters": {}, "duration": 239.098858, "end_time": "2024-07-23T19:43:44.328823", "environment_variables": {}, "exception": null, "input_path": "/tmp/tmp.DL1QmpHQMI/Ax-main/tutorials/tune_cnn_service.ipynb", "output_path": "/tmp/tmp.DL1QmpHQMI/Ax-main/tutorials/tune_cnn_service.ipynb", "parameters": {}, "start_time": "2024-07-23T19:39:45.229965", "version": "2.6.0" } }, "nbformat": 4, "nbformat_minor": 5 }