learn_shared_params.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "civil-newsletter",
   "metadata": {},
   "source": [
    "# Meta-learn optimal weight sharing in MLP\n",
    "\n",
    "Here is an idea to find “the optimal weight sharing” in a one linear layer:\n",
    "Let the weights of linear layer be w \\in R^n. Let v \\in R^k be the vector of all unique weights (k unique weights).\n",
    "during training we calculate w with w=Av, essentially taking linear combination of v_1,…, v_k.\n",
    "However we soft enforce (with a loss function) that the rows of A have low entropy (pick a single unique weight) and/or rows of A prefer v_1, v_2, etc. more than v_1000, v_1001, etc… (to use as few unique weights as possible)\n",
    "We train over the matrix A and the unique weights v.\n",
    "The resulting weight vector w will be in some sense the optimal weight sharing version of w.\n",
    "We can analyze which weights are learned to be shared by looking at rows of A.\n",
    "My hope is that it can recover CNN weight sharing or transformer weight sharing or some other human interpretable weight sharing that may give insight into how to engineer inductive biases in the future.\n",
    "Maybe it finds optimal inductive bias for images."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "compound-teens",
   "metadata": {},
   "outputs": [],
   "source": [
    "import mnist\n",
    "\n",
    "import torch\n",
    "from torch import nn\n",
    "import numpy as np\n",
    "\n",
    "from tqdm.notebook import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "agreed-delight",
   "metadata": {},
   "outputs": [],
   "source": [
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "stylish-dating",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/work2/08258/akumar01/maverick2/env/lib64/python3.6/site-packages/torchvision/datasets/mnist.py:498: UserWarning: The given NumPy array is not writeable, and PyTorch does not support non-writeable tensors. This means you can write to the underlying (supposedly non-writeable) NumPy array using the tensor. You may want to copy the array to protect its data or make it writeable before converting it to a tensor. This type of warning will be suppressed for the rest of this program. (Triggered internally at  ../torch/csrc/utils/tensor_numpy.cpp:189.)\n",
      "  return torch.from_numpy(parsed.astype(m[2], copy=False)).view(*s)\n"
     ]
    }
   ],
   "source": [
    "m = mnist.MNIST()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "finished-discrimination",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "given-banana",
   "metadata": {},
   "outputs": [],
   "source": [
    "A = torch.randn(28*28, 500)\n",
    "\n",
    "v = torch.randn(500)\n",
    "# w.shape, v.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "together-manufacturer",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([784])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Ap = A.softmax(dim=-1)\n",
    "w = Ap@v\n",
    "w.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "elect-titanium",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(5.7187)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.distributions.Categorical(probs=Ap).entropy().mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "impossible-proceeding",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "81840\n",
      "815320\n"
     ]
    }
   ],
   "source": [
    "class Net(nn.Module):\n",
    "    def __init__(self, hidden_sizes=[100], nonlin=torch.relu):\n",
    "        super().__init__()\n",
    "        hidden_sizes = [28*28] + hidden_sizes + [10]\n",
    "        self.hidden_sizes = hidden_sizes\n",
    "\n",
    "        \n",
    "        self.weights = [torch.randn(o, i+1)*0.01 for o, i in zip(hidden_sizes[1:], hidden_sizes[:-1])]\n",
    "        self.weights = nn.ParameterList([nn.Parameter(w) for w in self.weights])\n",
    "        \n",
    "#         for w in self.weights:\n",
    "#             print(w.shape)\n",
    "            \n",
    "        self.nonlin = nonlin\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = x.reshape(len(x), -1)\n",
    "        for wb in self.weights:\n",
    "            w, b = wb[:, :-1], wb[:, -1]\n",
    "            x = x@w.T+b\n",
    "            \n",
    "            x = self.nonlin(x)\n",
    "#             x = x+y if x.shape==y.shape else y\n",
    "        return x\n",
    "\n",
    "class SharedNet(nn.Module):\n",
    "    def __init__(self, hidden_sizes=[100], ks=[10, 10], nonlin=torch.relu):\n",
    "        super().__init__()\n",
    "        hidden_sizes = [28*28] + hidden_sizes + [10]\n",
    "        self.hidden_sizes = hidden_sizes\n",
    "        \n",
    "        self.As = [torch.randn(o*(i+1), k)*0.0001 for o, i, k in zip(hidden_sizes[1:], hidden_sizes[:-1], ks)]\n",
    "        self.vs = [torch.randn(k)*0.01 for k in ks]\n",
    "        \n",
    "        self.As = nn.ParameterList([nn.Parameter(A) for A in self.As])\n",
    "        self.vs = nn.ParameterList([nn.Parameter(v) for v in self.vs])\n",
    "        \n",
    "#         for w in self.weights:\n",
    "#             print(w.shape)\n",
    "            \n",
    "        self.nonlin = nonlin\n",
    "        \n",
    "    def forward(self, x):\n",
    "        x = x.reshape(len(x), -1)\n",
    "        for A, v, i, o in zip(self.As, self.vs, self.hidden_sizes[:-1], self.hidden_sizes[1:]):\n",
    "            A = (A/10.).softmax(dim=-1)\n",
    "            wb = A@v\n",
    "            wb = wb.reshape(o, i+1)\n",
    "            \n",
    "            w, b = wb[:, :-1], wb[:, -1]\n",
    "            x = x@w.T+b\n",
    "            \n",
    "            x = self.nonlin(x)\n",
    "#             x = x+y if x.shape==y.shape else y\n",
    "        return x\n",
    "    \n",
    "    \n",
    "net = Net([100,30])\n",
    "print(np.sum([p.numel() for p in net.parameters()]))\n",
    "net = SharedNet([100,30], [100, 100])\n",
    "net = SharedNet([100,30], [10, 10])\n",
    "print(np.sum([p.numel() for p in net.parameters()]))\n",
    "\n",
    "# net(torch.randn(100, 28*28)).shape\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "970f288d-3b08-484b-973f-7c5f8e6e0662",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(0.0005)"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net.As[0].data.detach().cpu().abs().max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5d7e8049-5104-42f3-861a-43b5d59726ab",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6cf6abc7-2bc8-4d64-8668-c8f7d7d829c3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "5f37cbe6-c7b4-4ee7-afab-136419cba9d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "net = net.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "atomic-photographer",
   "metadata": {},
   "outputs": [],
   "source": [
    "loss_function = torch.nn.CrossEntropyLoss()\n",
    "opt = torch.optim.Adam(net.parameters(), lr=1e-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "metropolitan-membrane",
   "metadata": {},
   "outputs": [],
   "source": [
    "def calc_ds_stats(net, loader):\n",
    "    losses = []\n",
    "    accs = []\n",
    "    for batch_idx, (X_batch, Y_batch) in enumerate(loader):\n",
    "        X_batch, Y_batch = X_batch.to(device), Y_batch.to(device)\n",
    "        Y_batch_pred = net(X_batch)\n",
    "        loss = loss_function(Y_batch_pred, Y_batch)\n",
    "        losses.append(loss.item())\n",
    "        Y_batch_pred_idx = Y_batch_pred.argmax(dim=-1)\n",
    "        acc = (Y_batch_pred_idx==Y_batch).sum().item()/len(Y_batch)\n",
    "        accs.append(acc)\n",
    "    return np.average(losses), np.average(accs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "crude-banner",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3.4011974930763245, 0.11083333333333333)"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "calc_ds_stats(net, m.loader_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "amended-testament",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ff45decd58c14c17a2b033377cedb001",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/10 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch  0\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c30fbed51f144b7ba383caec86738603",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/4 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.3395002563752872e-20 1.3025013956280418e-08 9.693863830761984e-06\n",
      "0.0 0.0 0.0\n",
      "0.0 0.0 0.0\n",
      "0.0 0.0 0.0\n",
      "(3.4011974930763245, 0.09783333333333333)\n",
      "Epoch  1\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b01b3c262d914233a0e8d2650d534dfc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/4 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0 0.0 0.0\n",
      "0.0 0.0 0.0\n",
      "0.0 0.0 0.0\n",
      "0.0 0.0 0.0\n",
      "(3.4011974930763245, 0.09833333333333333)\n",
      "Epoch  2\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "31da45f946964fe09da0c8ccccff220a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/4 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0 0.0 0.0\n",
      "0.0 0.0 0.0\n",
      "0.0 0.0 0.0\n",
      "0.0 0.0 0.0\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-63-26ce406c0ecb>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     18\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m \u001b[0;31m#     plt.plot(losses);plt.show()\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 20\u001b[0;31m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcalc_ds_stats\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnet\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloader_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-61-efcfae06fd31>\u001b[0m in \u001b[0;36mcalc_ds_stats\u001b[0;34m(net, loader)\u001b[0m\n\u001b[1;32m      2\u001b[0m     \u001b[0mlosses\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0maccs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m     \u001b[0;32mfor\u001b[0m \u001b[0mbatch_idx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mX_batch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY_batch\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloader\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m         \u001b[0mX_batch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY_batch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mX_batch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY_batch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m         \u001b[0mY_batch_pred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnet\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/work2/08258/akumar01/maverick2/env/lib64/python3.6/site-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    519\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sampler_iter\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    520\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 521\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    522\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_yielded\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    523\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_kind\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0m_DatasetKind\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIterable\u001b[0m \u001b[0;32mand\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/work2/08258/akumar01/maverick2/env/lib64/python3.6/site-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m_next_data\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    559\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_next_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    560\u001b[0m         \u001b[0mindex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# may raise StopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 561\u001b[0;31m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_fetcher\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# may raise StopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    562\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_pin_memory\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    563\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_utils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/work2/08258/akumar01/maverick2/env/lib64/python3.6/site-packages/torch/utils/data/_utils/fetch.py\u001b[0m in \u001b[0;36mfetch\u001b[0;34m(self, possibly_batched_index)\u001b[0m\n\u001b[1;32m     47\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mfetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     48\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mauto_collation\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 49\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0midx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     50\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     51\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/work2/08258/akumar01/maverick2/env/lib64/python3.6/site-packages/torch/utils/data/_utils/fetch.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m     47\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mfetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     48\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mauto_collation\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 49\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0midx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     50\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     51\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/work2/08258/akumar01/maverick2/env/lib64/python3.6/site-packages/torchvision/datasets/mnist.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, index)\u001b[0m\n\u001b[1;32m    132\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    133\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 134\u001b[0;31m             \u001b[0mimg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransform\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    135\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    136\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtarget_transform\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/work2/08258/akumar01/maverick2/env/lib64/python3.6/site-packages/torchvision/transforms/transforms.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, img)\u001b[0m\n\u001b[1;32m     58\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mimg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mt\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtransforms\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 60\u001b[0;31m             \u001b[0mimg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     61\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mimg\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/work2/08258/akumar01/maverick2/env/lib64/python3.6/site-packages/torchvision/transforms/transforms.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, pic)\u001b[0m\n\u001b[1;32m     95\u001b[0m             \u001b[0mTensor\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mConverted\u001b[0m \u001b[0mimage\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     96\u001b[0m         \"\"\"\n\u001b[0;32m---> 97\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto_tensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpic\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     98\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     99\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__repr__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/work2/08258/akumar01/maverick2/env/lib64/python3.6/site-packages/torchvision/transforms/functional.py\u001b[0m in \u001b[0;36mto_tensor\u001b[0;34m(pic)\u001b[0m\n\u001b[1;32m    127\u001b[0m     \u001b[0mmode_to_nptype\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m'I'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mint32\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'I;16'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mint16\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'F'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfloat32\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    128\u001b[0m     img = torch.from_numpy(\n\u001b[0;32m--> 129\u001b[0;31m         \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpic\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmode_to_nptype\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpic\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muint8\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    130\u001b[0m     )\n\u001b[1;32m    131\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/work2/08258/akumar01/maverick2/env/lib64/python3.6/site-packages/PIL/Image.py\u001b[0m in \u001b[0;36m__getattr__\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m    530\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_exif\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    531\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 532\u001b[0;31m     \u001b[0;32mdef\u001b[0m \u001b[0m__getattr__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    533\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"category\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    534\u001b[0m             warnings.warn(\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "losses = []\n",
    "for epoch_idx in tqdm(range(10)):\n",
    "    print('Epoch ', epoch_idx)\n",
    "    for batch_idx, (X_batch, Y_batch) in tqdm(enumerate(m.loader_test), total=len(m.loader_test)):\n",
    "        X_batch, Y_batch = X_batch.to(device), Y_batch.to(device)\n",
    "#         X_batch, Y_batch = X, Y\n",
    "        Y_batch_pred = net(X_batch)\n",
    "        loss = loss_function(Y_batch_pred, Y_batch)\n",
    "\n",
    "        opt.zero_grad()\n",
    "        loss.backward()\n",
    "        opt.step()\n",
    "\n",
    "        losses.append(loss.item())\n",
    "        \n",
    "        g = torch.cat([p.grad.flatten() for p in net.parameters()])\n",
    "        print(g.abs().min().item(), g.abs().mean().item(), g.abs().max().item())\n",
    "        \n",
    "#     plt.plot(losses);plt.show() \n",
    "    print(calc_ds_stats(net, m.loader_test))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "aerial-recruitment",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.5778536349534988, 0.8353333333333334)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "calc_ds_stats(net, m.loader_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "honey-opening",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "specified-sarah",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "f271c1a7-2465-48c6-8a8e-d1b7666a9bb7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.colorbar.Colorbar at 0x2b31563de668>"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUIAAAD7CAYAAAAfH52VAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABqmElEQVR4nO29e7As213f9/mt7p7Zj3PvuffqSiAkGclBxiWTMhjFxkVMOcgGYWOZKvMQUTC4wLLjB/hVBpyySQhOgeMy4MRxIguIoTCPEsTITxUGlLJTsYyEKEASBCEeeiFd6V7de85+zHT3+uWPtVb36p6emZ6Zntkze8/31JzZ04/1/K3f+r3WWqKqHHHEEUfcZZibLsARRxxxxE3jyAiPOOKIO48jIzziiCPuPI6M8IgjjrjzODLCI4444s7jyAiPOOKIO4+NGKGIvFpEfkVE3isi3zRUoY444ogjdglZN45QRBLg/wP+KPAB4GeBr1TVdw9XvCOOOOKI7SPd4N3fD7xXVd8HICI/DPxJYC4jTB491+z5j7kfoqACgOYGMwUE1LiPjKy7NzGIdY/jPxKlqf4dxL8PkCqYOn1ywRT+fuLTwf8eWYxRrDVgwUwEMwVNwcatE88XUl+T6LpNwYxLd0vjUkbJTA0mBynBlGATsJlPs51eBiSKGEVEURVUgVIwU3HVO3XlL0sBFcyVkExdWWzm2yaziEBzzhPIBbFdhcRdF9+EAjpSTGJJjCUR5XqSYa6lekYNMLLN9Kcufc0Uk1lsbpA8ahfj2l+iS5obpAQdW+6Np1xMR8jlHMUl7kefRkVW4bcvUqARKZvvhXQ0ceXEKMaoI7VJginAjoDEkiaWk7TgMs/Qq6TZX62ymMJ9bOLeDzStCkxNo581delTONqoyhvS9e0vRqs+FAERxZauTVWAVCk+9gzlg4tu4uuJL/yvzvXjT5e9nn3HL0zeoqqv3iS/fcAmjPBFwPuj3x8A/kD7IRF5PfB6gPTJ+7z4f/pvMUlgPoKqUHzojPP3G2wG5Snk50ryOy5QBf3Nc9JLIbnGMRAbEbc4IirOHCHbTLEpFM/LMSclWhjUCulTGScfE2wGxbmCgimEcqzYT73i5CTn8uEYvU44f1/GvfdbJo8brp+MKqIghf8zAzWKyaW6hsD1k5aTT32ACBSFcYzLOpoU4yi4/K1zTj8qjJ5VTp+2XN83XLxYUFPXz/g0L15k4X5OOi4YjQvyPKGYJtgHGfd+I6UcQfF7Ljg/m/Dg4oTyOuXeu0fc/42Sy+cbLj9ZyB+xmE++doPGutGlCmoF89SI9KGpR7MK4uuZTFydihOwIyV/yYSzRyY8dnbFYydXvOu9L+LeL4/QFFeOM8W+6BqTOIZtSyH50JjsoXD14oLzF1xw8dFzTt+fVhNSeaqUL74mGxUVo84/dM7oE4b85Vd8zst+nXd88CWYdz7iGFjoCs8kTOEYWzn2zETdb/WTiyhuggWKU0UTyB4I6ZW7b1P3jJRQnMP1S6aYccnp2RRrhfx9jzD6hHD1KSXmiQlP3L/gdz3+FL/40Rdy/YuPkUxd/4c2E1uX5eQp5exjlqsnDJef4vohfckFZZHAb52SXktFw9MnS+T+FJ4ac/oRgxrPfPH1Gyn5S6Zkpzm2NCiQZSVZVvDgmTOy3x5RjhV9IufD//3/0mvwLsLHny75T2/5Hb2eTV74q08uf2r/sXVniaq+QVVfqaqvTB49Hz799tx3CCsGhyxjQ7Rofks7n5kLq8MMkAZQS+8rPD+DuOobyUCbY6hm2QgDFUIB2/PfbcEmEuEHgZdEv1/srx0cjsut9xsq3Xxw7zGHroaqy7baRFFy7aca3xZswgh/Fni5iLwMxwBfC/zXg5RqE6xBHXKQo2wNDCA22ZsWveZgWxLZRtXdz6bqhdsk7fXB2qqxqhbAXwLeArwH+FFVfddQBTs4bDIQFw2YZSO8a6SuOwB3rN91arubMI8Nit8n303NAvMcaAvfuQFmqiil9vv0wbIwOxH5PBH5OREpRORLW/e+WkR+1X++Orr+2SLyiz7NfyiymTiziUSIqv5r4F+v+t5OJbBDUHu9Z7D73hqNtW6dhxx1y8qwjX65QQlMZTvzyKI0Z24N2H92oA7yYXb/iCjMTkTe3Aqz+y3ga4C/0Xr3CeBbgFfiqvsO/+4zwD8G/izwNhwPejXwb9Yt53FlyW3CAatibaxqt12JB3QJ0T3yG8ossO9zswIl2uvTA1WYnapOgRBmV+en+huq+gswo49/IfCTqvq0Z34/CbxaRF4IPKqq/1FdIPT3A1+ySZ1vhBHurXOig8731CTWjR21a8UQttg2+2i3XVk1bj0ue+Fa7geL9vr0QFeY3Yt6FmPeuy/yf6+TZic2Uo0PAgMMqF5qz7aM6ssy3pPBNZgndM3qbKsZQrp7yJe3BgXy/tLKkyLy9uj3G1T1DcOXaru4/YxwAGyd1yyyER4I9oMdbw+3vX4xtL/aC/AxVX3lgvubhNl9EPjDrXff6q+/eM00O7F/NsKhKa5Heosmv95McFsjZUjdfCCOXqmIi5JbVOxFDgD/3jbMJ5tWf5GNsDPtQ53cFMqenx6owuxEZIQLs3tzz5K8BfgCEXlcRB4HvgB4i6p+GHhORD7He4v/NPATK9czwv4xwiOOOOJG4VaW9PssTWtOmJ2IfKuIvAZARP4LEfkA8GXA/yEi7/LvPg38jzhm+rPAt/prAH8BeCPwXuDX2MBjDEfVGIgM8zel/2xiIwzoIzkO7flZO15xxccH6pdF1W/fC46N+PqmzpIq7dVSmSnT9iGUA4qzXWF2qvp3or9/lqaqGz/3vcD3dlx/O/AZQ5XxyAiXYFvxYYPhoNza/bGPXuNF2Hs6WQHOWXJgHbAh9o8RDt3+G6bXi7g3LfMmzpKbHH1rB27Pv7VRdTaw587Ld53yVDxkYFpeZ2XKWvnAoBLhIWD/GOGmOIRZeRUa24D4tzVuFjoN/L6J7scw+R3spgsHjH1dU74t3D5GeAj9t5XlZbOJbktYXGQr24bUMje7dlZ9s94CjdwWtRiOEuFBYi8IcNMy3EaaU+a3Sw+nxa5XH+0iSmnrqu1g+xEK5R0LKDl4RriUtnbnaDsiRrRd/lqv3+L23BpJDrnpwlE13g3mzo66I4N5oyz+jx2FgwzKNCsR6gYI9xaNlTbNtelz6Xzbw2s8WPjLlvtaEaaabDWPfcNeSoR3dTPMW4d9MFscsTJcQPVRNd4JGt7Fxo3dl6XCJuEgN1XuGzCSVmrTvoXPDJD/zKNRVVdeYrct7CCzo7PkDuI226P2EkO1947ngKFWluw7VIVSjxLh/mDXdq+e2c0U60al2BCzt+VR57Mxov0ZwsKlg4v5xL7sWdl3G65VVpZsTC47GBf2KBEesXXsySA/4pZhwPCZqd4t1rDftR2iY7cUPDvopLzIxth3Y9Y+Bdqk4D4bq9I/tGJe0X2M4UKBcZsCSZ+m2qKEve/z4NFZcsQMeqk8t2Bj1YPEsc23hvIYR3jLsMr0u8Ji/J2hL0HKEhFrQHTZCFfK+gCdJUtParhpOoHB1JTjypJ9wD4QFMxlioPQ2sY74tz+2XotZ8mAtDPIcrhtddMOuK69Y17j/avtvmzD1fHeYPynTccrpqs3KH7M8xgPXaJeNsIde+/72kaXPbbv05jbdMH0+vRBjwPexyLyI/7+20Tkpf7660Tk56OPFZHP9Pfe6tMM916wSZ13KhHuS0jEVrBi3XrzslWY3jbbd9XRO6R0NvQ2XFtop51uzDqP0w7oNc4HWmLX84D3rwWeUdVPE5HXAt8BfIWq/iDwgz6d/xz456r689F7r/M7VW+M/ZMIbxI7nKp3od0O7tluY0ft1Wd8b1rXW2FtGMpGqFCq6fXpgaUHvPvf/9T//SbgVf5Qphhf6d/dCg6DEa6yEcMhSp0bbTKx22WKg+xKsqC+KyW/brutkMetYJArQ7A9P/hzjaPP61uJ9TngvXrGH/b0LPC81jNfAfxQ69r3ebX4b3cwzpWwU9V4k6IedylegnjEHuJkcNexR32msMoSu2XnGm8MEfkDwKWq/lJ0+XWq+kEReQT4MeCrgO9fN4+9lQhViZaPrfDiJtxyEwljW1y6QyRpeI0XiMozt4YWbxa115ptspKZq18zrJVvCKjuvWyuj/p+QOLlgM6SPge8V8+ISArcBz4e3X8tLWlQVT/ovx8A/wyngq+NvWWEfSCb7l14xBE7xGCrVbZM9IpUK4iWfXqgzwHvbwa+2v/9pcBPqzrXqogY4MuJ7IMikorIk/7vDPhi4JfYAPsXR+ghAtpDRNuLHap3vLKkM3xmhwHVvbHttt9i+kF6672aMPIaV90zb8v+rot71HcK5AOtNVbVQkTCAe8J8L3hgHfg7ar6ZuB7gB8QkfcCT+OYZcDnAe9X1fdF18bAWzwTTIB/B/yTTcq5t4xwa+igwr0O6zmKvEfsHDs/4P0a+LI5774V+JzWtQvgswcrIHeBEXYF3c7Yg3ZVmC1j0yMHdgVvPzwEFr/RPLTv/TAHbhPag7aarYz9Y4Rr7B588BiiHtvefWbHWFVK32lA8x3AcYfqm0bc/luOWTsIrLLpQvXOdoqyS6wqpe+CCS7akPY2MWFVuXMS4dLaishLRORnROTdIvIuEfkGf/0JEflJEflV//34kAW7cbtdn/xvuox3HevOk1124hXSWrZD91prjVfaJWm70ppzliS9PrcFfdh+Afx1VX0Fzmj5F0XkFcA3AT+lqi8Hfsr/vlvYh+2kVjnOc8/U4q7S7KSIG+axLGzk0L3GIEMusTsILK2Jqn5YVX/O//0AeA9uSUy8PvCfAl/SN9OFao+PDRRh9ztUy5y/bxK3SecK8DtUbyXddbCgr2OeN48BqvSch/ZsIpoH5ywZLI7wILCSjdBvj/NZwNuAT1LVD/tbvw180px3Xg+8HiB98j7g1N5FzPAWte8Rh4CusMyOlSXzVOLeq08OaFK7axuz9q6tiNzDren7K6r6XHzPR4F39rKqvkFVX6mqr0wePe+RUd8S9cQh0N4mdb7Jc42XYd5jvaSnHulvuK9j49VtNOO83bL6vn9DtDvwypKDQC+J0Edw/xjwg6r64/7yR0Tkhar6YRF5IfDRbRVyZzgEpnlIOLbnweKuHd7Ux2ssuCUw71HVfxDditcHfjXwE4OU6Dh47jT2RXvsWmI31A7Vvau4qsA11MasCrk1vT63BX0kws/FbXHziyLy8/7a3wK+HfhREfla4DdxC6OPuCncBAe5PZrRERGcanx7mFwfLGWEqvofmE/yr1on04Ve46EDqnc1WNfcPqp6d4hyttLYlglnWRzdUvR4/aaXPa66DVfj3ZuSagfs8OPKkiNuLwZYYrcxEzyiG3vUrCF85i7hRhjhsvCZCocy6FalmW1Vq72ZxKobs/ZglA1v4RY2st3J+OvwNu9y3HdmtUp7bH1cHFXjI/YNq2yId4O4C/LDymuND2Qe74K9Ez1a48gIbwuGkBI2YKYHPOa3V/hVmnNOGW5CKXJe49uzjrgPDkP+3Wgt7mClGC6PlZb97R+LGcRZcgMDf1Ha7XurLocbTCDfg+4eOqB6gwPeXyoiV9Eh7v979M5ni8gv+nf+4aan2B0GI9xo5cXyR258p5sjKmxTw1+Udvte13K4RQN/YwYure8bxgrHeS5EdMD7FwGvAL7Sb9oSozrgHfhO3AHvAb+mqp/pP38+uv6PgT8LvNx/Xr12ZdkXRjiPiPbMc7XXh0X13Zh1XQzpENpCt25EKjdMZp1n0KyDoQKqGXTThaEOeK/gV7I9qqr/0S/v/X5W2PSlC/vBCOdhz7hO311GlifU+r1JmruaLParK2awZ6SyEmSoPhyQFqyaXh+2f8D7y0TknSLyf4vIH4qe/8CSNFfCfjhLbjx4dv69G98Cfh3invfKQAOllySwZlY7aeuOgPZNbYQ3gW2VUFUo9uOA9w8Dv0NVPy4inw38cxH5PdvIaD8Y4RHDYu34vsM502QjrNk+dymYfMCA6lUOeP9AfMC7V3snAKr6DhH5NeB3+edfvCTNlbDfqvERu8UeMME9KAKwurNkFQzFTrcZl78PB7yLyPO9swUR+Z04p8j7/D6oz4nI53hb4p9mw01fjhLhTWBPBvtKOMQy31bsYLYYjOlvdsD75wHfKiI5YIE/r6pP+3t/Afg/gVPg3/jP2jgywiU4GG0oLujcDVE3UH177NR8MFizCYaq98YsZl45BvMaD7vp6roHvKvqj+H2Qe1K8+3AZwxVxtvPCA98zC7FPjOlXUuRN9gUu3Cq7XKr/+MSu9uGXfTnPmy6sI+n2M2ND53/ykZjfcvVu+kdWSpP9raP81QobtGmq32w34xwDyznNx4+c0exjdU+y47Z7CzHzZPgjeCmmf6usd+McJCNBDZP4iAQt9Wu6rzFsbKNjVmrsd1r2aV7qPcJdbeIzoa2ER4C9psR7gF6EfhQO0z3xKKVCA1+eEC0LKKoykGV+cYwjygHdXDcrY44MsIDxKK1qTH97lJKWWnYdDy8qgR2Ezh4b/kKODpLbhv20Vky5DZcq8zcQ68cWVS0feYZHWWb1ywbNdecZXuzG4frXrEd1aON8Ih9w4HsUH1bMaQQWB0ItWlCW+9roTx6jW8ZNjldbpU8booPtUbqLrS3Lmlh6GzX8hoveWeVthkkZq8nTQy2+8yAONoIjzjiDuIOmf+W4niK3W3EAOdGDJb+NhAI1o/krdHvNu2gXa+v8/6SdxaFz7TbrUsiWsQc+sSbDiZlbZtr693btf32M8INcesDqhtrlA93G66b7CcVz1vXXca9Z84SOHqN7zbW6fsw+LYteS5Nc80ldp6DLGUk/p4RrcNI9mjL/W0xwZXKs8xGeSAzqh6dJdvFXRO37zRuYqfpvjggOuxkxDuQ2u/aWN0pI9zGsqkjhsHgwoqwHfvpEGmuGHq5bazsNd5BoY5e4yO2jyFp7CbVrQ13iqmsCj2cHMuabFMbYR+zwE1jV6q16t1jhHfLEHDExthGWMXaatiAfGFfx31j7fgOC7knB7z/URF5hz/I/R0i8vnRO2/1aYbD31+wSX33w0a4JzPu2pgJvbiZYlQYcgnfFrHWsN4geH3tbbha+RlRJ9H2kFQPFUPRcHTA+x/FHbv5syLyZlV9d/RYdcC7iLwWd8D7VwAfA/6Eqn5IRD4Dt91/fGzn6/xO1RvjxiTCXrPbOjPgsg7sWme6yY4tWxoJ21oSPI/AG3t+tvNuXaskgUVlbHnTu/Jda6zJnL/b17wnfGkaHb/XZgK+neauW1aZrXPgzutuKLOFzSoUwVrT69MDax/wrqrvVNUP+evvAk5FZDxAFWewc0aoKvXic5Wa6GIiEp25Xv0N9UHrbcJT/4h6A5So/8w+V//tCSk8S/1s/HyTm9TvafvZRj2jMkCzTtR1CHVu5yta22s63/V/2HB/pozROIvTaD2r7faJEV2TOHSm9W6VR1zPLixhFnHdYtWrwaA7aKLxd5R+aE9ptUd3xlTt0pioo/7urE/Xtfj6nL6ZSbLVJw0a97QgUcyhejoX69PTeQVaHdrzw/YPeA/4U8DPqeokuvZ9Xi3+2/40u7WxP86SZTPaUBLSwJrgvsUgb0PT3af6bQuD7AG8QTupyv6o2as5S7Z5wDsA/lD37wC+ILr8OlX9oIg8gjvg6auA7183j94SoYgkIvJOEfmX/vfLvGHzvd7QOeqfllbf0p45oZuioilI2rNr33y77EtzRsDCgdFKIxYmG9fn0VKrHusMwvgd027Hee90SVFRmfocjrbMQN5r+CyTzJaUYVG6y57vO75nPLRzE+x4pC0BrpNfK32ROfQUSY6DhqetIBIuwSoHvBMf8O5/vxj4v4A/raq/VhVP9YP++wHwz3Aq+NpYRTX+BuA90e/vAL5TVT8NeAZn8NwYM/SwTJUZCEN75BqE3dceuml9FmQzO7A3zCskswUJNKDXRqir5L8lkWtpMbvsrj3eW+bc2WY4TTBhLfv0wCYHvD8G/Cvgm1T1/wkPi0gqIk/6vzPgi4Ff2qS+vRih58p/HHij/y3A5+MMm+AMnV/SL61+ndcpKcb3u2biwEuCXbAy4LXSkDlEWBmWZtPtK0nOll26r1f3Z5MNH5UgCUQSX1/JKLLFLSPXmbacI3HPJfxKagmSfkceS8of08Xcpu0rcUX9F7flTDrt3/Mkq3nmwXX50BzG2LDHzhMQOx0ua5ZjDhSwVnp9lqblbH7hgPf3AD8aDngXkdf4x74HeJ4/4P2vASHE5i8Bnwb8nVaYzBh4i4j8AvDzOInyn2xS5742wu8C/ibwiP/9POATvpLQbQAFwBtPXw+QPnm/O/V57XkDoR19NaIKXer2MgxZrYHaaJWA4kpSW2Y+WMKwKk193YHcOZkteHyNfNomiKX5LJPy5hVq3TaY2XVoCGMn6zXWvOTWP+D924Bvm5PsZw9WQHpIhCLyxcBHVfUd62Sgqm9Q1Veq6iuTR8/nPNT1HodhpV+niFus1rIm29SO1GAGG6a162lunTljZyS4+zl/IYKXetnntqCPRPi5wGtE5I8BJ8CjwHcDj4lI6qXCLgPoQWKbhL8zwhm6DmtLK/0fbavdcVvtWjHYqJ82afs1nCxVtjowL71FTK4PlkqEqvrNqvpiVX0pztD506r6OuBncIZNcIbOn9haKbcM6WGbGTqfZegdDDzPw74puuyyqzTOioO6nfYyj3v13E0N2DUZXq/irlKnrczc/Rwlt2k98iYB1d8I/DVv4HwezuC5EItm2q0T9BwHwNaxacVWfH/l7KLA8L5YZ73xPCbamIQ2GFd9693nuXnlCPVeWP++7b9meTvbcRv0rD0/twQrBVSr6luBt/q/38eKsTuLCH2nk8sBTGR16MQBFHbLWGtN70DtttSDvXbC673WKYUNTSIK2sMjfJuwPytLdoXgzRwKy7zGQzOyJWVffa10/8bQ9o4jQzgotfvvbaBrKeQqWBTXGIcdLWrSw2Evh1PSIXD7GGFf29oRDtVC3BaWxHEuRau9V7Un3bRHcufb6sucv5e+1w6fGQi3SO3tgxtjhCJaD445Abg6VEzUQM/2CZ/bOTZZp9dOZlXb1ybxf1q/Hm8W0MmTe8QGzuPnmyJukk33YlzKXOcFWt8E9orIt4/bJxG2sS+EdUQvrMRrjn27HRxKDO+AuH0bs/ZZCTIPS2xIy7apuhHEItE6iFQr6dwbat1yLcqzR7FiU2TsLJHW9wpprxT909EOi2yEcRnnrkRcw9u8tDu0lqQHHUZHifAWYp9UDtiMYrdcj7ke2k1XkbTrvO3+WGd/tGD6WNI/YYfq7nyXZ9F4pM+yvWXYBtM6eo13g2XG82VL7Ba+Pufm/Jl6cVKr2p7m7gIdrvegsY0EvEWCR5+6tN9fVpYZx0h0vacQ1LAZL0PPx3oHpfdE20a4NKznRqSqLYcN3VLcComwk+CHDjPZEWHE5QrqzlYi+NesT+OA9wNAr6Zb5iHvQUvr9NBam0CsMmGsi1sWLN0HO2eEQwWExppPp32kK5s9Vkm70AjZqwKsB0p8A6loOyfZrZDmstjNCCv3+Zq0uA7W0t534sTYMODyALEf4TOd9zcw2O6pxLLuErKVY9p2Xf2BnWA3NgZ75hsmgl4bss7DCsx8KRrpDNT5+zmEtoadH9600qAeiqF1iPp9tqVfF0OsmZ1dY9rv/UG24dqwDfqWda4Jr0/+PZ7p3Ze7jFoI6HIxb0A3g08etuenB9Y919jf+2Z//VdE5Av7prkqdsoIBz1TYV3MCd4GOom418ash4yWdNO9czdVPXvbCIOjJKTbbvclE9Hcbbjm/d2+1sEZdEE/N8rQstPOO+Ol17V22ivf6MqnXcCBERyVfT5LEJ1r/EXAK4CvFJFXtB6rzjUGvhN3DAj+udcCvwd4NfC/+bOT+qS5Em7sXOOtYR+Y7S3GSvbBA5okNpWo9tQaszbiYw4WfXpg7XON/fUfVtWJqv468F6fXp80V8LtY4RtbEigSwdIkHo2yWdh8PEy8WXAETikfrWJ/YyeEuQWBaNgwtloe/8Z88bqhbwxBqs9P8uxybnG897tk+ZKuBXhMwuxbQlxXySBXurqGm7KfUGkZh+xV3hSRN4e/X6Dqr7hxkqzJm4/I+yBnW8Y207zwHhTpR4vlGSXJLJCu64bpzcE5h7edJPYQTlWyGLZAe+rnGv8gda5xoveXZbmSrj9qvERtxd7wpfm4sAmuAqKW2LX57Mca59r7K+/1nuVXwa8HPhPPdNcCfsnEd4AcW/kzV7n3S7716aDpo/KO5BavDeSUYSVlkLeMIPav9brwFCRa6qFiIRzjRPge8O5xsDbVfXNuGM+fsAf+/E0jrHhn/tR4N1AAfxFVS0ButLcpJz7xwhvEvs4g+/SpnfINsQN0YeJbmNFzb5iUB/cmuca+3t/F/i7fdLcBPvHCA+N1taR5gYN4B3S07s/XuObPM6zUY6bpMebFB0PQmwdDvvHCOdgD7WxGkMul2pjryu+JrTxNQhmmmmH0m2ILd5ZV+2iXreQ7BbhYBjh2p7DVTr0pjr/0KTgPlhUJx8KM2S1ZxjRhsxilc0NtnbS3Q1hhWDpW4ODYYR7iyGcJbvC0FLSMMtkG6h2FNpmG3WEL81rliHPS46y238cN2Y9YiWEAbAtulm6i8KKAXlDYgtSw41IIvsq/Wy0K8WGWe9rm2wJR0Y4FDZdYrcpj9o25fryzQudmbm6qDg3xEAXnjd8Gwb+oK7e4ZI6BNx+Rni3JPwjFmBTgXgf4ye3gqON8IidYMgldpueYtc7nxWfX+Ys2Xb+i7DKBjoDBds3dhvfIMmF2GX40y3DfjPCOxTAuhc4tIDqhcx2s7qss1PMbYL03HT1tmC/1xofEjGuMuaG5DUDtVFfnjHI6ooeRd74+M9VytnDa7zvJtwjNsPeHuc5XEYrPHtTwtA2A7J7YpWB2osZHgf+YeOO9d/eHt40XEYL7oUVDnM20oQtrRjYxjZcHYWcad5l6uKWYgz3Hoc26DcNelya/t2TYPfbRnjEehiaCcmcNNfNZ2sxl+u/2h74g0zSh8xMDrnsa+BuM8Ieqxh2chLa0KrxtiXYLWKllSX7vsHtIa8/PzLCO4x1l8sNGVpykxjSa7zjgbTTTQ/2FQP1nXD3vMZHRhhjnYG07cHXd3R3DIKVGcNC+2HPa1XmK+a9IbbNBG8ojPFmcAdthL3CZ0TkMRF5k4j8soi8R0T+oIg8ISI/KSK/6r8f33Zhe+EQO3DvR0YTg6ywGMK+uOOtGBd5y3cafrmrbbj6fDZAXx4iIl/tn/lVEflqf+1MRP6V50nvEpFvj57/GhF5SkR+3n++bllZ+sYRfjfwb1X1dwO/F3gP8E3AT6nqy4Gf8r9vHgfGVA4Ru4olHPS9JbhrEtBS7IAR0oOHiMgTwLcAfwB3nvG3RAzz73ue9FnA54rIF0Wv/oiqfqb/vHFZQZYyQhG5D3we7lwBVHWqqp+geSjzPwW+ZFlaN4INO2srk+9KeyQu28p5mAKqsN/cYIlqvgshqa8kvK2yVKtddrT7zEAHvC9CHx7yhcBPqurTqvoM8JPAq1X1UlV/BhxPAn4Od5rdWugjEb4MeAr4PhF5p4i8UUTOgU9S1Q/7Z34b+KSul0Xk9SLydhF5e/ncxbrlvDHsM29oYMOCirLfy+v2uGi7ws4WIcAqEuGTYXz7z+tXyKUPD1l6mLuIPAb8CZxUGfCnROQXvEkvPvqzE32cJSnw+4C/rKpvE5HvpiXCqqrKnMWZ/rDnNwCc/GcvOhS2sjr2YaDuQxn6YAEVBCpadNb0ovfWRZvHbLLWeNmrGw+CHWywsYLXeOG5xiLy74BP7rj13zWyXMBDFsGfg/xDwD9U1ff5y/8C+CFVnYjIn8NJm5+/KJ0+jPADwAdU9W3+95twjPAjIvJCVf2wiLwQ+OiqlThU3LTgJDdcgDuzHdVdxlCLVFT/yLx7ItKHh3wQ+MPR7xcDb41+vwH4VVX9rijPj0f33wj8vWXlXKoaq+pvA+8XkU/3l16FO2c0PpT5q4GfWJbW2tizcdfJBzYNqF7l8a4CqMyksy1+OYizZNnG29vc5WAL9LTTuWFeZgMWYkc2wj485C3AF4jI495J8gX+GiLybcB94K80yu6YasBrcM7dhegbR/iXgR/0p8q/D/gzOCb6oyLytcBvAl/eM63BcBcPmdkrbHhmSed8Mi+hLY/9vaSjmyzTbvL+djp4iIi8Evjzqvp1qvq0iPyPwM/6d77VX3sxTr3+ZeDnxM2c/6v3EH+9iLwGdyj808DXLCtIL0aoqj8PdNkBXtXn/eUZzPk7YAjJZuXg4gHyHALLRui2RrAyt90r1Tj22i5Jbsb8o3XyIlvYhGOpt324rAbNY5iwlM2wozJ4FXaGh6jq24Gvi35/L/C9rWc+wBwKVdVvBr55lbLszcqSfZqRZ8bsFpZvNdIbcPeZON1By7y1kJAVntX5/K1vH808s0IbDWISWIQlAkF7MtnaUmb2azzuAjtlhKp1CEB79q9+Sv3solk93qG+eqxy63t7mahT4IXaGuqfaWydHjrdqHvVv1NtAGCZJczq5uJ6xhRV5SOz9XV6vjTr4/NRlca7cxuC1jPedFAJd3E61bMaVUVRxDGnjhUcYSDaqO4aPevqrL7v/HV/z9pWe/ku6vIOh+RDPiqOYc62WQeJhDzF1aXRDbb1XPwNNW3EdCrM9L0RrfKumjr8HbdJq2yqguJJsWqfWek6frdJp9KQnG0gltDP2pHpmjgywh2gHjCtgVkRkTautT+NARFfCzbD+PVAK9Iiak/MgdFIxBDUM6RKBIltkSGfUJcOYq//ntMArXrMML9o8FoNDKPJ6OKfNrSjtuoZtYVGz7gLUfmlviQoKvPzWlSfKtloEqvq2FX+eNKLKx0YbnVtThnicd/RphLr9yp1h7QYTVXdatKRaCKr783kHcqGpzNa9YompMBgq0la3YQbt0+DJ1eTj89Dm8UP78Q03+jfTXFkhNuFtYKIgIkIg5oo1DQZVLhWMydPCMa/EyQ+0XrWt4Ja96AYiyaKGv/bSiUFKaDWSysCkiiagk1BU5e2WHXvQDWAGjFWsd9d6gFvS3fDJO5hVVBrXN5GfNn9d+IlDOOrHqfv62KtY2SVtCZ1fq4eUh3KrcZ9i3X1tVactBvqoe55Fd+OEQ+K86ja19BQy2zUZzT6zKcbmG4op4WyNNXgb79jrWCMl3RCvgqFNc06e6hpfSf136J1mbUSv3xbzNCau14xJHXtbW1NexVTs3Xfimidh6fHkH9Fl4lUNBc4pFqDWpDS55tqU1MJZTO+fTxRVWl7/d9JhZ5uS3HplTLsoexHRrhFqKClYMUgqnVbiyfmpGZAYYBj1BOVuy9+gIXfmoBNndhkcs/dLGhpMKMSY5QyU+zIZ1UCRtDMEaGWhrJIEKNIZinHSnkC5ciljfp3AJu5vE3uroeyEtUjMKygCiapIgJFnrhJgHigaFUH1wZh1If2AgpBE0NpEpeFZ0iRVoS1Bi3doACfHtGAs+FBqScf68ubRH0QeJP1zFrcYK0mG1+kBsMK/RYxWrVxWRRRocgTxEr9bPSOtYZKb5W6/Ndl5toslDE0i29z65vLlSGI6YImik08D/ZtIKXvM/ETXcTIsJ7xWEefRVFnGMqiVrAKRiyJcW1iU8WEto2ed/lLlY94GrKlYAuDKRzz0qiuWpqINhxtAVB6ZunbKUlLksRirVAUBkrB5OLGQNksy9qINaA7gt2rxkEtbvxmRv2Nb7fVDZHWPer7TrOSWgMOKrCfmYPNrK2OOiE02H5kphxxHjP2nLh6sfrjyyotO1BcZpeXNtWqOE0vXWl8OaZSlZk8G3axoDbRei6UvdV+DfW6zmKhU6NWAYNY6BKYUdPjsld90pGwvxTbCBttEpc5+jRsiSFfqVuu7aCqpLm4X5VGmZp1m1MGrW9rq0xVuvgJJG5/oj81KmqrXxrF9hJhVQGN3h2SeR0Z4XahVpx6YJr2wdiOV9FKoNRK/W09X6lZngEGp4Z1HxEQo36GVkxZS0JBnQ6GcZOUjqaDhBMNLLGz+ZsSypRahYlGgmqkmleEi7dFUqlPQM24wnUiVTwamOrVW2PsLIP2z1YrTsJXGSQhP1qD9OIbuGpDoLKHWpC4bAYwWqlk0LLdmah/vGpM7FwK6mtbNY6Zbfwj+rN2ykSSctVeCt4EoN7JVTHB0L62/m1yLx37MlfliLu1ITVHk2ik8idBNZVmXcT453xZqkmu6huXNtZLqIGmQltHam2jbRpt6y8ZpSgEq4KU4tKyRC9tjuPGrNtEUFu9nQ7PhCQaNNXAC5JQZM9pE3s8ENXWhE6wl4mSJNar1+4ZY6lUIvUqh1qQ1BNYQ91zxXLqdDRWg6qFf8ZLHio0bHZi3KAxxkvAlQ0vInahthGWXnVO6hlCvLqmVrx0SVMKUqcaYwWqMkmljlWqcWiXttQTDVyxTU9rNSmZejzWBWsxNROlG7W/ei+CllLb70L/eTteY0L0aqcIlL6hqrSp2xkc86nyNmHy1IqGHJ34RlKnjpLQnEBtTUviaUFLP4mF/vW0G2yERtTRqDcfiInqEMoSNVjlzAjtEjFCNVqNi9iT3MjXUE2wInUcZ0hPLPUkP5Akd1SNtwkVZJI4ZmGiwY6zv5UjxZ5YNFOst3foyFKKYIqaExl1dp7yRN17Y8VMhWTiCCy/EsrEkCSWR8+umU4yiokhfWgYPSuUgB2rsxPmBmuF0b0J904m/PYjp+T3UopzxZ5aTGFIL0FTmN53gyuZCMm1UpxJZXt0cRFKWXimdJ2giZLet4zSksuHY3RqUOPLPRI0EcoR2FN1ajmmImzAzfbXYNU5DexJyclpSZ7Zyv6kVigLg0wMybXjBuXItUN6qZRjgakTmaUwNSMESJ39NHZqOGeSVHZZe6LYsSVLS9LEcdrCO31sBnaklCcKiULhRq8UblBq4tpZCoGHKWLd884G5iR1SsFiEKNugCeOBgzwcDpyUk/mytpW4623Gbi0qJwGmirlqfV1cbbj8TOQXCvT+67PbHjHT2pqwEwEWwoqScUobebolqmhVGFkCkZJiT3Ryr4YM/iQrs28jVAEkzuayScGyYX0ykmoNnF9jwWZGOfYSnHlO7EVDagApaBTQ/pIySMnE64nGfYqJbsSsoeuHuY6mq03GqcMxlAPBTciEVbqqe+0ODTFSUdaSxexqhMNhFhKDPeCBBS8xyJKaizGWOcUCM/Ekol/1ogyTspaeoy8uWKdWllLTk41hjCjS81M/Mwv1jF7I0piPGcLEqHUdaikgiCh4AdYaJtSnFfAv2uM9aoqDam6kjKo28n49nBEHUkMwfaW2Fo186qmopXXOS6bMRo00VrFD9JZcFSE/CM1LUhzpqByDlRtG3tDA0x9vbSmSqMRTF3ZXCMJ2+cTvMBOgvdOH18mU/j+ids/8tO4SUhrCTrQFl5a9PklxqJGEe8VrppU6rJUppUgjfpJRiovb00/4uml0iygNilUHixfPlEyPyHVHmOtVeShcGSEW4REA8BQ6xPWzZqa+hmwsgm62ZIyEJJPxxO8KZz6Zf0Ar7yvBqcCqVBY4yUmQQoq+59Yn71RSBWrwnWRIhMnAVpfFpeuOC9k4M/G3Qf/DFG5jNP1NPFSEt7WFRiG1lKIKZy0JEEV86E/Unq+EryhiXtXQohJrN4KiLGVyifWSxshDChuSxPedeUSFdT6PvAqrFjBFOIYl3XSnWb+eVGSiLGLZxyU4tox8+0Q+tb3m82opL/gcQ9MV1JFUkuSWIxRcqgmsZO0wIi6csQD09veAk3YTLCila2s/pZq4sP3mVgwU2fnFKNIAclUsJk6LcHgpU/3UqA59WYOq4bSmiptZ3t0Xls1eOaomALSiZInEbNOFFIqM41E9KOJY5hm6ibQssRP3J7ZZl4dx00QIgqZxSYJNq091EMgkMtdwo6dJYEJekLzko+ok15ssKFoiJUSJNgQPbNrdFBM+GGmr+xWzhFSWoOWhqSQSlWCmBG68qiKI7DCDVZTRBJUJO0JPv1KPPLphTJJq37hckRdYQBLNGiDDU9ag76KKTPa8NxWK0ZEvd00sp2F9jBhUgkSp2fEDW9jkGq8kya0sw0Sjf8tTrqt7FPV+5EUVpU5dvg4Dq+p+vo60U2tOObuGWCS2Nq77t/NkrKuqK34X20uDPFziWeC6m2sQcIKqr6GdpSqblLV0zPLRJx5wrd12+Mb96UN7eclvJBm6JfKIVLW16ubJproQ/q1AB6VLZrwqnfd5F5WTjdtxtMOxAgBN8HdIexcIsQoknhiU3BUClKA5F5liDyuTipxxuCKeWitmiLiYsXUSZSKjys0oNYwLRJ0YjATr54FY3kY9IliUheXdZ2nJBMhvVSKU3HSmnrJKqmZg03qlmsMAC8RikCZuUFuVSi9SkPigr5NIb5OfrAU4oKWfT2kcN82qyVLySJG4QdJiKIwYUCkLmbP5EqZiWsPL01W6pU6Scy1uVQ2KAmTUhjAhecHhWuHsPpGREnFVnUPap4acZ4oP1OoKCZyUMhJidqkZlhGIREkUdKsrBmhf0dEOU1zjHESv1T9XTNEM5WqT62Jyo5v04jpaeJsb2Ihmbp0nNQf0YW3XUoWxEDfv55BCo4JKqHtfF9ZvN3bS3b4/i20mnzVAJmiWDRN0BLHML3EHRxkpgBbCKbwnvAgbQuVRpCXiZuYMusk2UyiWMoBGFjMgO8Idh9H6KUA8Z7eoOqZIgwsp65J8ByHWTeaxWMJQSI7WO2FxUl5QFEapDAkE6kll/i9xGIS9YHQCWYqpNdKMiVSjak8dxJ+B0mwjOqm4lQ58HX08YGBiQSJLDDAUjHevoNq5ek1fhCDVmqxSa1bJaPRUjl1bWCMhcQPpmiSqIK1g1QYPKiE9z2jEqhV47p8EBiJ4zxBIgwMq6F+Juol19A/dXqaKOmopLhOKg986GdjnFqcmoj5lE4UPUkcI9TSe0U9bJhQckc3pvABxUFD8P1btbX1k1egs8Kp61XZC5xa76VB4xmKEkl0SS0Nl9ZU9BjssBaqyQxx1wNNB2lcEutWlxgqU4uUoCFvYtqQik4V8QKE9VqOo6cksRSZWw3lJr3huNdRNd42wvI3IntXmDEjcT+EzwTnRRWqEMaD1CpB43oYfH4VhwlxhCPF5FINkEqFtnVYQpJY8lSdJzcQlkgdluPVT4k8u80ldtpc8kbgMaGukXQQhXBU4TNetW/Yerzd0IV0+DSjNoAQPkNtb5L63Ur1NkHtjfVrKpsp4qQXDXWKVK7Yw299GEesbteOJ6kdO0Hl886IMndlrJazRf1srWAjB01gCoV1EQZOJdS6bsH5nfrmaTheQptS5RerqlDXWUMIjLcth7AftVEjV21Zt5t4NbUKqYpUU219QplEqcNnAo2GSTtUTKP0AlMLbR2EAgnzmYsjJHK+SFTGjXFkhFuE4sIAIvtRJaCYSD0x0Q4mpr4WdmeRmAkGlSCodVpLA4CTNDJLOUoaqrUGCca6kAljLImxTDIox+IN/NFAihhVkDSAahke4BiD+oFUSoOWYuZYDxS3HlWDqSD3TNQvGQwSm5aClsYtVxOtmZAX8NRLYI3YRmq1sB1I3bANBrVZcDYykZo5hsGc4B0FzYFW90Foy9AIblAGj76Ugk6SyhGmUZ/awAi91IxoxRymfv1f2wlQLbFLcZpFFdguEc34uhj3TBUmk9T5t9erS+i30rmpA53hGY71En8iWq1frxhuFHvqJgepmb5vD/x64KpfIoZXO02i8nnKCR7xQD/BWWULE0mQ+GWNwzDDuyYRDmhe7YlWCzcChANmOMj8pNodpq204sEb/gzE2XhPa8bVMGbTKl8Yb9KdTttJ0pueltW5clIIaAexx2WMpeaqfZeUZCa/5r14Vxa7aLDFAmekwnc1hPi6zOz40qd8VQZzntHWt7Q+rXvabrs+eemCR5wqEE0y7fRCOkGKn1OXjnyCVF4f79lNEhuh3XfzPhtgkwPe/fW3isivRAe5v8BfH4vIj4jIe0XkbSLy0mVl2S0jbA/WqPMb6lzcq4HZtXo6phuJNzMIs7AXDioPn8+vocbFTFLdovagroQQm0rVajPGuOW6CCMwXQ2bCsyi6eX1klSrkhq1kxJJgFEbxGuvqweDNNPBHCvpz0uSdXxhVGffPIGZVapYs3jN/pFW+qEsoXCN50O68Uei5zyDWDYIZTb9SpoO7RnqE9FGXDbtKjd05hOcJXXnNWlDonsa90E0QTbiHucxaqUyZcT9bK1zvsUrWWpTy0BinNbaxLLPhtj0gHeA10UHuYfDn74WeEZVPw34TuA7lhVk9xIh9cB1P3p0Xsz0+kiLMR+NB1frXvC6hm2NNBqks7N7xBRi4m8xpbh+MxsVhPCZDsmlLT3VA0Tr/Kt35kz/XVJHPOFEEkQVshGYTsdkE5cxXgZnokaXHn0Q5x0Y01xpIrpn2u3VfqfFXKuqake52tJgV9Zxey14zgbm1KbLLtohopd4UmtLpfGfS4bEzF6JcfoDGPeqJujx2RB/kjUPeF8h3TcBrxJZvBf67hlhiImrVkc0JY8qviuaGSvnRAezqBec05pVHcey8dpOba4Oce+oXyvqJLfKExitUqmILAyAMNMTzYwRjwnhFlV8ojqJVYLTIbJViqoPGfLFDp7YYEf1cY7h3WoQRFKC+PwqW1ck6VT7/sWrUUJBNSpLZLuajYnrUF1VaonF1u3SjBGtJ5tqiVyod+zhjUlU6/yNt9PVNCDzJZM2I4odRWHySuo8YmbZnnQkau84fKZxjGmIFIrTictiW8zYp9tYJRXVtSGlaqtuIf41ooFqQ5HQ7wPHEbZE9fmfmz/g/fu8Wvy3I2ZXvaOqBfAs8LxFBdm91ziWyII3NaCSTjwRyuzAbzzbviTRd/jbT8fzZrA40LlTDfNlnmvjicsS7DaeUJdCtUo4HjTV3xFjbxxw5OvS3t275oCtckeSQ/VAu93Dd6uNuqoRb9LakGQj6XWmf9rtFUvpMSJx28zr5EYIUMcnyifu98aE68swI/n7b5lTPGjR5IJ8CbvTxnWPJfwGjWmrjbrpPjZRiP+vUnoG9HCskNRNHvD+OlX9oIg8AvwY8FXA96+YBnBTAdWmjhOs9VGpJYUwqLS+Ft+rCChIFbZKgmr3GhPt4huHTph6QLgQBhcoq1ZQorWbWq8aaEhaQaoMm5+27SQ+vVBPt0Grm6qrOMIgqfhqBm9qdT1Ks15VEjHreDNQUR9QrZVEULWF94yKl9KqnU5Cu6upYvactBat4gkI3lZvm2pVtWFOqJhxFTYSbSARgoJDHGHoswiV7dO6AW/iwO1GOzcnj0pVCzGP1q+9DXXRepfnGRU2LkNL+q5ozt8TUSzew90qU1WWkmpLrqZkDtWONVFojfhQsnh7nzjfqg4SlljWDsB69/VAJ3UaG6FjQlw7qS0e8K6qH/TfD0Tkn+FsiN/v33kJ8AERSXFnH8eHvs9g56pxUIXm9leYqcGPNBod07avNSS9WDUOjyhINWVG96PBG+8X2FC5qdPVWMqRWLqIpmRfP0RpT26N3+26RIMytr9UanyjPh2SXFz3uHliSTZum6ACalyHDsbQlU/7esxUQlnb0l8srbbem5smtY2w7QiZbTdptmlHe87YReN6hYkjtEtcBa1favZhS2SM66PN6rr3aYQ+Nd5pt0+7jcIz4VaD1qCr7zfFjpwlb2bNA95FJBWRJwFEJAO+GPiljnS/FPhpVW33SAM3oBpr4+/qyJpAuHQQUfvVNpH4a9pFGGGQR3a8zoS9N1CsIIXWdkciZgIzabSqE/1gVj2OZ/yw7bSvc62Wd+tk1SDqTtKXMxS4NXLa78VMPeQf9UPFiDteBRq2srnMMx7sgpNmJEqX+r1qvbTWrypUq1ja5ah+t5ldF31E+cw4KFpMMxBfNTF2VKtRbzraR6XRt+02mfHwdyHilaG9IHo3arc47SExAJPrg00OeD/HMcQMp5/9O+Cf+Ge+B/gBEXkv7oD31y4ryM2ca9w1oNuzNNHveYMtfmYmQX+rHbbRHgyNdFqq+LznlmCpfTCqz8J6Vwl2M8E6vzn325JCpTMz83yTkTQnp1raXdAY1WjVzgYQA7ZDKkbqzSTmtVujjaT+bjPALn4/w3G02Zxdkltsa62e65LKtflMzOjnSaN1wWjQQUNajRlzZ0NQ24vjtBf1zypQYLEANUw2mx3wfgF89px0r4EvW6UsN8MIK72yeQno6PyOa/G98N1mKPGs6dNQopmuK82K6uskwrhYOIO3sJSGuuq06HfQcecQ+twg60UTSOv5MKZqJhlGJqsNsDkSrcbxoW2JvUeSEt6LvhsaQPxceA8adWlo2vMks4jxEz8XC18xnbQ15Hk0GfJuM7/WxNyuU7NstUqsHWmvRKRLMBRPPRTsPnymFYKx6IDvBqRFZB2Ms1O68oxwZrVIdxazA6SLQWn3oGpWYn4+7SiU6tHAdFZhunHRqgHcloKiSrWZ47zyh/LEyfgH24HVSycrrd9tvNpiMrEzOOTTfq/r/c7r7d8LJoW2yjxvlUvjzOp5jHTOtTh9aDGaZbQS03jVZu1wpvlprAXt+bkl2B8b4ZxHKmiL0XUwvc7ZtGvQzkHnxNqlskbMa+7MGdNoW2XrUOGCOjU3rMQXLGyh1X17zo2GHhiJu/67mgDabRmn3aFiNv6OG6R9JGn0d2zzauQXk0XrtYWWAamf6SxP+C2tT1c61Tyis2WS+LoslJiqe+2Ju4txx13SUTbRZjnaZ7w0yz8MZxowqYPBftgIY0LpYjxzGUPrmyChaZWWthiLxvnEM3yHaCfxOyGfHtLaXC/vzIPN8ncO+PbFVn0EZjZiqMrcGtyVQySm9EjLq4z0XsfUqJ3U12VGGoyS6vIcdToT0CrsJiyta9vdOrs8dOuiibLF4Ntqbhez6bTfxdWYJ/F3lGOGcUmzyxpe9ZZkXrveO+oRZx3bbKWu2yqaxEKoctc2Zr2BTRf8l/hh2TagtwisGuOL+iW8E4gpGuhB0prhvR3XNLo24w2NiFfbxBw9G6SG6kyPllRRF6pVnrn1oVGfeZgJW6GLGUbp+MEWGGDXpBJ+h1e6mODM81EeDb7YvtbBSMKO5AEmZtZd5dP2R6o8KsYev9vB5BrN2tXWLSZfqjvofenkzPz2j+mnKlI8gbXrHBW4S20flAlWifb83BLsOKBa69PKAiNUcfHBgTCqJXaO47gAYnetsTlA9C3h6Mh4SyXxM6ff5FJKn0e0NVLIxxilKP2ef+1YKaUZjAsNyasRZqCB2TXPMrZhmVZrgAX/R9ibrtpePiYwH0ztVjqo28BBmytlwhK7igOoK0i9V6NfRmjrJWsagrJtxKDtbDHFP6O23p0nXgMc+keq40aVsKW8ax8n4UiiVbtXZ51EolK84exM6EbU166d6u/QftWEaXEVslqVPe5DivrZtuQWblQTRlQ+iQ6v8iTQ2P+vKnMoQ3siC5siiFTbdIlSnVVcLTmN0/Dpuv0R6661fmPWEEhfb5sWpbEhjqrxjhAGdjyLV/cWdMKMRKGtm9J8sLLneMKsCL5rRmvbfuZIEkudLm3JLNQxLk/jgUgy68rXM5i53uGOcjWkkSgd1w51OzVsdiHvRhs4VbYTgfm0pao5RdM4M+3IO84/qmeD6WtdhZnn2+03I1HNyyt6JG5r6KTFtrmlXc5Oic1nLzGNttLvdrrVenJsu5zZuGRQ1Ri3we4dws4ZYVheJ8EeYurpv5rRPaUL+I0sO9YKR4PQbcEv0QL0IBnhBnK09bwNJ7IF2jF+F2sv9UjZ3HQB9cvfogGj8fK6lvRSqYC+Xu2wC6UpwYTZv5aW2hKhl0aMNqSmWKIV364aMTegWnoVBndY+lUdxxm1I1H+bdNEyKthI4ykKvEHblUHTPmT/ILEqaIkaYk/dLBmZD7f9qqbIA2Fs1FmmHMkLc1svBCkp9b1sHmriY8kaEv6nnZMtZStzlfCEjs1rv1ss68qibDq27gRfd7GIppE23BJTSDRsj6N2qCik4rLegYdS4TxRg5D4W7xwRtYYrfsgUXGeKgHrv+7MUgiCajh+YulwgX5zLV9BEFsmeTTkiwaScyzPUVlka78A6Pro6u0n6naozkog3Ol0SbxoI+/IyY4g64idZVTQv2p+2+ed3yB1Cvzytd1P65XuNglEbYymGnnVRhCO0/mVCWW3rqaq2of6aTHLmdJp9F7A1RCxpLPbUEvRigif1VE3iUivyQiPyQiJyLyMr/763v9brCjfmnVNq/wXUtvNRHUdhqYy6Cid4JUo1ATvSemStKEmlFWEkl9dGg4l0K0Ph4yfqfKMxqrbaKo6xfscfXDnc6S8H44y0LrsqlnIDPrlFtEaKJ2rAZe9DswUmndq8rf2MyirdfXeXUxw5hZ1g4wqpfCu8Y0pbtFg6h6p8q4zmfeN53Xa01CW3VuM/24XUL/Na93MMgWw5V2eWbSpaZNmaWbtsrcbFsqu3fF2/072kh/fruuArHa63NbsJQRisiLgK8HXqmqn4Fb1/da3K6v3+l3gX0GtyvskrTiv+c0Ys+2nTuQFtiBKhVwXl4x8UV5tJle8+ZsOr224FqCGeFuXpptMSmUad7j7fo3pNEOybl6riMxbX3HeUST24xNLPRHV/kqptvxTlee7epX37O22EXm1Xb5F0ngYdfzuOkb9ZrzapgoGrfjybaNZRNGlXlctwGYk67wuSXoqxqnwKnf0uYM+DDw+bjdX2H+7rIHgUWMK2Z+y3wVvbFtAupbxlXqEj1rFo3Oea8H0+IKzGhe/rvEkOpf7DTrl/mS9DYqzeJsRbXX57ZgKSP0e379feC3cAzwWeAdwCf87q8wu2tsBRF5fdi9tnzuolehBrU9rDuADqWPdUUOsyF6M8E1229H1ThiGWzPzy1BH9X4cdwZAC8DPgU4Z/mZARVU9Q2q+kpVfWXy6Hm/d3Y1GBblcygDclcWa59NZ1B1F9Zs230zwA9Ji6tvwHxzuGsSYZ/wmT8C/LqqPgUgIj8OfC7wmIikXip8MW5X2MPGJkR/e2hiKXozww3Qa4wN1eZ3qO964ZbZ//qgj43wt4DPEZEzfzjKq4B3Az+D2/0V5u8uu3vcVAfetATZx1lyQBLJbcb+90I/j/Gd8hqr6ttwTpGfA37Rv/MG4BuBv+Z3gX0eblfY9dAONRgSPfqqsbfbHUW14mYnmS0pB8N43lfBXE/vrmhiiHyG9ez0+2yATQ54F5FHooPdf15EPiYi3+XvfY2IPBXd+7qudGP0Wlmiqt+CO2Q5xvtwh6UcccRWEWIh9xlh3fitgLKrrfrDAe/fLiLf5H9/Y/xAdMD7K13JeIeIvNmfcfyZ0XPvAH48evVHVPUv9S3IjRzwPoNtUvmitEMc2Cb5DxHEumgArTK6lpVjgW2vM5h6DirP8b5zp3kYmGHNxKB3Bc5H6Gy2OXGEvTJcdn0d7EAiZKAD3kXkdwEvAP79ugXZD0a4xziGc2wJx3bdb/QPqL7pA97BLfD4kdZJdX9KRH5BRN4kIi9ZVpAb232mgT12cCwVyPZJHVpWlmqt2fowfZwyVX4rXp/3eB877w2qpnPX3W7K7G9wshDbWze+yQPeA16LO9w94F8AP6SqExH5czhp8/MXJbAfjPCu4xCko0MoY1/cprpsA8pgwdLbPODdp/F7gVRV3xHlGR/m/kbg7y0r536oxkMS5l0m8h3UfaUYwkXz+8DS20qyxJYlx2Vl2SclogtCv2DqAQKq1z7gPbr/lcAPNcrvmGrAa4D3LCvIUSLcFDOr6AfGoRopd8gEh8Am/odD7aKF2M2qkbUPeI/S+HLgj7XS/XoReQ1Q4A54/5plBTkywhjrEPQ69NK1Q86+b+YQpd1LKjywJXZ98p23znrVMh8E39zzA96je7+z49o3A9+8SlmOjHAJ9j4+rGvThYMYaYux1XHYR8obQszbZ7pZhAFthIeC288INyTGvWaCh4o9bNM+fG/lNdYHPCGt4DW+FbgxRjh3xr3JQbLHYTzL09hd4bcZUL3uEruNJPeOvGaOH90lei753B6fHSRY+qCwfxLh0Ks01tx89OCwrlV/WyN94WqZFZOat8Ruj/orZsQba9XS+p6X54bZLEz4yAh3g/jIxO1nRr2deUf/zpxvS/P3Ql7RUYUhaGimHLugy3kB19EywnV2p+4UMGRLA7liIrN1WaXogTb7zi8S570kze6b89OegcwSrOjA7Xm3NOM9lAi3gVswuW1tN5a99watiS1MsmtNAgeK27Tpah/cfkYY2Vuqw5tafTxzxu866cdO2wHG4MwhUX1eWLLYfykW2W19mg2HQV9JWVpt4tNbJOzMPbypXa7We322+OpKZ5NtuBqHNy14Z+EKsjmbLizN3z8wON8/MsLtYyGjWDUma1v91dI+Bs1nyLR2LKVsSyoKppKNBnTfou37GL/p8qlCebd04/2QCDWaVVcYCL3sIrGUNE+yiJ9VGnuxbTzu59jd5ko5i5JqnX07A+0nGc3PICprh01vkzM3Zs8J9nHkEj6KRDNk+LN6b459N9CNbUudXWWIJbe4nbrSXcmgOP/6Sv0cnl3gLIlt61u1sB8lwluEiLM26HpZWMKqTGoZRa7KQOLB0FdE2lA3aku9Df69zi4yXVXuagepD1NvMx/jZ6R5XuN1JpO56OGsWCWOcO6OND3ea5DqTXnGj4xwe3B7OQrWCsa4v9se27BlvCr1CIg/0bPxO8EjLNVFd1+Mu6hhewmN3g9pqJ9p42cN7rcKUrYrAmK9V7Fj24pQT2hKKVVdJco/lN8oiPi//WMKVut2UpX68PSoIWxox7a90jrpVv1zjSMJQjuYWspUANOUzEL5JGJYMUNQcW2F+HLS7LtQFw2dFHvwJZRLIslPq761UeM2mGEor/H1a9NIaE8FlcirajvaKG5L6+gm7r/6oPr6QSPq6CPyFjf+NhFNRu3t2sczSAua+HICagXTGgvAbH9TmyesClhBfD+7I0QG4JxKSOzOYOcSYTUWW9JH40erLxuzYkxgbQYZqT01IWtjoMw4IfxAbDCl6NOY2eN8uiB1vqF+Qe3TeGS0idvXWaWD8P3gievTaIPquXZDBYnJPayRqKHVtdmyzCQTt11HfYmfr/KbbZrZSWCOmh/ugWfwMnM4fMNfE9FCqE/1HZURFaSaHaO8YumbMOnIjL0ylN+Idf0504dR23Uwr2oiUs+4tF2+qD5RGzgCCsw5uk7oR1yoS0W0DIAwa9wd7JYRqmCnCWROAgiSCoBN/WfkZ0prHGGliozA5q6TpXQCiJr4HUdVZipQQJEbNFHMuCRJLMWppThN3DO5oAbKsWITsLkhl5QksZgTpTjLyO8J5SnYDEyhmCs3KOzIVcNMvbSWgWZR/QTK0jhbc56AwGhcYIwln6aUpUFEKTPBZuK/oRzhB6UiVmop0wJTQykJU/+IEaU06qQO45hFaQ02TyB35SxHgqgrp8mhKByH1TIaLFYQ4+pQMeEwqFA0ceWwmWIzdZI1dX4Irv0y1/6aKloaLBYtDZT1+1IK11cjsFCOPCNPwabqrhUGTTyJJFq183P5CdYKkoHEkrf/u8RLQmOXDxaMEWyirl6K2y7KiuszL8XbrKYdsY7XqAEKQcUwTf2wEMVmfmIrDKV/PzHqym99f9m6fHbkaLYcCeXItYGUYEoopilaGLKpIAXo2JcT0Nw4/mPqcQAgpaNTrGDLhDQrSBJLWRqmkxSmhmTiOq+cmmEYoXJ0lmwVCjoxlEo1sMDRdej8cqRoptigpmSKtTiCVM8EPRNyH0dMYiF56NKTiaCJwZwWnJxOeXCSUZwZkgkkl4JN3UDUTNHcUALj8ylZVnBxfsL0EUNxBnbsCD2ZuEFfnjpJIL0QTI5nYtEMbZSySFALdpKAgeRsSmIsE2uw0wRjXLrlSLCpGyx2bCs10Q1cXw8FmQpKQgmYzJKNSki0Vt9LoSgMTA1m4kZjOfID9hqSEyHPHeOiiERJdUynPKEhHbu8BZu4QVmOHKNLjcX4yWtqk5qZeUZpU5e+lgmUntkaz0hKsBcpUjjGj3FMUFOF0lBWEprr4HKkCPBwOsaWBskUSZiRQI1xP+xI0cQzReMYR3ni2lEQpFBGl67PylHECDPFlFKp0jL1k5hxwyIRz2AVdGooyoRElDQp/eQr/lhLwRpHG5p5ZjaC4kSwiZcAc9BJAoWjJzOF/FFHCyhQOH06tGmgKykFjEIp2NyQnFhORjkPL08oJinJlSG5dn0m+VASIU2V7Q5g5xIhpUDhJLYGxA1MzdTdC4wwcYNJU3UDzUtCKpFEmCnGz7JivdRXCmIsJ1nBxbjEjlNM4QaDe9fnUwoqQpqWnI1yHvpZ2mauLEzcO64s7l1TQDJVcqSSZIKKpRYnERUGEsUYS5qUjq78oLOpT8v475GiokgkSQWpzeRgDahxbZYkFklqG5WqOGZRCCb3Ek/i3k+m6q75fCkFCRKh4tJIbK3KWdfuWtQqvKauHRJfF/AqmTjm4wYuEPoMP3iD7S51EiET46Sn1EuzoZ99PQnt6GlAgEnpLmriy+rLFGC9FK1JkDwd43Nltl6KAqOuHZNrR0s2tHsCVhTjOaspwOJpMwjPntlinS3WoCSi1QTt2txPBIHBJ6BJbUswJY7hluL7yeXlGKdrA8mlsndq4q9DvXWy7xdjlJOs4CFOipTcMVZNQOKJblMcGeEWYSG5MljVJiO0UjE1PbFg1KlxAJmixjoJyyqJOlXEZk691UyxJxbUkF672T+ZOAkyTS1PnF5yeT7iYpJgJxnJtSe0saKpRXKDWsNJVvCC84d85Ow+xbmhOFX0pEQvDcmVV8/9gEwmSnYBV8/3kkfEWGzuZn1zZdCRkiUlp1nBM6UgU69SjhWbBtUR9KR0161474hXbSfi1H31joORZZSWXKfWtZVvp9IYzERIrmuV3RSQXkNxSpVvGGyBaZfnFkbOYCVGUesmEGtBTeLKd2JhbMmygixxjLCwBhEnudnMtRPqBjmBOVSmA8VcG5IrJw3aU+v6PvXGV+ttoKkb5JLZynl1Pc2crc5LR7ETyV2g0ho0cQxXAhM5sY5OPIdNriF7CPl5rc7bTD3zcIwvTKalGD9JeK1AgYmhLA2pKRmnBTougQSdenr0Uqlmzulms1rdN7k3U1wbzFRIrpzJRVNFxyVylWAmnhEmgT79pGODSUhQDKmx3B9f85TeQyaG9ErILlyHJpMhbYTbZ4T+qM4fAV4K/Abw5f6kuvZz/xb4HOA/qOoXR9dfBvww7lz1dwBfpapTERkD3w98NvBx4CtU9TcWlWWnW/WL+lmrrKUGvOEYPxNKZpHUzeZOUlDI/OAJ0mCY0VM/E/t7UijJ1A1E42fPe9mEcZYjI4saR4DOY6fuPT9Lp0nJvXRCMvJMJvODUiDxM3jwJAeJEACv4jlJR32dpKpnlljGqZ/+y1qSrSRCA2ZUYrKyTiezLn9v05TCfdQKaWIxxuflpRW1Ti0KEqF6dczkisl9W5fSSEsKJ9WZrCQZWUzmPpLZWjoT17YmK0mNJQ0Soe9MNU4iJGu2ZZy+pq4eydQxd03dsy4f34a2lmIktZDVDFf9O5p65hm3t5dWNb6euG+Tla7PvYRmCkgnriwabKxJLW0CSOGeC+3kpF5vfy6cJJyIkopFQl54ugimllCW4FVWL6mWfiIqIfH9osbRO4AppDKJqKd5Qht5D7ijU8tZOnXlzb2kO1WSqSJ5w5eyPhScParHZzOEc41fDvyU/92F/5nm4UwB3wF8p6p+GvAM8LX++tcCz/jr3+mfW4jdnllinX0tuTSYa3Ez5MQ4xmTcrDo+n5KOC2fPKoTkpCA7n1KeWm/f0epZe6roaUlyXmBHluwCxs9Z0ktIroRRWvCpZ0/zwkcecO/RKzRTsgsnMcppQXJauhn60vDoaMKn3/sIj9y7ojy36HnJ6b0Jmimjh5b0Ul2eYyW9UkbPupgaPSn9x0my5AZzbUgvheTacJrlPD6+RAshuTJO2j31NiYvEZ7dm3B2b4KcFXBaImcleloiBaQXkF0I6aVjZvfGE8YnubNFJThVeuolg4feTjZyA2/8bEn2UEmuheTKfdIrl1Z6JTCynN2bcH5+zaP3rji/d83J+bSSUNWAOSs4PZ9yNso5zXIEyMsEMa4t7IlldD5FxiXJREiufR7XTkWV0xKxkD3wzOCsILmXO5vsaV6rxoAxymhcMD6fIsY6B4s6LUFPS/SscJ/TsvGRs4LkvEBOS/TUImcF549cc3ZvgjnPsSeu/8ZP585JESTCU1tJciiuXR4KyaVrKxKvoaijp7IwpFJynk1cGc8KNxknYE9L9LzAnOck54XTVnx4T3KtpFe4z0Nh9EAZXTgmfnpv4tK/FO/Ic6aZ8fmUk3tT0vPctWEumGthnBZ8yumzgCtv9hDGn7CMnlOyCy9cDIH9OdcYVf0p4EF8TVyM1+cDb+p4P073TcCrRBaH2+9UNRY/s4qltluJ+vgrN/NlWQGk5N7WkSSWJLHkxs+URmr7mFexTFJSmgxTKMlEKxtMYpRH02vO0injtOSh1Pmb1Nm8tHS/s6TkfnrJOCucOpeVjNKSKwEzVcxInASQKKYQkomzrUnqDDuqjglK6WZ9kzuGlBrLyHjV0eLqHOxIXgoepQUAV6lFxXtzvcPB5N676dXOkSlJjK3iJrGAeLtTXkubopBMLEnu1D/xdQ8rZzRxDquTrCAxlsRY8jJhAlyntbcmSSyjtCBLSpIQ4BxiBL1UnaYlZZG4di2cZOXeD+3jVEMUktSSpCVZViCSkEOlzomoU49FmU5TSutDmxKnRps0UheBEJJiPA2EsWlSyzgtKa1lkmTYIMVfl1WguAZJK1HX5jhbHjjpzIqz2+IlWlMIuQpGnOMoTUvyNPHMztNhajGJIsZWsZUh0sHRhDh7Ye7VbQNp4ujHFK7vSPHjoA5eVVw/SymkxnIvmbj2KgSTK8nUkuTeRj6Uarwbr3Gfc43n4XnAJ1Q1UFt83nF1FrKqFiLyrH/+Y/MS2y0jtE7CUeM9fQZCkJhzGlied37Jc9djJsUZCJydTDkbT7m+GKFlhs0F42035jwnyUru37vmY1cZo4cwemZK9vyEcuRmz08/+TAPyzHPTU/4ePo4o+cs03vCyekUYyzT6RlmKjw2uuIzT36Lt55/Oh89e5TT8ylP3rvgE9mjjD4xBRm5UJnUSZ6jp6+AMeOzHGvdgMx1hHngpKLM1/OR7JoXnDyA0kli+X0lO5tSjlNsarBj5ZPuOXd3USaU1mCMC4+w+QnZA2f7MiVMrHB/fMVz0zEPR07lcgG1QnoJ2QNl8phQjl3YRfb0NaN7CclVCuIkEvHmCE2dF/r55w8ZmZLUlFyXGZMy5XqSVRLhvfNrnn9+wWmak5qSh9MxRWm8RGhJzgoeP7/iaRXs5JTk2tkm1QCJcno2JbcnZA+dF/f0bMI4KzgfTbmYjrh65rR2sohyNp5yluV8pHgEe5UimSU7zRHjGK5IHaJk/XujUck4y5nkGdNpwslJzifde0BeJqgKzxaG0UPInnqIFGdOqxgp5rTAkqITgamQPnSqLFawI2H6fCU7zbEfz0iuYFIYxqbgfnbN42dXAFxkYyiF5LTg5CQnS0qMsXxifIZNDclESa8d4ysvnWNj/InCO0oMT9674KHeJ72A4gzKE6diP352hRHlukh5eD1mkp+STISTNOelJ248Zxcwek4ZPXWFlCdkT5rG8tC1oaD94wifFJG3R7/foKpvCD92dK7xxth5+IyzWbnZTCLJLoSDnKY512nqJR03Y56kBSZVyiTEz7nvJFHS1EkskihSKGZaVhJhaixPJA+5n15xL5ugophCKwnQGEvuJbjTJOcxc8VZOsUkyigtOE+nYMBMS6RQZzvz+cgkd+VLy2q1TO6N/0EilNKVYWyKSjJw71gmYTWGweUpyigtKUonqQqQW2eftLnzdooVRqYkM7ZSu8Q626PJne1JVKp7ZpI7CdnbJoNXXaxrXiPOhurSLKsQmTQtKw/tKC05TXNGSYERxYhi1UlQJE4SO81yksRJ15VEaADj2jFXVzZUGGcFJ2nBaZqTl0kt1eLmxFFScpLmbqVJ6cJJktRijHMUuee0anNVZwIZ+X4oS8M4LTlLpxTGMM5OnJRWKHI9cYwu2D4TxSb1SpZAGyb3ZgcBkzjnjSkAK2Ti2uo0zblIMy7E0WPqJedRWrryec9z0IKcV9fbbSeli80xwnk29XnXtnKMcprmJN4mO0lSpp5+RknJY8mFeyf39utJjplkVXTDIOi/smThAe8DnGs8Dx8HHhOR1EuFL8adgYz/fgnwARFJgfv++bnYuUToVFepmIB4n4gaILU8Mb5kUqYE3e80y7k/uiJJSxffZTy1GCVJS8ZZwSOjCR9NLek1JA8nZFdn5BPDOCl4UfocL8w+wYdOHnMe38sCU2Scjackolx5de7R9IpPTUtOk9w1TGJ53vgCjGIuc9LzDFLBpNbZGJ+7AJ7P6SintIK1hutkVDF5M3Uq9Hk65Yn0ojK4I5ClJRPvuFGBF5w4ifCp8T2mZYIAU6MUJaRXFpuYKvD20eyas2zqbI1h+ZgVJ3lcacVsTaHIg0vSq3NM7l5OJj62rgQZOQbzgvFD5wk1BVdlxoPkhA9k97n2Htnz0ZTnjS+qZV1ZUmLEq3+JU+EeH1/ydHqKLXCxmhOtnBBno5xLC9mFYkrD+WjKeTblsdGVWyLmmZB49fw8m/LE+JIPpY+6yUMgywo3oWROCyqtcUzPS4Rno5zzbMqFZ5DnoykvGD9kYlOenZ7yTFo6KfXZ55AymFQsaVZS5s5D7EwJYEoX46kpkCqjUcGVv6cKZ2ZKnibcH19xkY942iiCC78aZwVnWU4qlo+okFw7e3Qy9V7p3AV2J1c5GIOkhsdGl85kcK0Up74xUuXxk0sMTg2flAmXnk5PkpxPTp91JoTCmT/kwSXpKCW9PhlGIoSdeI2pzzX+duafa9wJL0H+DPClOM9x/H5I9//1939adXGFdr7EzhTOFmODrUq8VIMiAufphHFS+CVITr09S3OMUT/4fULijOvGWE6SAjEWyYFp7laDFJAYyyOm5JHkivNk4phaYZFSGSVOAgo2s7EpeNScME4KVN2756l7R6a5kygMiAGTW3Ti7DSOMbigYPH2zorZWBiZgjMzRbRes5wmXrLxRHvubT4ub3E2QH8/yV1dpAQUTs3UlVuobFCBySYTi1gnaooFnU6R3Fb5BvuolDhJ19fxxOSMpSDBYtV4uxVVmc7TSd1/1QYJzsaYJJbzdEqWWCY+bedhBcSFD6FCMnWGyXFScJZOOU8nPEzGLi2/jC4xzsN+nrpJKizMTb00OPbtlnupNDGOGMZpwTgtKNQwKdKqzKktGSduJYbkgr26ruy0oewSaEoFU1gf8+cD+n35r3CmBlTITMGJ5pynUxcN4OkxSSyZsWSm5CQpalovnBZiU5euKUEmpfOAG+VeGkmEVWiTcpo4hlqocfX2dJqK5VGZuPb39ka9niDTgmQoidDZHQZIaCmWnmvsf/974HcD90TkA8DXqupbgG8EflhEvg14J/A9Pt3vAX5ARN6LO9f4tcsKsltGKM6BYIPHUzwjzCGbCPllwkUxZlKm1WYAeZlwWWSUhQtSrma8UphOMhdvVqaoNdhRAuORW7qWQm4Tni4zfnP6JL/y4JOQqwSbGjQRpt5+ZEdKWQoTm/LR8pL3PXgePDXmmdEZDx4/cQb5UYamgpYGNSV2ZJDTUwCmhTPqByklhJSEegKUiA8MBxQmeVbF/SUTeK44IRHlusiYlAlapFxNnZ2uzJzNrzzxoScBngGqcSq7zZxdyxS4OLVSkZMxdpQ4Izw0lpTZFKw1XBRjCmO4NhlXZcZFOeJqmpFdOgZ1mY+4KMY8Mbqo1MIqsLowFEXCRTEiL01j2aPzxLp2LsfK5ZMp5YmroxFlZEoKNSTnOXZsuHd2zaMnEwzKRTEiTSxyVmIy58Qprak2ewjS4OX1iLJIKB+9JBXLdZEyLRImZeroyKZcFRlF4WgjvXdOMoXsWcM0SynPcux1yuiB+PhSoUzcipDixLVZ7oO6beoYY25Trm3GRTHiMs9IHiZICVeXY8e8ffmSK2H8XD0JqRHKsYAqepJWqtBVmVV9YwrIHgr5lWvTVKy32/qYzgQswoVmzo58CsWpQc5PKccZpV8uOQj261zjPzTn/fcBv7/j+jXwZauUZbe7z/iBbf2KhAAzcYbfyaVxTM+ayuuWW8N1mWGt98ha8bO3Uk4ScpM6xqlu/a49SV2wciIU1vAJe8oHJ4/zweceJbkWNHUDtiJwvzwv14SnbcJHH95j/LTh6v7YEakFHSfYzIf5lMYFy564xaDTIvFhV27pg/WxjeXIBRAnoi4YWkI4hTCdJpgppNeWZOKYf2ZKJmXCtEjIy4Q8T0mgWoZXjoHM1hujWqk8jySBATmpM71y0oKOR9hRtI43Fdz6RjforRUuyhG5GjJruSodM5xOU+5dOtX7ukiZ2JRMSs6Sqfdaa7VSpigMl8UIa0290iQVH+Po2tmOlcljhnLs1LzEpFyXKVaF09Mp1gqPnV7zSDbBiOW6TEmMd5KIUhSJ8/CTVuPcqjC9HKHXCRcj53zJy4SiNEzKhIty5Bw/RYq1ppq8kqmSPTQU9wxlKZAL6YWQXlHFdZYnVGt9y9JQxbiKo5OJTbkuM66nmWvrQri+TrnOMrLUmQ6SiZA9tN4GLtVyRbHiJmwcr7koRoTlilI6Z2Jy5dp0ZArHyMukilW0KlzYsVtvPHZMW8ejmu4HYYSKlu0tl243du8sKZ1BP453Cp5McM4FCWqRt2uHa2q0cpSoceqNGGfAR5xnTkptpJdJwdgUnGQFzxrvFVSn4kkrb4O6Qe7TTsWroBanyxuh2itJnUqVGOtCPLy6GpwR4QO4ffX80jb1ISLVJgfgHBViScTnr5bSS11ig0olVbmNBDOBVM6GoFZV6pXjznV7hHpoXWfxwcGZWFJTkqohtXX7i38mM25wG2qJx1XM1WVkCvdOVW/FB0WRGFs5IMRSBSSHAO2wIWvqy1Cr3rVEYowLIk+k3goMa6rwF2O0CgEKz6ViKb1jSfD9V5Z120C9+WugteC0KWvJ2fj+FtsUklIpfTu5CUm8mSaUI9CI+K1jKnoI7SPOHJBKbZ4Rq9USyNBGifj0ovIlYqudZ1x/K1JaArluDGUVZ8mtwG6dJVBH2XujPtREgLoBYbz3tbIPmQKTuIDlarOBRBEfYxgGqlsra/1CeNeRCcqZcSEZbpG/Y5jVSolg1xLlxBNdSDszpbMRqlbELkYrwkOct7C04u1cWkmtpvBrbH0ZQj3B2ZOKYN/D2RETb48K5FdY63hvqVVsIlZIsKRhCx4vYVaxi6XWg7waHLUDRayzUYXBKOJso2OTkxrrpCyTVvYntGZcTi0Ozgq/NZZf+1wxtNCWJVg/iyW+L5OJq0eWlGSJY/yh3njmMTK1FJJUDNHHkoqSea+x25qrxCRKmbp3M1OSSFr169iXNTEWk1hEE7QoPbNx81KStCcv54gzJdgSCJNONXkIFt8HxpKYeHmlo4VgJwQ3KQcTQegH15/OnIEoY29PrCcR9zM1JSNTMkrKmhH65klwTqFgbxdfr7j/N8ZxG67tQYVq5w9NqRhhtYbUwmUxcmqrtxFOyoSH+ZiySAibUIbgUpsnFKnlunCqs80MOs78Jg3ObvcJe8ZzxQlXeeZXXSTYBCZFSpmUlTp3VWY8bZ19zORuu7CLYuR2/Rg51VhL58bUVJxqrMIkTyvVWP2mApVq7O2Ul17PcoHOQlEkldSBhQf5iYvj8zauokzcM4ZqOye3a4sysSlTm1bLE2MbYTlyapwpncShJyPKUe1xtlkIrVHXPqXwXDFmbFIyUzobYTFy9licpDQpUi6KUaUeA7Vq7G2El8WIoky8WuzMH25LIfc+oli/JdV1kXobYeFMHpEuZ3E2RatO9kzC2uYiofQOkupZFRd0XhiK0nBdZExLZ1aYlAnPFWOm1qnGpTdnyNkJaqSa/IrCeFOLo0WbSr21WAZqhcKrxtYHOpdqyNXVeVokta22MEzzlOvM1S/0b9iAwaZ+rXUplGOnZ6sVHuRj3zf1eBArrk1twlWRMS1StwopdcsOn7POgOnoDPR0jB2nzts9gEiogB4lwu1BDeT3hOLMb53lr5uJVPFWH7m85xbb+903Hl6PXaDxVVItWjdhx43LhFzh6dNTyklCcW6YlmOm94T8HKZlwq9NX8D7rx7n6YdnSCHkjyTYEVxejUnTsgrsfjY/5V2TT+HhwxPuPYTrhykfengfKQz5oyOKMwNTRQuhODUUj58hCpeXYzcJq3OmhG2VinOhPFUeFGM+lt+DzFKcOfVxejHi7NpJjckUPnRxn8RYnrs8cYO+NNjccJLA9BGhuAf5uSKp8on8jOcmJ26DgNLtvaeJkt9Lqn0SzdQx9+LxM6aPJn6rLdfWIfbQplAWCR+8eKyKIZyUKZMiJb92AdgoPLwa8+HxozySXVP6FZlpUqLWIBNDfpXx0Yt7TKYpOnaiVlhHTSk8vBqjBqaPOvPBsxenXKYjrk9S52DyRSutYVomPDs95TJ3duLT8ZTracb1pZtIxK9NrpzlVwmSG66vRjydWPI8ZTpJsVb4YPYYpTU8e3FKeZWSnxvK59+vbMJmKuRXzmnlnE6Qn/vJ69xt6EHh0hajFGfiHDea8KA44WOX51xdjUiDFHdtmJgRqnA9Sv2ad4sdpY5eToXizPVLft/ZnpnCb188ihpHL8HjLiV87PLcbd+Wp1xNRm7vxjO4LjPenz8PESW/p0wfEfInzsgfTcnvRXtZbjRQ9SgRbhsalh75ibgtysfeQag9hGFbrvbOIy4KPhCz+A/VpqW5JhTW1PsbGr8dfnhPcLOtCrmmbvmWJ+6qaP4dl1/YTt+nh78WyhPqFsJboK6PzJbd7SwjYE0tVdrgTqdavF+lhdsOqipcFE5ULelSnERoxEkeJmp7mu9ZFUp1jiD3dyi8v++Dlq3W/RL2JQyblFRL4ULdq+Vl4sNQ6nq4tGSmn+P+1yqfmgZ85d0mqtU6OWnQQNXHPv0ipKWhXK38gn0vmH2D2SU+vsBKqErj2AXr61E1p+83Vw7ToGuVKG3BL6P0k0b0THysbGkFIyEagWq8BJoO5VUR74yRBp1sirvmLJElcYbDZibyFHDBgjV/e4YnOZyywmGV95DKCodT3k9V1edvkoDf9urJno9/TFVfvUl++4CdMkIAEXn7oiU5+4RDKiscVnkPqaxweOU9YjXsdhuuI4444og9xJERHnHEEXceN8EI37D8kb3BIZUVDqu8h1RWOLzyHrECdm4jPOKII47YNxxV4yOOOOLO48gIjzjiiDuPnTFCEXm1iPyKiLxXROadVnVjEJGXiMjPiMi7ReRdIvIN/voTIvKTIvKr/vvxmy5rgIgkIvJOEfmX/vfLRORtvo1/RERGN13GABF5TETeJCK/LCLvEZE/uK9tKyJ/1dPAL4nID4nIyT637RGbYyeMUEQS4B8BXwS8AvhKEXnFLvJeAQXw11X1FbgzVP+iL2PfIwdvAt8AvCf6Pe94w33AdwP/VlV/N/B7ceXeu7YVkRcBXw+8UlU/A7c9wmvZ77Y9YkPsSiL8/cB7VfV9qjrFba39J3eUdy+o6odV9ef83w9wA/VF9DxycNcQkRcDfxx4o/8tzD/e8EYhIveBz8PvIKyqU1X9BHvatrilp6f+vIsz4MPsadseMQx2xQir4/U84qP39g4i8lLgs4C3sdmRg9vEdwF/k3pnx0XHG940XgY8BXyfV+XfKCLn7GHbquoHgb8P/BaOAT4LvIP9bdsjBsDRWdKCiNwDfgz4K6r6XHzPHwBz4/FGIvLFwEdV9R03XZaeSIHfB/xjVf0s3Hrzhhq8R237OE5SfRnwKcA5cPBraY9YjF0xwnC8XkB89N7eQEQyHBP8QVX9cX/5I/6oQdY4cnBb+FzgNSLyGzgzw+fjbHCPeXUO9quNPwB8QFXf5n+/CccY97Ft/wjw66r6lKrmwI/j2ntf2/aIAbArRvizwMu9522EMz6/eUd594K3sX0P8B5V/QfRrXA0IKx45OC2oKrfrKovVtWX4tryp1X1dcDP4I4vhD0pK4Cq/jbwfhH5dH/pVcC72cO2xanEnyMiZ54mQln3sm2PGAY7W1kiIn8MZ9dKgO9V1b+7k4x7QkT+S+DfA79IbXf7Wzg74Y8CvwN/5KCqPn0jheyAiPxh4G+o6heLyO/ESYhP4I43/G9UdbLg9Z1BRD4T59gZAe8D/gxuIt67thWR/wH4ClwkwTtxJ6q9iD1t2yM2x3GJ3RFHHHHncXSWHHHEEXceR0Z4xBFH3HkcGeERRxxx53FkhEccccSdx5ERHnHEEXceR0Z4xBFH3HkcGeERRxxx5/H/A7eULIwCcRgSAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(net.As[0].detach().cpu().numpy()[:100])\n",
    "plt.colorbar()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "947e1cfe-ba22-4b52-b12b-5d339576507a",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0dc27cf1-974b-4485-bf0b-af7bb5f2dcde",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "protective-cylinder",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAD4CAYAAADFAawfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAANHklEQVR4nO3cf4zk9V3H8ee7t/woLRYoI2Kv69LYkJwklbpBTZvGXi0cPa0m8sc1sWK12RitoYmJ2Qb/UKPJtX+Yqm2sm1olEUuVSkq4CNIWohgLchQqx4HQ8wiHVETUcphCqG//mO8uw3Z293s7851539zzkUzmO9/5zHdfn/3uvu573/nORmYiSarrVdMOIEnanEUtScVZ1JJUnEUtScVZ1JJU3FwXGz3//PNzYWGhi01L0kw6ePDgM5nZG/ZcJ0W9sLDAvffe28WmJWkmRcTjGz3nqQ9JKs6ilqTiLGpJKs6ilqTiLGpJKs6ilqTiWhV1RJwTETdGxMMRcTgifrTrYJKkvrbXUf8+cGtmXhURpwNndZhJkjRgy6KOiNcB7wB+HiAzXwRe7DaWJGlVmyPqi4D/AP40It4CHASuycznBwdFxBKwBDA/Pz/unJphC8sH1paP7t87xSRSTW3OUc8BbwX+KDMvBZ4HltcPysyVzFzMzMVeb+jH1SVJ29CmqI8BxzLz7ubxjfSLW5I0AVsWdWZ+A3giIi5uVr0LeKjTVJKkNW2v+vhV4Prmio8jwAe6iyRJGtSqqDPzfmCx2yiSpGH8ZKIkFWdRS1JxFrUkFWdRS1JxFrUkFWdRS1JxFrUkFWdRS1JxFrUkFWdRS1JxFrUkFWdRS1JxFrUkFWdRS1JxFrUkFWdRS1JxFrUkFWdRS1JxFrUkFWdRS1JxFrUkFWdRS1JxFrUkFWdRS1JxFrUkFTfXZlBEHAWeA74NvJSZi12GkiS9rFVRN96Zmc90lkSSNJSnPiSpuLZH1An8bUQk8MeZubJ+QEQsAUsA8/Pz40soqbSF5QNry0f3751iktnV9oj67Zn5VuBK4Fci4h3rB2TmSmYuZuZir9cba0hJOpW1KurMfLK5fxq4Cbisy1CSpJdtWdQR8ZqIOHt1GbgceLDrYJKkvjbnqC8AboqI1fF/kZm3dppKkrRmy6LOzCPAWyaQRZI0hJfnSVJxFrUkFWdRS1JxFrUkFWdRS1JxFrUkFWdRS1JxFrUkFWdRS1JxFrUkFWdRS1JxFrUkFWdRS1JxFrUkFWdRS1JxFrUkFWdRS1JxFrUkFWdRS1JxFrUkFWdRS1JxFrUkFWdRS1JxFrUkFWdRS1JxFrUkFde6qCNiR0R8NSJu6TKQJOmVTuSI+hrgcFdBJEnDtSrqiNgJ7AU+3W0cSdJ6cy3HfRz4deDsjQZExBKwBDA/Pz9yMJ38FpYPrC0f3b93ikm0alr7xJ+F0Wx5RB0RPwE8nZkHNxuXmSuZuZiZi71eb2wBJelU1+bUx9uA90bEUeAGYHdE/HmnqSRJa7Ys6sz8SGbuzMwFYB/w5cz82c6TSZIAr6OWpPLavpkIQGbeCdzZSRJJ0lAeUUtScRa1JBVnUUtScRa1JBVnUUtScRa1JBVnUUtScRa1JBVnUUtScRa1JBVnUUtScRa1JBVnUUtScRa1JBVnUUtScRa1JBVnUUtScRa1JBVnUUtScRa1JBVnUUtScRa1JBVnUUtScRa1JBVnUUtScVsWdUScGRH3RMQDEXEoIn5rEsEkSX1zLca8AOzOzOMRcRpwV0T8TWZ+peNskiRaFHVmJnC8eXhac8suQ0mSXtbmiJqI2AEcBL4f+GRm3j1kzBKwBDA/Pz/OjDNvYfnA2vLR/XunmGRz1XMO5htUMeuJqv69V7davZmYmd/OzB8EdgKXRcQlQ8asZOZiZi72er0xx5SkU9cJXfWRmf8N3AHs6SSNJOk7tLnqoxcR5zTLrwbeDTzccS5JUqPNOeoLgeua89SvAv4yM2/pNpYkaVWbqz6+Blw6gSySpCH8ZKIkFWdRS1JxFrUkFWdRS1JxFrUkFWdRS1JxFrUkFWdRS1JxFrUkFWdRS1JxFrUkFWdRS1JxFrUkFWdRS1JxFrUkFWdRS1JxFrUkFWdRS1JxFrUkFWdRS1JxFrUkFWdRS1JxFrUkFWdRS1JxFrUkFWdRS1JxWxZ1RLwxIu6IiIci4lBEXDOJYJKkvrkWY14Cfi0z74uIs4GDEXF7Zj7UcTZJEi2OqDPzqcy8r1l+DjgMvKHrYJKkvjZH1GsiYgG4FLh7yHNLwBLA/Pz8OLLNnIXlA9OO8AqDeY7u3zt0/SjbnIQT/Xobzflk1WYfbjTPUb4XG33fZ+37W0XrNxMj4rXA54EPZ+Y31z+fmSuZuZiZi71eb5wZJemU1qqoI+I0+iV9fWb+dbeRJEmD2lz1EcCfAIcz8/e6jyRJGtTmiPptwPuB3RFxf3N7T8e5JEmNLd9MzMy7gJhAFknSEH4yUZKKs6glqTiLWpKKs6glqTiLWpKKs6glqTiLWpKKs6glqTiLWpKKs6glqTiLWpKKs6glqTiLWpKKs6glqTiLWpKKs6glqTiLWpKKs6glqTiLWpKKs6glqTiLWpKKs6glqTiLWpKKs6glqTiLWpKK27KoI+IzEfF0RDw4iUCSpFdqc0T9Z8CejnNIkjawZVFn5t8Bz04giyRpiMjMrQdFLAC3ZOYlm4xZApYA5ufnf+jxxx/fVqCF5QND1x/dv3db29ts+ye6zVFeu/71GxncbpvxJ7qdE10/TdWybrTP2/xctM3fZm4V99V2jfp7NOnf4S63GREHM3Nx2HNjezMxM1cyczEzF3u93rg2K0mnPK/6kKTiLGpJKq7N5XmfBf4RuDgijkXEL3YfS5K0am6rAZn5vkkEkSQN56kPSSrOopak4ixqSSrOopak4ixqSSrOopak4ixqSSrOopak4ixqSSrOopak4ixqSSrOopak4ixqSSrOopak4ixqSSrOopak4ixqSSrOopak4ixqSSrOopak4ixqSSrOopak4ixqSSrOopak4ixqSSrOopak4loVdUTsiYhHIuKxiFjuOpQk6WVbFnVE7AA+CVwJ7ALeFxG7ug4mSeprc0R9GfBYZh7JzBeBG4Cf6jaWJGlVZObmAyKuAvZk5gebx+8HfjgzP7Ru3BKw1Dy8GHhk/HFHcj7wzLRDjMmszGVW5gGzM5dZmQecfHP5vszsDXtiblxfITNXgJVxbW/cIuLezFycdo5xmJW5zMo8YHbmMivzgNmaS5tTH08Cbxx4vLNZJ0magDZF/U/AmyPioog4HdgH3NxtLEnSqi1PfWTmSxHxIeA2YAfwmcw81Hmy8St7WmYbZmUuszIPmJ25zMo8YIbmsuWbiZKk6fKTiZJUnEUtScXNVFFHxHkRcXtEPNrcn7vBuKubMY9GxNUD62+NiAci4lBEfKr5VObEjTKPiDgrIg5ExMPNPPZPNv13ZBx1n/xuRDwREccnl/oVuTb98wkRcUZEfK55/u6IWBh47iPN+kci4oqJBh9iu3OJiNdHxB0RcTwiPjHx4EOMMJd3R8TBiPjn5n73xMNvR2bOzA34GLDcLC8DHx0y5jzgSHN/brN8bvPcdzX3AXwe2HeyzQM4C3hnM+Z04O+BK0/iffIjwIXA8Slk3wF8HXhT8718ANi1bswvA59qlvcBn2uWdzXjzwAuarazY4r7YZS5vAZ4O/BLwCemNYcxzeVS4Hub5UuAJ6c9nza3mTqipv/R9uua5euAnx4y5grg9sx8NjP/C7gd2AOQmd9sxszR/wGY1jut255HZv5vZt4BkP2P/N9H/9r3aRl1n3wlM5+aRNAh2vz5hMH53Qi8KyKiWX9DZr6Qmf8KPNZsb1q2PZfMfD4z7wK+Nbm4mxplLl/NzH9r1h8CXh0RZ0wk9QhmragvGPil/gZwwZAxbwCeGHh8rFkHQETcBjwNPEd/B0/DyPMAiIhzgJ8EvtRBxrbGMpcpaZNrbUxmvgT8D/D6lq+dpFHmUs245vIzwH2Z+UJHOcdmbB8hn5SI+CLwPUOeunbwQWZmRJzwEXFmXhERZwLXA7vpH92NXdfziIg54LPAH2Tmke2lbP21Op2LNG4R8QPAR4HLp52ljZOuqDPzxzd6LiL+PSIuzMynIuJC+kfG6z0J/NjA453Aneu+xrci4gv0//vUSVFPYB4rwKOZ+fHR025uEvtkStr8+YTVMceafxxfB/xny9dO0ihzqWakuUTETuAm4Ocy8+vdxx3drJ36uBlYvWLgauALQ8bcBlweEec2VyBcDtwWEa9timT1aHQv8PAEMg+z7XkARMTv0P/B/HD3Ubc00lymrM2fTxic31XAl7P/TtXNwL7m6oOLgDcD90wo9zCjzKWabc+lOR14gP4b3P8wqcAjm/a7meO80T8H9SXgUeCLwHnN+kXg0wPjfoH+mzuPAR9o1l1A/wfga8CDwB8CcyfhPHbSfxP0MHB/c/vgybhPmvUfo38O8v+a+9+ccP73AP9C/yqDa5t1vw28t1k+E/irJvc9wJsGXntt87pHmOKVN2Oay1HgWeB4sx92TTr/OOYC/Abw/MDvxv3Ad09732x18yPkklTcrJ36kKSZY1FLUnEWtSQVZ1FLUnEWtSQVZ1FLUnEWtSQV9//ynh+IkAELMAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(net.vs[0].data.cpu().numpy(), bins=100);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "charming-legend",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAD4CAYAAADhNOGaAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAilklEQVR4nO3deXxddZ3/8dcne7O1TZO2adN9pbTQQiiLyFLZUYqoLDpSEab+Rh11HGcGHjojg6g440PFQUcrW2GQdRypiCLQVmRrm7ZA6Z6uaZtmaZp9vfd+fn/kwoRMCpSb5Obe834+Hnnknu/5JudzetL7vud7NnN3REQkuFLiXYCIiMSXgkBEJOAUBCIiAacgEBEJOAWBiEjApcW7gA+isLDQJ0+eHO8yREQSyvr162vdvah3e0IGweTJkykrK4t3GSIiCcXM9vXVrqEhEZGAUxCIiAScgkBEJOAUBCIiAacgEBEJOAWBiEjAKQhERAJOQSAikgDW76vjZ6vKae4I9fvvVhCIiCSAZzZXcefzO8lI7f+3bQWBiEgC2Lj/KHPH5ZORpiAQEQmcrnCENw40sGDiyAH5/QoCEZEhbltlEx2hCAsmjhiQ368gEBEZ4jZWHAXQHoGISFBt3F/P6LxMxg3PGpDfryAQERniNu4/yoKJIzCzAfn9CgIRkSGsrqWTvUdaB2xYCBQEIiJD2mtvHR+YMGLAlqEgEBEZwjburyc1xZhXMnzAlqEgEBEZwjbur2f22DyyMwbuycIKAhGRISoccV6rqB+w6wfeoiAQERmidtU009wRYsGEgTtQDAoCEZEha+P+ty4kGzGgy1EQiIgMUWv21DEyO50phTkDuhwFgYjIEOTu/GVnLR+aXjhgF5K9pV+CwMwuMbPtZlZuZjf3MT/TzB6Nzl9jZpOj7Rea2Xoz2xT9vqg/6hERSXTbDjdR09TBOTOLBnxZMQeBmaUCPwMuBeYA15nZnF7dbgSOuvt04MfAD6LttcDH3H0esAR4MNZ6RESSwQs7agA4Z0YCBAGwECh3993u3gk8Aizu1WcxsDz6+gngI2Zm7r7R3Q9F2zcDw8wssx9qEhFJaH/ZWcvMMbmMHaAbzfXUH0EwHqjoMX0g2tZnH3cPAQ3AqF59PgFscPeOvhZiZkvNrMzMympqavqhbBGRoamtM8zavXWDsjcAQ+RgsZmdSPdw0ReO1cfdl7l7qbuXFhUNzj+OiEg8rNlzhM5QZFCOD0D/BMFBYEKP6ZJoW599zCwNGA4ciU6XAP8DXO/uu/qhHhGRhPbCjloy01JYOKVgUJbXH0GwDphhZlPMLAO4FljRq88Kug8GA3wSWOnubmYjgN8DN7v7S/1Qi4hIwvvLzhoWTikgKz11UJYXcxBEx/y/DDwDbAUec/fNZnabmV0R7XYPMMrMyoGvA2+dYvplYDrwL2b2WvRrdKw1iYgkqkP1beysbubcQRoWAuiX29m5+9PA073a/qXH63bgU3383O3A7f1Rg4hIMvjLzu6TYT48SAeKYYgcLBYRkW7Pb61mbH4WM8fkDtoyFQQiIkNEQ1sXq7fXcNm84gG/rURPCgIRkSHimc2H6QxHuGL+uEFdroJARGSI+N3rh5g0KpuTB/CxlH1REIiIDAHVTe28VF7LFSePG9RhIVAQiIgMCU+/UUnE4YqTB3dYCBQEIiJDworXDzF7bB4zxuQN+rIVBCIicVZR18qG/fWDfpD4LQoCEZE4e6ys+wbOHzspPkHQL1cWi4jI8dtT28LtT23h+W3VnDuziAkF2XGpQ0EgIjKIwhFn7Z46fvfGIR4vqyAzLZVbLp3N5z40OW41KQhERPpRS0eIV3cf4VBDOw2tndS3dtHSGaajK0xrZ5iyfUepbe4gKz2FqxaU8PcXz2R03sA/hezdKAhERGIUiTiPrKvg6U2VrN1TR2c48va8Yemp5GSmkZWeQlZ6KqdPKeCyecWcP7uI7Iyh8RY8NKoQEUlQHaEw33j8DX73+iGmFeWw5KxJnD97NNNH5zJ8WDqZaYPzTIFYKAhERD6ghrYuvvBgGa/uruPmS2fzhXOmDvpVwf1BQSAi8h7CEWdXTTObDjSwtbKRhrYumjtCbD7USGVDGz+5Zj5XLhgf7zI/MAWBiARSZyjC6wfq2VHVRMQBd0IRp7UzTEtHiIa2Lg7Wt3HwaBsHjrbR1hUGIDMthYKcDHIz0xiTn8n3r5rHh6YXxndlYqQgEJHAOHC0lT9trmL1jhrW7al7+829t7QUIy8rjXEjhjGlMIdzZhZx4rh8TioZzpTCXFJTEm/4590oCEQkqbV3hXlk7X6e2HCANw82AjCtKIerS0s4a3oh88YPJy3VMIzUFCMnM5WM1JSEHOv/oBQEIpKU3gqAn6/eRXVTByeXDOeWS2dz0YljmVKYE+/yhhQFgYgkjUjEWbu3jidfO8jv36iksT3E6VMKuPPaBZw5bVS8yxuyFAQikvDcnVXbq/nu77eyq6aF7IxULjlxLFefNoEzpioA3ouCQEQS2vbDTXznqS28WF7L1MIcfnzNyVx84tghc9VuItC/lIgkpFA4wi/+vIufPLeTnMw0vv2xOfzVGZNIT9Xd9Y9Xv/yLmdklZrbdzMrN7OY+5mea2aPR+WvMbHK0fZSZrTKzZjO7qz9qEZHkt6e2hU/98hV++KcdXDJ3LKu/cR43fGiKQuADinmPwMxSgZ8BFwIHgHVmtsLdt/TodiNw1N2nm9m1wA+Aa4B24J+BudEvEZE+RSLOS7tqeejV/Ty7tYqcjFR+et2CuDzjN9n0x9DQQqDc3XcDmNkjwGKgZxAsBm6Nvn4CuMvMzN1bgBfNbHo/1CEiSepgfRs33LeWHVXNjMxO56azp/D5s6cwJj++t29OFv0RBOOBih7TB4DTj9XH3UNm1gCMAmrf70LMbCmwFGDixImx1CsiCaSyoY3rlr3K0dZOfnzNyVw2rzgh7uiZSBJmQM3dl7l7qbuXFhUVxbscERkEhxvau0OgpZMHbzydjy8oUQgMgP7YIzgITOgxXRJt66vPATNLA4YDR/ph2SKSgDx6g7eIO+4Qiji1TR1UN3VQ2dDG/iOt7D3Syiu7amlo6+KBG09n/oQR8S47afVHEKwDZpjZFLrf8K8FPt2rzwpgCfAK8Elgpbt7PyxbRBLApgMN3P/yXlZvr6a1M0x7KMx7vQOMyc9kamEu37h4FqdOGjk4hQZUzEEQHfP/MvAMkArc6+6bzew2oMzdVwD3AA+aWTlQR3dYAGBme4F8IMPMrgQu6nXGkYgkoEjE+dOWKpa9sIsN++vfvtp3VG4GWendN3ZLid7FMzXFKMzNZEx+JqPzsphQMEwXhA0iS8QP5qWlpV5WVhbvMkSkD+GI8/SmSu5aWc72qiYmjcpmyZmT+WRpCflZ6fEuL9DMbL27l/ZuV+SKSL9wd57ZfJgfPbuDHVXNTB+dy53XzufyecWk6UKvIU1BICLHzd15YWctWw41EgpHCEWclduq2XSwgWlFOfzHdQu4fF7x20M/MrQpCETkfWvvCvPkawe5+y972Fnd/I55k0Zl88NPncyV88dpDyDBKAhE5H2pqGvlc/etZVdNCycU5/Ojq7vv8pmZlkJqigXqiV7JRkEgIu/ptYp6blq+jq6wc9/nTuO8WUV6408iCgIR6VMoHGHvkRZe3V3H7b/fQlFeJo/esJBpRbnxLk36mYJARN7hSHMH//DEG7xUXktHKALA/AkjuHtJKYW5mXGuTgaCgkBE3rarppkb7ltHVWM7nzl9EnPH5zN7bD6zxuaRqjOAkpaCQERo7gixbk8dX3v0NdJSjIeXnsEpE3Vbh6BQEIgEUDji/OHNSpa/vJed1c3Ut3YBMK0oh/tvWMiEguw4VyiDSUEgEiCdoQgrXj/Ez1eXs7umhamFOVw+r5iSkdmUjBzGebOKyNNtIAJHQSASAA2tXTy0dh/LX95LVWMHs8fmcdenF3Dp3GKN/YuCQCTZvVRey9IHymjpDHP29ELuuOokXQcg76AgEEli6/cd5a8fKGPCyGx+cu18TijOj3dJMgQpCESS1JZDjdxw31pG52Xy4E0LGZ2nB71L33RnKJEktLummevvXUNOZhr/ddPpCgF5VwoCkSRT2dDGZ+9Zizv8102nUzJSp4LKu9PQkEgSOdrSyfX3rKWhrYtHlp6h+wLJ+6IgEEkSrZ0hbrh/HfvqWll+w0Lmjh8e75IkQWhoSCRJ/HzVLl4/UM9/XLeAM6eNinc5kkAUBCJJoL61k/tf3stlc4u5+MSx8S5HEoyCQCQJ3PPiHpo7QnzlIzPiXYokIAWBSIKrb+3kvpf2ctm8scwamxfvciQBKQhEEty92huQGPVLEJjZJWa23czKzezmPuZnmtmj0flrzGxyj3m3RNu3m9nF/VGPSFA0tHZx30t7uXTuWGaP1e0j5IOJOQjMLBX4GXApMAe4zszm9Op2I3DU3acDPwZ+EP3ZOcC1wInAJcDPo79PRN6He1/aQ5P2BiRG/bFHsBAod/fd7t4JPAIs7tVnMbA8+voJ4CPWfevDxcAj7t7h7nuA8ujvE5H30NjexX0v7eGiOWN0MzmJSX8EwXigosf0gWhbn33cPQQ0AKPe588CYGZLzazMzMpqamr6oWyRxPbgK/tobA/xt4u0NyCxSZiDxe6+zN1L3b20qKgo3uWIxFVLR4i7/7Kb82cVMa9EVxBLbPojCA4CE3pMl0Tb+uxjZmnAcODI+/xZEenloTX7ONraxd/q2ID0g/4IgnXADDObYmYZdB/8XdGrzwpgSfT1J4GV7u7R9mujZxVNAWYAa/uhJpGk1d4VZtkLezh7eiGnTBwZ73IkCcR80zl3D5nZl4FngFTgXnffbGa3AWXuvgK4B3jQzMqBOrrDgmi/x4AtQAj4kruHY61JJFk1tHXxr7/bTG1zB3+7aEG8y5EkYd0fzBNLaWmpl5WVxbsMkUHj7vxmw0G+/4et1LV08tcfnsotl50Q77IkwZjZencv7d2u21CLDHHbDjfyL7/dzNq9dZw8YQT36xbT0s8UBCJDVFN7Fz9+difLX9lLXlYa379qHteUTiAlxeJdmiQZBYHIEFRe3cRNy8vYV9fKdQsn8g8XzWJkTka8y5IkpSAQGWJWba/mK7/eSGZ6Co994UxOm1wQ75IkySkIRIYAd2dLZSO/3XiQe17cw+yx+fxqSSnjRwyLd2kSAAoCkThwd/bXtfJaRT0b99ezans1+460kmJwxcnj+N5V88jO0H9PGRz6SxMZZEeaO/ire9aytbIRgGHpqZw2pYC/OXcaF84Zw6jczDhXKEGjIBAZRI3tXVx/71p21zRz68fmsHDKKGaOySUtNWFu+yVJSEEgMkjaOsPceP86dlQ1sez6Us6fNTreJYkACgKRAePubD7UyJbKRvbUtvDizlo2H2rgP647RSEgQ4qCQKQf1LV00tYVJhJxmtpDPLP5ML997SD7jrQCkJ5qTCzI5kdXz+fyk4rjXK3IOykIRD6gts4wT2+q5NGyCtbuqXvHPDM4c+oovnTedE6bUsCEkcN0HECGLAWByLvoCIXZuL+eupZOjrZ2UtvUyd4jLeyubWFnVROtnWEmj8rm6xfOZGx+FikpRlqKcfrUAoqH6xoASQwKApE+tHSE+PWa/dz94m6qGjveMW/8iGFMLszmk6eWcPm8YhZOKaD7EdwiiUlBIAJU1LXy+PoDVDe2U9XYzsaKeupbuzhr2ihuWzyXSaOyGZmdwYjsdDLTUuNdrki/UhBI4LV3hfn8/esor2mmMDeTsflZnDuziM+dNZkFegKYBICCQALvJ8/tZGd1M8s/v5BzZxbFuxyRQafTGCTQNu4/yrIXdnFN6QSFgASWgkACq70rzDcef50x+Vl886N67KMEl4aGJJAiEefWFZvZVdPCA59fSH5WerxLEokbBYEETigc4R//+w1+s+EgXzxvGudoSEgCTkEggdIRCvPVh1/jj5sP8/cXzuTLi6bHuySRuFMQSGCs21vHd57awhsHGvj2x+Zww4emxLskkSFBQSBJrbG9i51Vzfzqhd38cfNhxuRn8rNPn6Ibv4n0EFMQmFkB8CgwGdgLXO3uR/votwT4VnTydndfHm3/LnA9MNLdc2OpRZLbtsONVDd20NwRork9RHsoTHtXmPauCHUtndQ0d1DT1EFHKIK7E444VY0d1DZ33x4iOyOVr184k5s+PEWPgBTpJdb/ETcDz7v7HWZ2c3T6n3p2iIbFt4FSwIH1ZrYiGhi/A+4CdsZYhySpHVVNfPf3W/nzjppj9snLSqMoL5PC3ExGDEsnxcDMmFOcz9SiXKYW5XDqpJEU6hGQIn2KNQgWA+dFXy8HVtMrCICLgWfdvQ7AzJ4FLgEedvdXo20xliHJpKUjxIb9R3l602EeXbefnMw0brl0NqdOGkluVho5GWkMy0glKz2VzLQU0nV7Z5GYxBoEY9y9Mvr6MDCmjz7jgYoe0weibcfFzJYCSwEmTpx4vD8uCeDlXbX84A/bePNQI+GIk5ZifPaMSXz1gpkU5GTEuzyRpPWeQWBmzwFj+5j1zZ4T7u5m5v1VWG/uvgxYBlBaWjpgy5HBF444d60s587ndzCxIJsvnTeN0skFLJg4gjxd6CUy4N4zCNz9gmPNM7MqMyt290ozKwaq++h2kP8dPgIooXsISQLM3TnU0M6WQ40sf3kvL5bX8vEF47n9yrnkZOpgrshgivV/3ApgCXBH9PuTffR5Bviemb11P9+LgFtiXK4McUdbOnlqUyVbDjUSjkQIR7ov5qpr6aSupZPKhnYa2roAyEpP4QefmMfVpRN0vEgkDmINgjuAx8zsRmAfcDWAmZUC/8/db3L3OjP7DrAu+jO39Thw/G/Ap4FsMzsA3O3ut8ZYk8RJOOKs3FbNY2UVrN5eTVfYGZWTQXpqCqkpRkZaCgU5GZSMzObUSSOZXZzPnOJ8Zo/N016ASByZe+INt5eWlnpZWVm8y5Co+tZOHi87wAOv7qWiro3ReZksnj+Ojy8o4YTiPH3KFxkizGy9u5f2btfHMPlA2rvCrNxWzW83HmRV9NP/wikF3HLpCVw0ZwxpOqVTJGEoCOS4tHaGePCVfSx7YTdHWjopysvk+jMn84lTSpgzLj/e5YnIB6AgkPelIxTmgZf38Ys/7+JISycfnlHI0nOmcta0QlJTNPQjksgUBPKu3J1nNlfxvae3sr+ulQ/PKORrF8zk1El6qLtIslAQCACHG9pZv+8orx+o582DDbR2hnG6b/dQXt3MzDG5PPD5hXqIi0gSUhAID766j1tXbCYccTLSUjihOJ/hw7qv6B2Znc6SMydx3cKJOgAskqQUBAEWjjjfe3or97y4h0WzR/N3F8xk1tg8MtL0hi8SJAqCAHF3Dta3UVHXRsXRVv6wqZJV22v43FmT+eePztFBX5GAUhAksY5QmHV7jvJieS2bDtaz6UADje2ht+dnpKbokY0ioiBIRuXVzfzbH7fxYnktrZ1h0lON2WPzufykccwdn8/kUTmUjBxG8fBhGgYSEQVBsqmoa+XTv3qVznCEq04Zz/mzRnPmtFF6PKOIHJPeHZJIdVM7f3XPGjpCER77wpnMGpsX75JEJAFoXCBJVDe1c/09a6lp6uC+G05TCIjI+6Y9ggTWFY7w3JYq/nvDQVZvr8YM7v3caZwyUVf9isj7pyBIUO1dYW5cvo6Xyo8wOi+TG8+ewqdKJzB9dG68SxORBKMgSECdoQhffGgDL+86wnc/PpdrT5uoawBE5ANTECSYUDjC1x7dyMpt1Xz343P5zOmT4l2SiCQ4HSxOIO7OPz+5mac3HeZbl5+gEBCRfqEgSCC/Xrufh9fu52/Om8ZNH54a73JEJEkoCBLE+n1HuXXFZs6ZWcQ3LpoV73JEJIkoCBJAdWM7f/Nf6ykePoyfXjtfB4ZFpF8pCIa4irpWrr93LU3tIX752VMZkZ0R75JEJMnorKEh7JVdR/jiQ+sJRZxffvZUTijWw+FFpP/FFARmVgA8CkwG9gJXu/vRPvotAb4Vnbzd3ZebWTbwODANCAO/c/ebY6kn0TW0dbGtspF9R1rZeriRB17Zx+RR2fzq+lKmFulCMREZGLHuEdwMPO/ud5jZzdHpf+rZIRoW3wZKAQfWm9kKoAP4obuvMrMM4Hkzu9Td/xBjTQknHHEeWrOPf//jdpo6up8XkJZiXHziGH7wiZPIy0qPc4UiksxiDYLFwHnR18uB1fQKAuBi4Fl3rwMws2eBS9z9YWAVgLt3mtkGoCTGehLOlkON3PI/m3i9op4PzyjkxrOnMLUwl3EjsvSMYBEZFLEGwRh3r4y+PgyM6aPPeKCix/SBaNvbzGwE8DHgzhjrSShvHKjn6l++Qm5mGndeO58rTh6Hmc4IEpHB9Z5BYGbPAWP7mPXNnhPu7mbmx1uAmaUBDwM/dffd79JvKbAUYOLEice7mCHnUH0bNy4vY1ROJv/zpbMYnZcV75JEJKDeMwjc/YJjzTOzKjMrdvdKMysGqvvodpD/HT6C7uGf1T2mlwE73f0n71HHsmhfSktLjztwhpLmjhA3Li+jvTPMQ188XSEgInEV6yD0CmBJ9PUS4Mk++jwDXGRmI81sJHBRtA0zux0YDnwtxjoSRktHiC//egM7qpq46zOnMHOMHiAjIvEVaxDcAVxoZjuBC6LTmFmpmd0NED1I/B1gXfTrNnevM7MSuoeX5gAbzOw1M7spxnqGtG2HG7nirhf5844avrN4LufOLIp3SSIimHvijbKUlpZ6WVlZvMt4V5GIU17TTH1rF62dIXZWNfPDP20nf1g6d14zn7OmF8a7RBEJGDNb7+6lvdt1ZXE/ikScP22p4rmtVazeXkNtc8c75p89vZAfXzOforzMOFUoIvJ/KQj6yeGGdr7+2Gu8vOsI+VlpnDOziHNnFlE8fBjDMlLIzUxnxuhcUnTDOBEZYhQEMereCzjMzb/ZREdXhO9fNY9PnVqii8FEJGEoCN6nrnCEupZOtlY2su1wE9sPN7GzuondNS20doY5qWQ4P7lmvu4JJCIJR0HwLh4vq+C2p7bQ1hkmFHnnQfXi4VlMH53LNacVcEJxPlfOH09GmvYCRCTxKAjexSPrKsjPSuezZ0xiWHoqw7PTmTkmjxPG5jM8WzeCE5HkoCA4hrqWTjbsP8pXFs3g7y6cGe9yREQGjMYyjmH19mrc4SMnjI53KSIiA0pBcAwrt1VTlJfJ3HHD412KiMiAUhD0oSsc4c87alg0a7TO+xeRpKcg6EPZ3qM0tYdYpGEhEQkABUEfVm6rIiM1hbN1PyARCQAFQR+e31bN6VMLyMnUSVUikvwUBL3sqW1hd00LH5mtYSERCQYFQS8rt3U/ZG3R7L4evywiknwUBL2s2lbN9NG5TByVHe9SREQGhYKgh+aOEGv2HGGRhoVEJEAUBD28uLOWrrBz/iwFgYgEh4Kgh1XbqsnLSqN08sh4lyIiMmgUBFHuzqrt1Zwzo4h0PVRGRAJE73hRmw81Ut3Uwfk6PiAiAaMgiFq5rRozOG9WUbxLEREZVAqCqJXbqjmpZASFuZnxLkVEZFApCIAjzR28fqCeRTpbSEQCKKYgMLMCM3vWzHZGv/d5uo2ZLYn22WlmS3q0/9HMXjezzWb2CzNLjaWeD+rPO2pwR9cPiEggxbpHcDPwvLvPAJ6PTr+DmRUA3wZOBxYC3+4RGFe7+8nAXKAI+FSM9Xwgq7fXUJibyYnj8uOxeBGRuIo1CBYDy6OvlwNX9tHnYuBZd69z96PAs8AlAO7eGO2TBmQAHmM9x83deXX3Ec6aNkoPoRGRQIo1CMa4e2X09WGgrzu1jQcqekwfiLYBYGbPANVAE/BEjPUct/11rVQ3dbBwSsFgL1pEZEh4zxvum9lzwNg+Zn2z54S7u5kd9yd6d7/YzLKAh4BFdO8x9FXHUmApwMSJE493Mce0dk8dgIJARALrPYPA3S841jwzqzKzYnevNLNiuj/Z93YQOK/HdAmwutcy2s3sSbqHmvoMAndfBiwDKC0t7bchpLV76hiRnc70otz++pUiIgkl1qGhFcBbZwEtAZ7so88zwEVmNjJ6kPgi4Bkzy42GB2aWBlwObIuxnuO2bm8dp00u0PEBEQmsWIPgDuBCM9sJXBCdxsxKzexuAHevA74DrIt+3RZtywFWmNkbwGt07038IsZ6jkt1Yzt7j7SycLKGhUQkuGJ6KK+7HwE+0kd7GXBTj+l7gXt79akCTotl+bFau7f7+MBpOj4gIgEW6CuL1+2pIzsjVdcPiEigBToI1uyp45SJI3XbaREJtMC+Aza0dbG9qonTdHxARAIusEGwfl8d7rp+QEQksEGwZk8d6anGgokj4l2KiEhcBTIIDtW38duNB5k/YQRZ6XG54amIyJARuCCob+1kyb1rae0Ic+sVJ8a7HBGRuIvpOoJE09YZ5vP3r2NfXSvLb1jIieOGx7skEZG4C8weQSgc4Uu/3sDGinp+eu18zpw2Kt4liYgMCYHZI0gxY2phDotmz+WSucXxLkdEZMgIThCkGN/66Jx4lyEiMuQEZmhIRET6piAQEQk4BYGISMApCEREAk5BICIScAoCEZGAUxCIiAScgkBEJODM3eNdw3Ezsxpg3wf88UKgth/LSQRBXGcI5noHcZ0hmOv9QdZ5krsX9W5MyCCIhZmVuXtpvOsYTEFcZwjmegdxnSGY692f66yhIRGRgFMQiIgEXBCDYFm8C4iDIK4zBHO9g7jOEMz17rd1DtwxAhEReacg7hGIiEgPCgIRkYALTBCY2SVmtt3Mys3s5njXM1DMbIKZrTKzLWa22cy+Gm0vMLNnzWxn9PvIeNfa38ws1cw2mtlT0ekpZrYmus0fNbOMeNfY38xshJk9YWbbzGyrmZ2Z7NvazP4u+rf9ppk9bGZZybitzexeM6s2szd7tPW5ba3bT6Pr/4aZnXI8ywpEEJhZKvAz4FJgDnCdmSXr48pCwN+7+xzgDOBL0XW9GXje3WcAz0enk81Xga09pn8A/NjdpwNHgRvjUtXAuhP4o7vPBk6me/2Tdlub2XjgK0Cpu88FUoFrSc5tfT9wSa+2Y23bS4EZ0a+lwH8ez4ICEQTAQqDc3Xe7eyfwCLA4zjUNCHevdPcN0ddNdL8xjKd7fZdHuy0HroxLgQPEzEqAy4G7o9MGLAKeiHZJxnUeDpwD3APg7p3uXk+Sb2u6H7E7zMzSgGygkiTc1u7+AlDXq/lY23Yx8IB3exUYYWbv++HsQQmC8UBFj+kD0bakZmaTgQXAGmCMu1dGZx0GxsSrrgHyE+AfgUh0ehRQ7+6h6HQybvMpQA1wX3RI7G4zyyGJt7W7HwR+COynOwAagPUk/7Z+y7G2bUzvcUEJgsAxs1zgv4GvuXtjz3nefc5w0pw3bGYfBardfX28axlkacApwH+6+wKghV7DQEm4rUfS/el3CjAOyOH/Dp8EQn9u26AEwUFgQo/pkmhbUjKzdLpD4CF3/020ueqtXcXo9+p41TcAPgRcYWZ76R72W0T32PmI6PABJOc2PwAccPc10ekn6A6GZN7WFwB73L3G3buA39C9/ZN9W7/lWNs2pve4oATBOmBG9MyCDLoPLq2Ic00DIjo2fg+w1d1/1GPWCmBJ9PUS4MnBrm2guPst7l7i7pPp3rYr3f0zwCrgk9FuSbXOAO5+GKgws1nRpo8AW0jibU33kNAZZpYd/Vt/a52Telv3cKxtuwK4Pnr20BlAQ48hpPfm7oH4Ai4DdgC7gG/Gu54BXM+z6d5dfAN4Lfp1Gd1j5s8DO4HngIJ41zpA638e8FT09VRgLVAOPA5kxru+AVjf+UBZdHv/FhiZ7Nsa+FdgG/Am8CCQmYzbGniY7uMgXXTv/d14rG0LGN1nRu4CNtF9VtX7XpZuMSEiEnBBGRoSEZFjUBCIiAScgkBEJOAUBCIiAacgEBEJOAWBiEjAKQhERALu/wPJz1Txc50WzwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "plt.plot(net.vs[0].data.cpu().sort().values.numpy());"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "approximate-hollywood",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}