train_plot_images.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Our model with metric loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing Triplet Generation ...\n",
      "Done!\n",
      "Processing Triplet Generation ...\n",
      "Done!\n",
      "Train Epoch: 1 [0/60000]\tVAE Loss: 548.9819 (548.9819) \tMetric Loss: 0.1644 (0.1644) \tMetric Acc: 65.00% (65.00%) \tEmb_Norm: 4.01 (4.01)\n",
      "\n",
      "Test set: Average VAE loss: 522.4220, Average Metric loss: 0.1284, Metric Accuracy: 74.10%\n",
      "\n",
      "Train Epoch: 1 [6000/60000]\tVAE Loss: 206.6727 (427.4733) \tMetric Loss: 0.0248 (0.1016) \tMetric Acc: 94.17% (79.50%) \tEmb_Norm: 9.15 (6.99)\n",
      "\n",
      "Test set: Average VAE loss: 344.6516, Average Metric loss: 0.0762, Metric Accuracy: 84.61%\n",
      "\n",
      "Train Epoch: 1 [12000/60000]\tVAE Loss: 201.9872 (318.7520) \tMetric Loss: 0.0170 (0.0680) \tMetric Acc: 97.50% (86.33%) \tEmb_Norm: 14.11 (8.37)\n",
      "\n",
      "Test set: Average VAE loss: 290.5721, Average Metric loss: 0.0593, Metric Accuracy: 88.19%\n",
      "\n",
      "Train Epoch: 1 [18000/60000]\tVAE Loss: 179.0683 (278.1027) \tMetric Loss: 0.0307 (0.0567) \tMetric Acc: 95.00% (88.93%) \tEmb_Norm: 27.74 (11.31)\n",
      "\n",
      "Test set: Average VAE loss: 261.1336, Average Metric loss: 0.0553, Metric Accuracy: 89.77%\n",
      "\n",
      "Train Epoch: 1 [24000/60000]\tVAE Loss: 164.6855 (252.8452) \tMetric Loss: 0.0433 (0.0551) \tMetric Acc: 93.33% (90.09%) \tEmb_Norm: 32.29 (15.81)\n",
      "\n",
      "Test set: Average VAE loss: 240.8197, Average Metric loss: 0.0561, Metric Accuracy: 90.40%\n",
      "\n",
      "Train Epoch: 1 [30000/60000]\tVAE Loss: 156.9132 (234.7782) \tMetric Loss: 0.0675 (0.0563) \tMetric Acc: 92.50% (90.56%) \tEmb_Norm: 34.15 (19.27)\n",
      "\n",
      "Test set: Average VAE loss: 225.9077, Average Metric loss: 0.0565, Metric Accuracy: 90.80%\n",
      "\n",
      "Train Epoch: 1 [36000/60000]\tVAE Loss: 147.6596 (221.2490) \tMetric Loss: 0.0455 (0.0566) \tMetric Acc: 95.00% (90.91%) \tEmb_Norm: 34.43 (21.76)\n",
      "\n",
      "Test set: Average VAE loss: 214.2991, Average Metric loss: 0.0566, Metric Accuracy: 91.08%\n",
      "\n",
      "Train Epoch: 1 [42000/60000]\tVAE Loss: 140.4265 (210.6273) \tMetric Loss: 0.0742 (0.0568) \tMetric Acc: 88.33% (91.13%) \tEmb_Norm: 34.85 (23.64)\n",
      "\n",
      "Test set: Average VAE loss: 205.0127, Average Metric loss: 0.0567, Metric Accuracy: 91.26%\n",
      "\n",
      "Train Epoch: 1 [48000/60000]\tVAE Loss: 141.3883 (202.0099) \tMetric Loss: 0.0409 (0.0567) \tMetric Acc: 93.33% (91.32%) \tEmb_Norm: 34.81 (25.09)\n",
      "\n",
      "Test set: Average VAE loss: 197.3815, Average Metric loss: 0.0564, Metric Accuracy: 91.45%\n",
      "\n",
      "Train Epoch: 1 [54000/60000]\tVAE Loss: 133.8448 (194.8705) \tMetric Loss: 0.0885 (0.0562) \tMetric Acc: 87.50% (91.49%) \tEmb_Norm: 35.84 (26.23)\n",
      "\n",
      "Test set: Average VAE loss: 190.9475, Average Metric loss: 0.0559, Metric Accuracy: 91.59%\n",
      "\n",
      "Train Epoch: 2 [0/60000]\tVAE Loss: 133.3964 (133.3964) \tMetric Loss: 0.0502 (0.0502) \tMetric Acc: 94.17% (94.17%) \tEmb_Norm: 34.91 (34.91)\n",
      "\n",
      "Test set: Average VAE loss: 131.5912, Average Metric loss: 0.0490, Metric Accuracy: 93.35%\n",
      "\n",
      "Train Epoch: 2 [6000/60000]\tVAE Loss: 128.9171 (131.4645) \tMetric Loss: 0.0689 (0.0453) \tMetric Acc: 86.67% (93.62%) \tEmb_Norm: 35.59 (35.74)\n",
      "\n",
      "Test set: Average VAE loss: 130.5880, Average Metric loss: 0.0469, Metric Accuracy: 93.46%\n",
      "\n",
      "Train Epoch: 2 [12000/60000]\tVAE Loss: 126.6299 (130.3804) \tMetric Loss: 0.0549 (0.0457) \tMetric Acc: 92.50% (93.59%) \tEmb_Norm: 34.92 (35.71)\n",
      "\n",
      "Test set: Average VAE loss: 129.7473, Average Metric loss: 0.0462, Metric Accuracy: 93.48%\n",
      "\n",
      "Train Epoch: 2 [18000/60000]\tVAE Loss: 128.0024 (129.4531) \tMetric Loss: 0.0184 (0.0455) \tMetric Acc: 96.67% (93.56%) \tEmb_Norm: 35.49 (35.62)\n",
      "\n",
      "Test set: Average VAE loss: 128.8857, Average Metric loss: 0.0455, Metric Accuracy: 93.55%\n",
      "\n",
      "Train Epoch: 2 [24000/60000]\tVAE Loss: 128.0261 (128.7379) \tMetric Loss: 0.0429 (0.0453) \tMetric Acc: 95.00% (93.54%) \tEmb_Norm: 34.54 (35.54)\n",
      "\n",
      "Test set: Average VAE loss: 128.2310, Average Metric loss: 0.0458, Metric Accuracy: 93.47%\n",
      "\n",
      "Train Epoch: 2 [30000/60000]\tVAE Loss: 121.9922 (128.0245) \tMetric Loss: 0.0433 (0.0454) \tMetric Acc: 93.33% (93.50%) \tEmb_Norm: 35.24 (35.44)\n",
      "\n",
      "Test set: Average VAE loss: 127.5575, Average Metric loss: 0.0456, Metric Accuracy: 93.48%\n",
      "\n",
      "Train Epoch: 2 [36000/60000]\tVAE Loss: 121.4592 (127.4029) \tMetric Loss: 0.0248 (0.0452) \tMetric Acc: 96.67% (93.49%) \tEmb_Norm: 34.93 (35.42)\n",
      "\n",
      "Test set: Average VAE loss: 126.9946, Average Metric loss: 0.0451, Metric Accuracy: 93.51%\n",
      "\n",
      "Train Epoch: 2 [42000/60000]\tVAE Loss: 121.3444 (126.8043) \tMetric Loss: 0.0316 (0.0447) \tMetric Acc: 94.17% (93.54%) \tEmb_Norm: 34.95 (35.34)\n",
      "\n",
      "Test set: Average VAE loss: 126.4187, Average Metric loss: 0.0445, Metric Accuracy: 93.59%\n",
      "\n",
      "Train Epoch: 2 [48000/60000]\tVAE Loss: 122.8093 (126.2365) \tMetric Loss: 0.0300 (0.0441) \tMetric Acc: 95.00% (93.61%) \tEmb_Norm: 34.54 (35.29)\n",
      "\n",
      "Test set: Average VAE loss: 125.8841, Average Metric loss: 0.0440, Metric Accuracy: 93.62%\n",
      "\n",
      "Train Epoch: 2 [54000/60000]\tVAE Loss: 119.2118 (125.7019) \tMetric Loss: 0.0673 (0.0436) \tMetric Acc: 91.67% (93.65%) \tEmb_Norm: 34.67 (35.24)\n",
      "\n",
      "Test set: Average VAE loss: 125.3956, Average Metric loss: 0.0434, Metric Accuracy: 93.67%\n",
      "\n",
      "Train Epoch: 3 [0/60000]\tVAE Loss: 119.9634 (119.9634) \tMetric Loss: 0.0176 (0.0176) \tMetric Acc: 97.50% (97.50%) \tEmb_Norm: 34.68 (34.68)\n",
      "\n",
      "Test set: Average VAE loss: 120.5615, Average Metric loss: 0.0397, Metric Accuracy: 94.37%\n",
      "\n",
      "Train Epoch: 3 [6000/60000]\tVAE Loss: 121.1430 (120.7473) \tMetric Loss: 0.0199 (0.0356) \tMetric Acc: 96.67% (94.76%) \tEmb_Norm: 34.58 (34.59)\n",
      "\n",
      "Test set: Average VAE loss: 120.1614, Average Metric loss: 0.0377, Metric Accuracy: 94.46%\n",
      "\n",
      "Train Epoch: 3 [12000/60000]\tVAE Loss: 118.4633 (120.1375) \tMetric Loss: 0.0152 (0.0359) \tMetric Acc: 96.67% (94.64%) \tEmb_Norm: 34.10 (34.54)\n",
      "\n",
      "Test set: Average VAE loss: 119.8374, Average Metric loss: 0.0373, Metric Accuracy: 94.43%\n",
      "\n",
      "Train Epoch: 3 [18000/60000]\tVAE Loss: 118.3275 (119.7163) \tMetric Loss: 0.0269 (0.0364) \tMetric Acc: 95.00% (94.56%) \tEmb_Norm: 33.86 (34.41)\n",
      "\n",
      "Test set: Average VAE loss: 119.5614, Average Metric loss: 0.0371, Metric Accuracy: 94.44%\n",
      "\n",
      "Train Epoch: 3 [24000/60000]\tVAE Loss: 117.2759 (119.5453) \tMetric Loss: 0.0184 (0.0364) \tMetric Acc: 95.83% (94.52%) \tEmb_Norm: 33.85 (34.28)\n",
      "\n",
      "Test set: Average VAE loss: 119.3736, Average Metric loss: 0.0372, Metric Accuracy: 94.40%\n",
      "\n",
      "Train Epoch: 3 [30000/60000]\tVAE Loss: 120.9721 (119.3007) \tMetric Loss: 0.0253 (0.0365) \tMetric Acc: 95.83% (94.49%) \tEmb_Norm: 33.77 (34.19)\n",
      "\n",
      "Test set: Average VAE loss: 119.1325, Average Metric loss: 0.0368, Metric Accuracy: 94.44%\n",
      "\n",
      "Train Epoch: 3 [36000/60000]\tVAE Loss: 121.6988 (119.0876) \tMetric Loss: 0.0216 (0.0363) \tMetric Acc: 95.83% (94.50%) \tEmb_Norm: 34.06 (34.16)\n",
      "\n",
      "Test set: Average VAE loss: 118.9323, Average Metric loss: 0.0364, Metric Accuracy: 94.50%\n",
      "\n",
      "Train Epoch: 3 [42000/60000]\tVAE Loss: 121.4201 (118.8824) \tMetric Loss: 0.0555 (0.0361) \tMetric Acc: 91.67% (94.53%) \tEmb_Norm: 33.80 (34.15)\n",
      "\n",
      "Test set: Average VAE loss: 118.7542, Average Metric loss: 0.0363, Metric Accuracy: 94.51%\n",
      "\n",
      "Train Epoch: 3 [48000/60000]\tVAE Loss: 117.1634 (118.6995) \tMetric Loss: 0.0333 (0.0361) \tMetric Acc: 93.33% (94.52%) \tEmb_Norm: 33.19 (34.10)\n",
      "\n",
      "Test set: Average VAE loss: 118.6002, Average Metric loss: 0.0362, Metric Accuracy: 94.50%\n",
      "\n",
      "Train Epoch: 3 [54000/60000]\tVAE Loss: 115.9415 (118.5514) \tMetric Loss: 0.0333 (0.0359) \tMetric Acc: 94.17% (94.53%) \tEmb_Norm: 33.36 (34.04)\n",
      "\n",
      "Test set: Average VAE loss: 118.4423, Average Metric loss: 0.0359, Metric Accuracy: 94.52%\n",
      "\n",
      "Train Epoch: 4 [0/60000]\tVAE Loss: 113.3122 (113.3122) \tMetric Loss: 0.0089 (0.0089) \tMetric Acc: 98.33% (98.33%) \tEmb_Norm: 33.18 (33.18)\n",
      "\n",
      "Test set: Average VAE loss: 116.2482, Average Metric loss: 0.0370, Metric Accuracy: 94.24%\n",
      "\n",
      "Train Epoch: 4 [6000/60000]\tVAE Loss: 119.6212 (116.6157) \tMetric Loss: 0.0147 (0.0328) \tMetric Acc: 96.67% (94.84%) \tEmb_Norm: 32.97 (33.19)\n",
      "\n",
      "Test set: Average VAE loss: 116.3493, Average Metric loss: 0.0347, Metric Accuracy: 94.59%\n",
      "\n",
      "Train Epoch: 4 [12000/60000]\tVAE Loss: 117.0124 (116.3986) \tMetric Loss: 0.0181 (0.0322) \tMetric Acc: 95.83% (94.92%) \tEmb_Norm: 33.42 (33.32)\n",
      "\n",
      "Test set: Average VAE loss: 116.1970, Average Metric loss: 0.0337, Metric Accuracy: 94.72%\n",
      "\n",
      "Train Epoch: 4 [18000/60000]\tVAE Loss: 120.9662 (116.1721) \tMetric Loss: 0.0271 (0.0323) \tMetric Acc: 95.00% (94.92%) \tEmb_Norm: 32.89 (33.30)\n",
      "\n",
      "Test set: Average VAE loss: 116.0044, Average Metric loss: 0.0338, Metric Accuracy: 94.71%\n",
      "\n",
      "Train Epoch: 4 [24000/60000]\tVAE Loss: 113.8136 (115.9508) \tMetric Loss: 0.0382 (0.0328) \tMetric Acc: 93.33% (94.84%) \tEmb_Norm: 33.76 (33.30)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Test set: Average VAE loss: 115.8772, Average Metric loss: 0.0333, Metric Accuracy: 94.77%\n",
      "\n",
      "Train Epoch: 4 [30000/60000]\tVAE Loss: 116.8738 (115.8832) \tMetric Loss: 0.0226 (0.0325) \tMetric Acc: 96.67% (94.88%) \tEmb_Norm: 33.27 (33.33)\n",
      "\n",
      "Test set: Average VAE loss: 115.7960, Average Metric loss: 0.0329, Metric Accuracy: 94.82%\n",
      "\n",
      "Train Epoch: 4 [36000/60000]\tVAE Loss: 116.9307 (115.7676) \tMetric Loss: 0.0134 (0.0322) \tMetric Acc: 97.50% (94.90%) \tEmb_Norm: 33.13 (33.32)\n",
      "\n",
      "Test set: Average VAE loss: 115.6980, Average Metric loss: 0.0326, Metric Accuracy: 94.85%\n",
      "\n",
      "Train Epoch: 4 [42000/60000]\tVAE Loss: 115.0538 (115.6722) \tMetric Loss: 0.0174 (0.0321) \tMetric Acc: 96.67% (94.92%) \tEmb_Norm: 33.34 (33.31)\n",
      "\n",
      "Test set: Average VAE loss: 115.5993, Average Metric loss: 0.0326, Metric Accuracy: 94.84%\n",
      "\n",
      "Train Epoch: 4 [48000/60000]\tVAE Loss: 116.1591 (115.5713) \tMetric Loss: 0.0291 (0.0322) \tMetric Acc: 93.33% (94.88%) \tEmb_Norm: 33.30 (33.29)\n",
      "\n",
      "Test set: Average VAE loss: 115.5039, Average Metric loss: 0.0324, Metric Accuracy: 94.86%\n",
      "\n",
      "Train Epoch: 4 [54000/60000]\tVAE Loss: 115.0576 (115.4883) \tMetric Loss: 0.0274 (0.0320) \tMetric Acc: 95.00% (94.90%) \tEmb_Norm: 32.97 (33.28)\n",
      "\n",
      "Test set: Average VAE loss: 115.4192, Average Metric loss: 0.0322, Metric Accuracy: 94.88%\n",
      "\n",
      "Train Epoch: 5 [0/60000]\tVAE Loss: 117.3061 (117.3061) \tMetric Loss: 0.0294 (0.0294) \tMetric Acc: 95.83% (95.83%) \tEmb_Norm: 32.74 (32.74)\n",
      "\n",
      "Test set: Average VAE loss: 114.4752, Average Metric loss: 0.0324, Metric Accuracy: 94.95%\n",
      "\n",
      "Train Epoch: 5 [6000/60000]\tVAE Loss: 112.7127 (114.5390) \tMetric Loss: 0.0140 (0.0272) \tMetric Acc: 98.33% (95.72%) \tEmb_Norm: 32.64 (32.81)\n",
      "\n",
      "Test set: Average VAE loss: 114.3251, Average Metric loss: 0.0311, Metric Accuracy: 95.09%\n",
      "\n",
      "Train Epoch: 5 [12000/60000]\tVAE Loss: 114.1715 (114.2735) \tMetric Loss: 0.0063 (0.0284) \tMetric Acc: 98.33% (95.45%) \tEmb_Norm: 33.13 (32.79)\n",
      "\n",
      "Test set: Average VAE loss: 114.1156, Average Metric loss: 0.0303, Metric Accuracy: 95.22%\n",
      "\n",
      "Train Epoch: 5 [18000/60000]\tVAE Loss: 114.6495 (114.1695) \tMetric Loss: 0.0200 (0.0287) \tMetric Acc: 97.50% (95.43%) \tEmb_Norm: 32.91 (32.90)\n",
      "\n",
      "Test set: Average VAE loss: 114.0823, Average Metric loss: 0.0301, Metric Accuracy: 95.20%\n",
      "\n",
      "Train Epoch: 5 [24000/60000]\tVAE Loss: 108.8397 (114.0948) \tMetric Loss: 0.0224 (0.0293) \tMetric Acc: 95.83% (95.33%) \tEmb_Norm: 32.67 (32.89)\n",
      "\n",
      "Test set: Average VAE loss: 114.0373, Average Metric loss: 0.0302, Metric Accuracy: 95.18%\n",
      "\n",
      "Train Epoch: 5 [30000/60000]\tVAE Loss: 114.3037 (114.0070) \tMetric Loss: 0.0132 (0.0295) \tMetric Acc: 97.50% (95.28%) \tEmb_Norm: 32.81 (32.83)\n",
      "\n",
      "Test set: Average VAE loss: 113.9719, Average Metric loss: 0.0301, Metric Accuracy: 95.19%\n",
      "\n",
      "Train Epoch: 5 [36000/60000]\tVAE Loss: 114.7545 (113.9652) \tMetric Loss: 0.0263 (0.0293) \tMetric Acc: 95.00% (95.28%) \tEmb_Norm: 32.71 (32.86)\n",
      "\n",
      "Test set: Average VAE loss: 113.9127, Average Metric loss: 0.0297, Metric Accuracy: 95.24%\n",
      "\n",
      "Train Epoch: 5 [42000/60000]\tVAE Loss: 112.3423 (113.8872) \tMetric Loss: 0.0273 (0.0293) \tMetric Acc: 95.00% (95.30%) \tEmb_Norm: 33.00 (32.83)\n",
      "\n",
      "Test set: Average VAE loss: 113.8566, Average Metric loss: 0.0295, Metric Accuracy: 95.25%\n",
      "\n",
      "Train Epoch: 5 [48000/60000]\tVAE Loss: 109.6970 (113.8587) \tMetric Loss: 0.0152 (0.0291) \tMetric Acc: 98.33% (95.30%) \tEmb_Norm: 32.28 (32.84)\n",
      "\n",
      "Test set: Average VAE loss: 113.8324, Average Metric loss: 0.0296, Metric Accuracy: 95.24%\n",
      "\n",
      "Train Epoch: 5 [54000/60000]\tVAE Loss: 114.5270 (113.8241) \tMetric Loss: 0.0207 (0.0292) \tMetric Acc: 97.50% (95.29%) \tEmb_Norm: 32.39 (32.81)\n",
      "\n",
      "Test set: Average VAE loss: 113.7893, Average Metric loss: 0.0295, Metric Accuracy: 95.25%\n",
      "\n",
      "Train Epoch: 6 [0/60000]\tVAE Loss: 112.0616 (112.0616) \tMetric Loss: 0.0175 (0.0175) \tMetric Acc: 97.50% (97.50%) \tEmb_Norm: 32.66 (32.66)\n",
      "\n",
      "Test set: Average VAE loss: 113.1779, Average Metric loss: 0.0345, Metric Accuracy: 94.56%\n",
      "\n",
      "Train Epoch: 6 [6000/60000]\tVAE Loss: 115.1115 (113.2233) \tMetric Loss: 0.0160 (0.0277) \tMetric Acc: 96.67% (95.50%) \tEmb_Norm: 32.54 (32.64)\n",
      "\n",
      "Test set: Average VAE loss: 113.1085, Average Metric loss: 0.0296, Metric Accuracy: 95.20%\n",
      "\n",
      "Train Epoch: 6 [12000/60000]\tVAE Loss: 114.1961 (113.1407) \tMetric Loss: 0.0055 (0.0271) \tMetric Acc: 99.17% (95.57%) \tEmb_Norm: 32.79 (32.72)\n",
      "\n",
      "Test set: Average VAE loss: 113.0700, Average Metric loss: 0.0287, Metric Accuracy: 95.38%\n",
      "\n",
      "Train Epoch: 6 [18000/60000]\tVAE Loss: 111.9646 (113.0379) \tMetric Loss: 0.0168 (0.0271) \tMetric Acc: 98.33% (95.64%) \tEmb_Norm: 32.99 (32.68)\n",
      "\n",
      "Test set: Average VAE loss: 112.9896, Average Metric loss: 0.0283, Metric Accuracy: 95.44%\n",
      "\n",
      "Train Epoch: 6 [24000/60000]\tVAE Loss: 111.9963 (112.9647) \tMetric Loss: 0.0128 (0.0273) \tMetric Acc: 98.33% (95.58%) \tEmb_Norm: 33.02 (32.67)\n",
      "\n",
      "Test set: Average VAE loss: 112.9024, Average Metric loss: 0.0282, Metric Accuracy: 95.42%\n",
      "\n",
      "Train Epoch: 6 [30000/60000]\tVAE Loss: 113.2129 (112.8884) \tMetric Loss: 0.0108 (0.0274) \tMetric Acc: 98.33% (95.54%) \tEmb_Norm: 32.85 (32.66)\n",
      "\n",
      "Test set: Average VAE loss: 112.8598, Average Metric loss: 0.0282, Metric Accuracy: 95.42%\n",
      "\n",
      "Train Epoch: 6 [36000/60000]\tVAE Loss: 113.1185 (112.8536) \tMetric Loss: 0.0117 (0.0275) \tMetric Acc: 97.50% (95.51%) \tEmb_Norm: 32.49 (32.62)\n",
      "\n",
      "Test set: Average VAE loss: 112.8189, Average Metric loss: 0.0281, Metric Accuracy: 95.41%\n",
      "\n",
      "Train Epoch: 6 [42000/60000]\tVAE Loss: 112.9500 (112.8165) \tMetric Loss: 0.0132 (0.0276) \tMetric Acc: 98.33% (95.49%) \tEmb_Norm: 32.37 (32.59)\n",
      "\n",
      "Test set: Average VAE loss: 112.7857, Average Metric loss: 0.0281, Metric Accuracy: 95.40%\n",
      "\n",
      "Train Epoch: 6 [48000/60000]\tVAE Loss: 115.7041 (112.7821) \tMetric Loss: 0.0057 (0.0276) \tMetric Acc: 100.00% (95.48%) \tEmb_Norm: 32.36 (32.59)\n",
      "\n",
      "Test set: Average VAE loss: 112.7437, Average Metric loss: 0.0281, Metric Accuracy: 95.40%\n",
      "\n",
      "Train Epoch: 6 [54000/60000]\tVAE Loss: 112.4923 (112.7431) \tMetric Loss: 0.0166 (0.0277) \tMetric Acc: 97.50% (95.46%) \tEmb_Norm: 32.81 (32.56)\n",
      "\n",
      "Test set: Average VAE loss: 112.7133, Average Metric loss: 0.0280, Metric Accuracy: 95.42%\n",
      "\n",
      "Train Epoch: 7 [0/60000]\tVAE Loss: 115.7377 (115.7377) \tMetric Loss: 0.0201 (0.0201) \tMetric Acc: 95.00% (95.00%) \tEmb_Norm: 32.16 (32.16)\n",
      "\n",
      "Test set: Average VAE loss: 112.2741, Average Metric loss: 0.0323, Metric Accuracy: 94.66%\n",
      "\n",
      "Train Epoch: 7 [6000/60000]\tVAE Loss: 111.0920 (112.1655) \tMetric Loss: 0.0082 (0.0248) \tMetric Acc: 98.33% (95.83%) \tEmb_Norm: 32.23 (32.45)\n",
      "\n",
      "Test set: Average VAE loss: 112.0724, Average Metric loss: 0.0284, Metric Accuracy: 95.24%\n",
      "\n",
      "Train Epoch: 7 [12000/60000]\tVAE Loss: 113.3828 (112.0921) \tMetric Loss: 0.0047 (0.0257) \tMetric Acc: 99.17% (95.66%) \tEmb_Norm: 32.38 (32.37)\n",
      "\n",
      "Test set: Average VAE loss: 112.0328, Average Metric loss: 0.0282, Metric Accuracy: 95.30%\n",
      "\n",
      "Train Epoch: 7 [18000/60000]\tVAE Loss: 109.5582 (112.0124) \tMetric Loss: 0.0212 (0.0267) \tMetric Acc: 96.67% (95.53%) \tEmb_Norm: 32.05 (32.34)\n",
      "\n",
      "Test set: Average VAE loss: 111.9965, Average Metric loss: 0.0281, Metric Accuracy: 95.33%\n",
      "\n",
      "Train Epoch: 7 [24000/60000]\tVAE Loss: 112.7028 (111.9797) \tMetric Loss: 0.0068 (0.0267) \tMetric Acc: 98.33% (95.53%) \tEmb_Norm: 32.20 (32.27)\n",
      "\n",
      "Test set: Average VAE loss: 111.9655, Average Metric loss: 0.0279, Metric Accuracy: 95.36%\n",
      "\n",
      "Train Epoch: 7 [30000/60000]\tVAE Loss: 111.1011 (111.9711) \tMetric Loss: 0.0206 (0.0270) \tMetric Acc: 97.50% (95.49%) \tEmb_Norm: 32.24 (32.24)\n",
      "\n",
      "Test set: Average VAE loss: 111.9665, Average Metric loss: 0.0276, Metric Accuracy: 95.38%\n",
      "\n",
      "Train Epoch: 7 [36000/60000]\tVAE Loss: 110.4355 (111.9659) \tMetric Loss: 0.0121 (0.0268) \tMetric Acc: 98.33% (95.51%) \tEmb_Norm: 32.70 (32.25)\n",
      "\n",
      "Test set: Average VAE loss: 111.9311, Average Metric loss: 0.0275, Metric Accuracy: 95.42%\n",
      "\n",
      "Train Epoch: 7 [42000/60000]\tVAE Loss: 111.3423 (111.9666) \tMetric Loss: 0.0156 (0.0269) \tMetric Acc: 96.67% (95.51%) \tEmb_Norm: 32.24 (32.28)\n",
      "\n",
      "Test set: Average VAE loss: 111.9681, Average Metric loss: 0.0273, Metric Accuracy: 95.45%\n",
      "\n",
      "Train Epoch: 7 [48000/60000]\tVAE Loss: 112.5868 (111.9821) \tMetric Loss: 0.0417 (0.0268) \tMetric Acc: 92.50% (95.53%) \tEmb_Norm: 32.24 (32.25)\n",
      "\n",
      "Test set: Average VAE loss: 111.9682, Average Metric loss: 0.0272, Metric Accuracy: 95.47%\n",
      "\n",
      "Train Epoch: 7 [54000/60000]\tVAE Loss: 113.4203 (111.9589) \tMetric Loss: 0.0175 (0.0267) \tMetric Acc: 97.50% (95.54%) \tEmb_Norm: 32.06 (32.25)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Test set: Average VAE loss: 111.9447, Average Metric loss: 0.0271, Metric Accuracy: 95.47%\n",
      "\n",
      "Train Epoch: 8 [0/60000]\tVAE Loss: 110.8053 (110.8053) \tMetric Loss: 0.0098 (0.0098) \tMetric Acc: 98.33% (98.33%) \tEmb_Norm: 32.28 (32.28)\n",
      "\n",
      "Test set: Average VAE loss: 111.5764, Average Metric loss: 0.0315, Metric Accuracy: 94.70%\n",
      "\n",
      "Train Epoch: 8 [6000/60000]\tVAE Loss: 111.8951 (111.5987) \tMetric Loss: 0.0120 (0.0236) \tMetric Acc: 98.33% (96.02%) \tEmb_Norm: 31.71 (32.16)\n",
      "\n",
      "Test set: Average VAE loss: 111.5715, Average Metric loss: 0.0273, Metric Accuracy: 95.43%\n",
      "\n",
      "Train Epoch: 8 [12000/60000]\tVAE Loss: 110.6374 (111.5220) \tMetric Loss: 0.0028 (0.0243) \tMetric Acc: 100.00% (95.87%) \tEmb_Norm: 32.03 (32.06)\n",
      "\n",
      "Test set: Average VAE loss: 111.4044, Average Metric loss: 0.0272, Metric Accuracy: 95.42%\n",
      "\n",
      "Train Epoch: 8 [18000/60000]\tVAE Loss: 111.0103 (111.3835) \tMetric Loss: 0.0127 (0.0253) \tMetric Acc: 97.50% (95.72%) \tEmb_Norm: 32.13 (32.05)\n",
      "\n",
      "Test set: Average VAE loss: 111.3347, Average Metric loss: 0.0271, Metric Accuracy: 95.44%\n",
      "\n",
      "Train Epoch: 8 [24000/60000]\tVAE Loss: 115.2000 (111.3489) \tMetric Loss: 0.0114 (0.0259) \tMetric Acc: 98.33% (95.61%) \tEmb_Norm: 32.09 (32.04)\n",
      "\n",
      "Test set: Average VAE loss: 111.3380, Average Metric loss: 0.0268, Metric Accuracy: 95.47%\n",
      "\n",
      "Train Epoch: 8 [30000/60000]\tVAE Loss: 113.0391 (111.3406) \tMetric Loss: 0.0111 (0.0259) \tMetric Acc: 97.50% (95.63%) \tEmb_Norm: 31.42 (32.05)\n",
      "\n",
      "Test set: Average VAE loss: 111.3458, Average Metric loss: 0.0267, Metric Accuracy: 95.49%\n",
      "\n",
      "Train Epoch: 8 [36000/60000]\tVAE Loss: 111.3480 (111.3356) \tMetric Loss: 0.0101 (0.0259) \tMetric Acc: 99.17% (95.63%) \tEmb_Norm: 31.92 (32.00)\n",
      "\n",
      "Test set: Average VAE loss: 111.3157, Average Metric loss: 0.0266, Metric Accuracy: 95.51%\n",
      "\n",
      "Train Epoch: 8 [42000/60000]\tVAE Loss: 113.1322 (111.3165) \tMetric Loss: 0.0154 (0.0260) \tMetric Acc: 97.50% (95.61%) \tEmb_Norm: 32.13 (31.99)\n",
      "\n",
      "Test set: Average VAE loss: 111.3091, Average Metric loss: 0.0265, Metric Accuracy: 95.53%\n",
      "\n",
      "Train Epoch: 8 [48000/60000]\tVAE Loss: 112.3676 (111.3001) \tMetric Loss: 0.0077 (0.0259) \tMetric Acc: 99.17% (95.62%) \tEmb_Norm: 31.97 (32.00)\n",
      "\n",
      "Test set: Average VAE loss: 111.2836, Average Metric loss: 0.0264, Metric Accuracy: 95.54%\n",
      "\n",
      "Train Epoch: 8 [54000/60000]\tVAE Loss: 109.4236 (111.2756) \tMetric Loss: 0.0073 (0.0260) \tMetric Acc: 98.33% (95.62%) \tEmb_Norm: 31.98 (32.02)\n",
      "\n",
      "Test set: Average VAE loss: 111.2758, Average Metric loss: 0.0263, Metric Accuracy: 95.56%\n",
      "\n",
      "Train Epoch: 9 [0/60000]\tVAE Loss: 110.0447 (110.0447) \tMetric Loss: 0.0106 (0.0106) \tMetric Acc: 96.67% (96.67%) \tEmb_Norm: 31.90 (31.90)\n",
      "\n",
      "Test set: Average VAE loss: 110.9539, Average Metric loss: 0.0345, Metric Accuracy: 94.36%\n",
      "\n",
      "Train Epoch: 9 [6000/60000]\tVAE Loss: 110.8069 (110.7985) \tMetric Loss: 0.0155 (0.0253) \tMetric Acc: 97.50% (95.81%) \tEmb_Norm: 31.53 (31.97)\n",
      "\n",
      "Test set: Average VAE loss: 110.7580, Average Metric loss: 0.0295, Metric Accuracy: 95.10%\n",
      "\n",
      "Train Epoch: 9 [12000/60000]\tVAE Loss: 113.0105 (110.7328) \tMetric Loss: 0.0171 (0.0260) \tMetric Acc: 97.50% (95.67%) \tEmb_Norm: 31.84 (31.87)\n",
      "\n",
      "Test set: Average VAE loss: 110.7459, Average Metric loss: 0.0282, Metric Accuracy: 95.26%\n",
      "\n",
      "Train Epoch: 9 [18000/60000]\tVAE Loss: 107.6744 (110.7955) \tMetric Loss: 0.0128 (0.0260) \tMetric Acc: 97.50% (95.62%) \tEmb_Norm: 31.80 (31.85)\n",
      "\n",
      "Test set: Average VAE loss: 110.7838, Average Metric loss: 0.0275, Metric Accuracy: 95.39%\n",
      "\n",
      "Train Epoch: 9 [24000/60000]\tVAE Loss: 108.4960 (110.7662) \tMetric Loss: 0.0070 (0.0261) \tMetric Acc: 99.17% (95.63%) \tEmb_Norm: 31.83 (31.87)\n",
      "\n",
      "Test set: Average VAE loss: 110.7723, Average Metric loss: 0.0273, Metric Accuracy: 95.43%\n",
      "\n",
      "Train Epoch: 9 [30000/60000]\tVAE Loss: 112.4896 (110.7864) \tMetric Loss: 0.0141 (0.0263) \tMetric Acc: 96.67% (95.59%) \tEmb_Norm: 32.04 (31.85)\n",
      "\n",
      "Test set: Average VAE loss: 110.7975, Average Metric loss: 0.0268, Metric Accuracy: 95.51%\n",
      "\n",
      "Train Epoch: 9 [36000/60000]\tVAE Loss: 108.0867 (110.7856) \tMetric Loss: 0.0080 (0.0259) \tMetric Acc: 98.33% (95.65%) \tEmb_Norm: 31.90 (31.87)\n",
      "\n",
      "Test set: Average VAE loss: 110.7938, Average Metric loss: 0.0266, Metric Accuracy: 95.55%\n",
      "\n",
      "Train Epoch: 9 [42000/60000]\tVAE Loss: 111.4153 (110.7999) \tMetric Loss: 0.0106 (0.0258) \tMetric Acc: 99.17% (95.67%) \tEmb_Norm: 31.66 (31.88)\n",
      "\n",
      "Test set: Average VAE loss: 110.7907, Average Metric loss: 0.0264, Metric Accuracy: 95.57%\n",
      "\n",
      "Train Epoch: 9 [48000/60000]\tVAE Loss: 107.7935 (110.7883) \tMetric Loss: 0.0134 (0.0258) \tMetric Acc: 95.83% (95.67%) \tEmb_Norm: 31.47 (31.86)\n",
      "\n",
      "Test set: Average VAE loss: 110.7810, Average Metric loss: 0.0264, Metric Accuracy: 95.58%\n",
      "\n",
      "Train Epoch: 9 [54000/60000]\tVAE Loss: 110.9774 (110.7857) \tMetric Loss: 0.0162 (0.0259) \tMetric Acc: 98.33% (95.66%) \tEmb_Norm: 32.35 (31.83)\n",
      "\n",
      "Test set: Average VAE loss: 110.7811, Average Metric loss: 0.0263, Metric Accuracy: 95.60%\n",
      "\n",
      "Train Epoch: 10 [0/60000]\tVAE Loss: 108.7348 (108.7348) \tMetric Loss: 0.0175 (0.0175) \tMetric Acc: 96.67% (96.67%) \tEmb_Norm: 31.48 (31.48)\n",
      "\n",
      "Test set: Average VAE loss: 110.7991, Average Metric loss: 0.0307, Metric Accuracy: 94.76%\n",
      "\n",
      "Train Epoch: 10 [6000/60000]\tVAE Loss: 110.6233 (110.6591) \tMetric Loss: 0.0348 (0.0231) \tMetric Acc: 95.00% (96.03%) \tEmb_Norm: 31.47 (31.71)\n",
      "\n",
      "Test set: Average VAE loss: 110.6075, Average Metric loss: 0.0276, Metric Accuracy: 95.36%\n",
      "\n",
      "Train Epoch: 10 [12000/60000]\tVAE Loss: 109.4552 (110.5011) \tMetric Loss: 0.0063 (0.0242) \tMetric Acc: 98.33% (95.95%) \tEmb_Norm: 31.55 (31.67)\n",
      "\n",
      "Test set: Average VAE loss: 110.4515, Average Metric loss: 0.0267, Metric Accuracy: 95.52%\n",
      "\n",
      "Train Epoch: 10 [18000/60000]\tVAE Loss: 111.8864 (110.4336) \tMetric Loss: 0.0033 (0.0246) \tMetric Acc: 99.17% (95.85%) \tEmb_Norm: 31.79 (31.71)\n",
      "\n",
      "Test set: Average VAE loss: 110.4020, Average Metric loss: 0.0265, Metric Accuracy: 95.53%\n",
      "\n",
      "Train Epoch: 10 [24000/60000]\tVAE Loss: 110.9128 (110.4259) \tMetric Loss: 0.0118 (0.0250) \tMetric Acc: 97.50% (95.77%) \tEmb_Norm: 31.71 (31.71)\n",
      "\n",
      "Test set: Average VAE loss: 110.3874, Average Metric loss: 0.0265, Metric Accuracy: 95.55%\n",
      "\n",
      "Train Epoch: 10 [30000/60000]\tVAE Loss: 109.4759 (110.3753) \tMetric Loss: 0.0044 (0.0255) \tMetric Acc: 99.17% (95.71%) \tEmb_Norm: 31.64 (31.71)\n",
      "\n",
      "Test set: Average VAE loss: 110.3862, Average Metric loss: 0.0264, Metric Accuracy: 95.56%\n",
      "\n",
      "Train Epoch: 10 [36000/60000]\tVAE Loss: 111.0612 (110.3814) \tMetric Loss: 0.0086 (0.0255) \tMetric Acc: 97.50% (95.71%) \tEmb_Norm: 31.47 (31.72)\n",
      "\n",
      "Test set: Average VAE loss: 110.3601, Average Metric loss: 0.0264, Metric Accuracy: 95.56%\n",
      "\n",
      "Train Epoch: 10 [42000/60000]\tVAE Loss: 110.2338 (110.3688) \tMetric Loss: 0.0130 (0.0257) \tMetric Acc: 97.50% (95.67%) \tEmb_Norm: 32.04 (31.72)\n",
      "\n",
      "Test set: Average VAE loss: 110.3708, Average Metric loss: 0.0262, Metric Accuracy: 95.59%\n",
      "\n",
      "Train Epoch: 10 [48000/60000]\tVAE Loss: 111.8270 (110.3565) \tMetric Loss: 0.0146 (0.0255) \tMetric Acc: 95.83% (95.69%) \tEmb_Norm: 31.21 (31.75)\n",
      "\n",
      "Test set: Average VAE loss: 110.3474, Average Metric loss: 0.0263, Metric Accuracy: 95.57%\n",
      "\n",
      "Train Epoch: 10 [54000/60000]\tVAE Loss: 111.4027 (110.3425) \tMetric Loss: 0.0008 (0.0257) \tMetric Acc: 100.00% (95.66%) \tEmb_Norm: 31.86 (31.73)\n",
      "\n",
      "Test set: Average VAE loss: 110.3487, Average Metric loss: 0.0262, Metric Accuracy: 95.60%\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "from __future__ import print_function\n",
    "import argparse\n",
    "import torch\n",
    "import torch.utils.data\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.autograd import Variable\n",
    "from torchvision import datasets, transforms\n",
    "from triplet_mnist_loader import MNIST_t\n",
    "import torch.nn.functional as F\n",
    "\n",
    "batch_size = 120\n",
    "epochs = 10\n",
    "no_cuda = False\n",
    "seed = 1\n",
    "log_interval = 50\n",
    "margin = 0.2\n",
    "cuda = not no_cuda and torch.cuda.is_available()\n",
    "\n",
    "\n",
    "torch.manual_seed(seed)\n",
    "if cuda:\n",
    "    torch.cuda.manual_seed(seed)\n",
    "\n",
    "\n",
    "kwargs = {'num_workers': 1, 'pin_memory': True} if cuda else {}\n",
    "train_loader = torch.utils.data.DataLoader(\n",
    "        MNIST_t('../data', train=True, download=True,\n",
    "                       transform=transforms.ToTensor()),\n",
    "        batch_size=batch_size, shuffle=True, **kwargs)\n",
    "test_loader = torch.utils.data.DataLoader(\n",
    "        MNIST_t('../data', train=False, download=True,\n",
    "                       transform=transforms.ToTensor()),\n",
    "        batch_size=batch_size, shuffle=True, **kwargs)\n",
    "\n",
    "\n",
    "class VAE(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(VAE, self).__init__()\n",
    "\n",
    "        self.fc1 = nn.Linear(784, 400)\n",
    "        self.fc21 = nn.Linear(400, 20)\n",
    "        self.fc22 = nn.Linear(400, 20)\n",
    "        self.fc3 = nn.Linear(20, 400)\n",
    "        self.fc4 = nn.Linear(400, 784)\n",
    "\n",
    "        self.relu = nn.ReLU()\n",
    "        self.sigmoid = nn.Sigmoid()\n",
    "\n",
    "    def encode(self, x):\n",
    "        h1 = self.relu(self.fc1(x))\n",
    "        return self.fc21(h1), self.fc22(h1)\n",
    "\n",
    "    def reparametrize(self, mu, logvar):\n",
    "        std = logvar.mul(0.5).exp_()\n",
    "        if cuda:\n",
    "            eps = torch.cuda.FloatTensor(std.size()).normal_()\n",
    "        else:\n",
    "            eps = torch.FloatTensor(std.size()).normal_()\n",
    "        eps = Variable(eps)\n",
    "        return eps.mul(std).add_(mu)\n",
    "\n",
    "    def decode(self, z):\n",
    "        h3 = self.relu(self.fc3(z))\n",
    "        return self.sigmoid(self.fc4(h3))\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = x.view(-1, 784)\n",
    "        h1 = self.relu(self.fc1(x))\n",
    "        mu = self.fc21(h1)\n",
    "        logvar = self.fc22(h1)\n",
    "        z = self.reparametrize(mu, logvar)\n",
    "        return self.decode(z), mu, logvar\n",
    "\n",
    "\n",
    "model = VAE()\n",
    "if cuda:\n",
    "    model.cuda()\n",
    "\n",
    "reconstruction_function = nn.BCELoss()\n",
    "reconstruction_function.size_average = False\n",
    "\n",
    "\n",
    "def loss_function(recon_x, x, mu, logvar):\n",
    "    BCE = reconstruction_function(recon_x, x)\n",
    "\n",
    "    # see Appendix B from VAE paper:\n",
    "    # Kingma and Welling. Auto-Encoding Variational Bayes. ICLR, 2014\n",
    "    # https://arxiv.org/abs/1312.6114\n",
    "    # 0.5 * sum(1 + log(sigma^2) - mu^2 - sigma^2)\n",
    "    KLD_element = mu.pow(2).add_(logvar.exp()).mul_(-1).add_(1).add_(logvar)\n",
    "    KLD = torch.sum(KLD_element).mul_(-0.5)\n",
    "\n",
    "    return BCE + KLD\n",
    "\n",
    "\n",
    "optimizer = optim.Adam(model.parameters(), lr=1e-3)\n",
    "criterion = torch.nn.MarginRankingLoss(margin = margin)\n",
    "\n",
    "\n",
    "class AverageMeter(object):\n",
    "    \"\"\"Computes and stores the average and current value\"\"\"\n",
    "    def __init__(self):\n",
    "        self.reset()\n",
    "\n",
    "    def reset(self):\n",
    "        self.val = 0\n",
    "        self.avg = 0\n",
    "        self.sum = 0\n",
    "        self.count = 0\n",
    "\n",
    "    def update(self, val, n=1):\n",
    "        self.val = val\n",
    "        self.sum += val * n\n",
    "        self.count += n\n",
    "        self.avg = self.sum / self.count\n",
    "        \n",
    "def accuracy(dista, distb):\n",
    "    margin = 0\n",
    "    pred = (dista - distb - margin).cpu().data\n",
    "    return (pred > 0).sum()*1.0/dista.size()[0]\n",
    "\n",
    "train_loss_metric = []\n",
    "train_loss_VAE = []\n",
    "train_acc_metric = []\n",
    "test_loss_metric = []\n",
    "test_loss_VAE = []\n",
    "test_acc_metric = []\n",
    "\n",
    "for epoch in range(1, epochs + 1):\n",
    "    losses_metric = AverageMeter()\n",
    "    losses_VAE = AverageMeter()\n",
    "    accs = AverageMeter()\n",
    "    emb_norms = AverageMeter()\n",
    "\n",
    "    model.train()\n",
    "    for batch_idx, (data1, data2, data3) in enumerate(train_loader):\n",
    "        if cuda:\n",
    "            data1, data2, data3 = data1.cuda(), data2.cuda(), data3.cuda()\n",
    "        data1, data2, data3 = Variable(data1), Variable(data2), Variable(data3)\n",
    "\n",
    "        recon_batch1, mu1, logvar1 = model(data1)\n",
    "        recon_batch2, mu2, logvar2 = model(data2)\n",
    "        recon_batch3, mu3, logvar3 = model(data3)\n",
    "\n",
    "        loss_vae = loss_function(recon_batch1, data1, mu1, logvar1)     \n",
    "        loss_vae += loss_function(recon_batch2, data2, mu2, logvar2)  \n",
    "        loss_vae += loss_function(recon_batch3, data3, mu3, logvar3)  \n",
    "        loss_vae = loss_vae/(3*len(data1))\n",
    "\n",
    "        dista = F.pairwise_distance(mu1, mu2, 2)\n",
    "        distb = F.pairwise_distance(mu1, mu3, 2)\n",
    "        target = torch.FloatTensor(dista.size()).fill_(1)\n",
    "        if cuda:\n",
    "            target = target.cuda()\n",
    "        target = Variable(target)\n",
    "        loss_triplet = criterion(dista, distb, target)\n",
    "\n",
    "        loss_embedd = mu1.norm(2) + mu2.norm(2) + mu3.norm(2)\n",
    "\n",
    "        loss = 0.01*loss_vae + loss_triplet + 0.001*loss_embedd\n",
    "\n",
    "        # measure accuracy and record loss\n",
    "        acc = accuracy(dista, distb)\n",
    "        losses_metric.update(loss_triplet.data[0], data1.size(0))\n",
    "        losses_VAE.update(loss_vae.data[0], data1.size(0))\n",
    "        accs.update(acc, data1.size(0))\n",
    "        emb_norms.update(loss_embedd.data[0]/3, data1.size(0))\n",
    "\n",
    "        # train\n",
    "        optimizer.zero_grad()          \n",
    "        loss.backward()\n",
    "        optimizer.step()  \n",
    "        if batch_idx % log_interval == 0:\n",
    "            print('Train Epoch: {} [{}/{}]\\t'\n",
    "                  'VAE Loss: {:.4f} ({:.4f}) \\t'\n",
    "                  'Metric Loss: {:.4f} ({:.4f}) \\t'\n",
    "                  'Metric Acc: {:.2f}% ({:.2f}%) \\t'\n",
    "                  'Emb_Norm: {:.2f} ({:.2f})'.format(\n",
    "                epoch, batch_idx * len(data1), len(train_loader.dataset),\n",
    "                losses_VAE.val, losses_VAE.avg,\n",
    "                losses_metric.val, losses_metric.avg, \n",
    "                100. * accs.val, 100. * accs.avg, emb_norms.val, emb_norms.avg))\n",
    "\n",
    "            train_loss_metric.append(losses_metric.val)\n",
    "            train_loss_VAE.append(losses_VAE.val)\n",
    "            train_acc_metric.append(accs.val)\n",
    "\n",
    "           # plot(data1, recon_batch1, recon_batch1)\n",
    "            data_metric_train = data1\n",
    "            recon_batch_metric_train = recon_batch1\n",
    "\n",
    "            # Test\n",
    "            model.eval()\n",
    "            model_vanilla.eval()\n",
    "            for batch_idx, (data1, data2, data3) in enumerate(test_loader):\n",
    "                if cuda:\n",
    "                    data1, data2, data3 = data1.cuda(), data2.cuda(), data3.cuda()\n",
    "                data1, data2, data3 = Variable(data1), Variable(data2), Variable(data3)\n",
    "\n",
    "                recon_batch1, mu1, logvar1 = model(data1)\n",
    "                recon_batch2, mu2, logvar2 = model(data2)\n",
    "                recon_batch3, mu3, logvar3 = model(data3)\n",
    "\n",
    "                loss_vae = loss_function(recon_batch1, data1, mu1, logvar1)     \n",
    "                loss_vae += loss_function(recon_batch2, data2, mu2, logvar2)  \n",
    "                loss_vae += loss_function(recon_batch3, data3, mu3, logvar3)  \n",
    "                loss_vae = loss_vae/(3*len(data1))\n",
    "\n",
    "                dista = F.pairwise_distance(mu1, mu2, 2)\n",
    "                distb = F.pairwise_distance(mu1, mu3, 2)\n",
    "                target = torch.FloatTensor(dista.size()).fill_(1)\n",
    "                if cuda:\n",
    "                    target = target.cuda()\n",
    "                target = Variable(target)\n",
    "                loss_triplet = criterion(dista, distb, target)\n",
    "\n",
    "                loss_embedd = mu1.norm(2) + mu2.norm(2) + mu3.norm(2)\n",
    "\n",
    "                loss = 0.01*loss_vae + loss_triplet + 0.001*loss_embedd\n",
    "\n",
    "                # measure accuracy and record loss\n",
    "                acc = accuracy(dista, distb)\n",
    "                losses_metric.update(loss_triplet.data[0], data1.size(0))\n",
    "                losses_VAE.update(loss_vae.data[0], data1.size(0))\n",
    "                accs.update(acc, data1.size(0))\n",
    "                emb_norms.update(loss_embedd.data[0]/3, data1.size(0))\n",
    "\n",
    "            print('\\nTest set: Average VAE loss: {:.4f}, Average Metric loss: {:.4f}, Metric Accuracy: {:.2f}%\\n'.format(\n",
    "            losses_VAE.avg, losses_metric.avg, 100. * accs.avg))\n",
    "            test_loss_metric.append(losses_metric.avg)\n",
    "            test_loss_VAE.append(losses_VAE.avg)\n",
    "            test_acc_metric.append(accs.avg)\n",
    "\n",
    "           # plot(data1, recon_batch1)\n",
    "            data_metric_test = data1\n",
    "            recon_batch_metric_test = recon_batch1\n",
    "            recon_batch_vanilla_test = model_vanilla(data1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Vanilla VAE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing Triplet Generation ...\n",
      "Done!\n",
      "Processing Triplet Generation ...\n",
      "Done!\n",
      "Train Epoch: 1 [0/60000]\tVAE Loss: 549.0129 (549.0129) \tMetric Loss: 0.1719 (0.1719) \tMetric Acc: 62.50% (62.50%) \tEmb_Norm: 4.09 (4.09)\n",
      "\n",
      "Test set: Average VAE loss: 521.9688, Average Metric loss: 0.1626, Metric Accuracy: 64.07%\n",
      "\n",
      "Train Epoch: 1 [6000/60000]\tVAE Loss: 210.0975 (425.1393) \tMetric Loss: 0.3325 (0.2538) \tMetric Acc: 57.50% (62.49%) \tEmb_Norm: 35.95 (20.77)\n",
      "\n",
      "Test set: Average VAE loss: 342.8424, Average Metric loss: 0.2843, Metric Accuracy: 63.99%\n",
      "\n",
      "Train Epoch: 1 [12000/60000]\tVAE Loss: 182.2006 (315.7575) \tMetric Loss: 0.3905 (0.2954) \tMetric Acc: 75.00% (65.34%) \tEmb_Norm: 46.55 (29.61)\n",
      "\n",
      "Test set: Average VAE loss: 283.9263, Average Metric loss: 0.2985, Metric Accuracy: 68.29%\n",
      "\n",
      "Train Epoch: 1 [18000/60000]\tVAE Loss: 159.6533 (269.7519) \tMetric Loss: 0.2419 (0.2978) \tMetric Acc: 84.17% (69.64%) \tEmb_Norm: 50.35 (35.97)\n",
      "\n",
      "Test set: Average VAE loss: 250.9841, Average Metric loss: 0.2897, Metric Accuracy: 71.64%\n",
      "\n",
      "Train Epoch: 1 [24000/60000]\tVAE Loss: 150.0022 (242.1075) \tMetric Loss: 0.2715 (0.2881) \tMetric Acc: 83.33% (72.37%) \tEmb_Norm: 50.21 (39.49)\n",
      "\n",
      "Test set: Average VAE loss: 229.6172, Average Metric loss: 0.2826, Metric Accuracy: 73.53%\n",
      "\n",
      "Train Epoch: 1 [30000/60000]\tVAE Loss: 141.9351 (223.3514) \tMetric Loss: 0.2710 (0.2811) \tMetric Acc: 79.17% (73.97%) \tEmb_Norm: 52.19 (41.93)\n",
      "\n",
      "Test set: Average VAE loss: 214.2180, Average Metric loss: 0.2771, Metric Accuracy: 74.65%\n",
      "\n",
      "Train Epoch: 1 [36000/60000]\tVAE Loss: 134.8300 (209.5280) \tMetric Loss: 0.2591 (0.2770) \tMetric Acc: 79.17% (74.85%) \tEmb_Norm: 52.62 (43.68)\n",
      "\n",
      "Test set: Average VAE loss: 202.5142, Average Metric loss: 0.2744, Metric Accuracy: 75.27%\n",
      "\n",
      "Train Epoch: 1 [42000/60000]\tVAE Loss: 130.0903 (198.8781) \tMetric Loss: 0.2630 (0.2739) \tMetric Acc: 78.33% (75.41%) \tEmb_Norm: 53.27 (45.01)\n",
      "\n",
      "Test set: Average VAE loss: 193.2865, Average Metric loss: 0.2722, Metric Accuracy: 75.66%\n",
      "\n",
      "Train Epoch: 1 [48000/60000]\tVAE Loss: 129.7983 (190.3468) \tMetric Loss: 0.1532 (0.2720) \tMetric Acc: 86.67% (75.75%) \tEmb_Norm: 52.44 (46.04)\n",
      "\n",
      "Test set: Average VAE loss: 185.7845, Average Metric loss: 0.2704, Metric Accuracy: 75.91%\n",
      "\n",
      "Train Epoch: 1 [54000/60000]\tVAE Loss: 125.1002 (183.3455) \tMetric Loss: 0.2047 (0.2697) \tMetric Acc: 81.67% (75.98%) \tEmb_Norm: 52.44 (46.80)\n",
      "\n",
      "Test set: Average VAE loss: 179.5278, Average Metric loss: 0.2682, Metric Accuracy: 76.11%\n",
      "\n",
      "Train Epoch: 2 [0/60000]\tVAE Loss: 123.3864 (123.3864) \tMetric Loss: 0.2333 (0.2333) \tMetric Acc: 80.00% (80.00%) \tEmb_Norm: 52.45 (52.45)\n",
      "\n",
      "Test set: Average VAE loss: 122.4764, Average Metric loss: 0.2466, Metric Accuracy: 77.64%\n",
      "\n",
      "Train Epoch: 2 [6000/60000]\tVAE Loss: 123.3148 (122.6280) \tMetric Loss: 0.2534 (0.2507) \tMetric Acc: 80.00% (77.37%) \tEmb_Norm: 52.60 (52.40)\n",
      "\n",
      "Test set: Average VAE loss: 121.9074, Average Metric loss: 0.2479, Metric Accuracy: 77.42%\n",
      "\n",
      "Train Epoch: 2 [12000/60000]\tVAE Loss: 118.9135 (121.8004) \tMetric Loss: 0.3206 (0.2474) \tMetric Acc: 67.50% (77.44%) \tEmb_Norm: 52.01 (52.23)\n",
      "\n",
      "Test set: Average VAE loss: 121.1853, Average Metric loss: 0.2465, Metric Accuracy: 77.34%\n",
      "\n",
      "Train Epoch: 2 [18000/60000]\tVAE Loss: 118.3526 (120.9973) \tMetric Loss: 0.3288 (0.2478) \tMetric Acc: 73.33% (77.22%) \tEmb_Norm: 51.12 (52.08)\n",
      "\n",
      "Test set: Average VAE loss: 120.5026, Average Metric loss: 0.2471, Metric Accuracy: 77.15%\n",
      "\n",
      "Train Epoch: 2 [24000/60000]\tVAE Loss: 118.6928 (120.3466) \tMetric Loss: 0.3489 (0.2485) \tMetric Acc: 70.83% (77.03%) \tEmb_Norm: 50.81 (51.95)\n",
      "\n",
      "Test set: Average VAE loss: 119.8736, Average Metric loss: 0.2476, Metric Accuracy: 77.01%\n",
      "\n",
      "Train Epoch: 2 [30000/60000]\tVAE Loss: 119.1225 (119.7140) \tMetric Loss: 0.2389 (0.2484) \tMetric Acc: 74.17% (76.91%) \tEmb_Norm: 50.77 (51.84)\n",
      "\n",
      "Test set: Average VAE loss: 119.2941, Average Metric loss: 0.2477, Metric Accuracy: 76.87%\n",
      "\n",
      "Train Epoch: 2 [36000/60000]\tVAE Loss: 116.8505 (119.1160) \tMetric Loss: 0.2259 (0.2483) \tMetric Acc: 79.17% (76.80%) \tEmb_Norm: 50.68 (51.72)\n",
      "\n",
      "Test set: Average VAE loss: 118.7682, Average Metric loss: 0.2475, Metric Accuracy: 76.78%\n",
      "\n",
      "Train Epoch: 2 [42000/60000]\tVAE Loss: 116.5143 (118.6428) \tMetric Loss: 0.2460 (0.2472) \tMetric Acc: 76.67% (76.78%) \tEmb_Norm: 50.87 (51.60)\n",
      "\n",
      "Test set: Average VAE loss: 118.2975, Average Metric loss: 0.2469, Metric Accuracy: 76.75%\n",
      "\n",
      "Train Epoch: 2 [48000/60000]\tVAE Loss: 115.9873 (118.1546) \tMetric Loss: 0.2698 (0.2473) \tMetric Acc: 74.17% (76.70%) \tEmb_Norm: 50.92 (51.51)\n",
      "\n",
      "Test set: Average VAE loss: 117.8466, Average Metric loss: 0.2468, Metric Accuracy: 76.70%\n",
      "\n",
      "Train Epoch: 2 [54000/60000]\tVAE Loss: 114.3040 (117.7133) \tMetric Loss: 0.2120 (0.2469) \tMetric Acc: 74.17% (76.66%) \tEmb_Norm: 50.89 (51.43)\n",
      "\n",
      "Test set: Average VAE loss: 117.4288, Average Metric loss: 0.2463, Metric Accuracy: 76.66%\n",
      "\n",
      "Train Epoch: 3 [0/60000]\tVAE Loss: 114.6934 (114.6934) \tMetric Loss: 0.2187 (0.2187) \tMetric Acc: 80.83% (80.83%) \tEmb_Norm: 51.23 (51.23)\n",
      "\n",
      "Test set: Average VAE loss: 113.0023, Average Metric loss: 0.2420, Metric Accuracy: 76.35%\n",
      "\n",
      "Train Epoch: 3 [6000/60000]\tVAE Loss: 111.1962 (113.3416) \tMetric Loss: 0.2548 (0.2445) \tMetric Acc: 75.83% (76.19%) \tEmb_Norm: 50.56 (50.71)\n",
      "\n",
      "Test set: Average VAE loss: 113.0566, Average Metric loss: 0.2422, Metric Accuracy: 76.27%\n",
      "\n",
      "Train Epoch: 3 [12000/60000]\tVAE Loss: 113.2112 (112.9649) \tMetric Loss: 0.2870 (0.2418) \tMetric Acc: 75.00% (76.25%) \tEmb_Norm: 50.34 (50.63)\n",
      "\n",
      "Test set: Average VAE loss: 112.7501, Average Metric loss: 0.2413, Metric Accuracy: 76.25%\n",
      "\n",
      "Train Epoch: 3 [18000/60000]\tVAE Loss: 114.4733 (112.7852) \tMetric Loss: 0.1929 (0.2430) \tMetric Acc: 80.00% (76.10%) \tEmb_Norm: 49.91 (50.58)\n",
      "\n",
      "Test set: Average VAE loss: 112.5693, Average Metric loss: 0.2417, Metric Accuracy: 76.13%\n",
      "\n",
      "Train Epoch: 3 [24000/60000]\tVAE Loss: 113.3732 (112.5641) \tMetric Loss: 0.1750 (0.2419) \tMetric Acc: 77.50% (76.07%) \tEmb_Norm: 49.75 (50.42)\n",
      "\n",
      "Test set: Average VAE loss: 112.3792, Average Metric loss: 0.2409, Metric Accuracy: 76.11%\n",
      "\n",
      "Train Epoch: 3 [30000/60000]\tVAE Loss: 109.0034 (112.3660) \tMetric Loss: 0.2234 (0.2413) \tMetric Acc: 80.00% (76.13%) \tEmb_Norm: 49.55 (50.33)\n",
      "\n",
      "Test set: Average VAE loss: 112.2150, Average Metric loss: 0.2410, Metric Accuracy: 76.16%\n",
      "\n",
      "Train Epoch: 3 [36000/60000]\tVAE Loss: 108.8693 (112.1955) \tMetric Loss: 0.1970 (0.2412) \tMetric Acc: 80.83% (76.14%) \tEmb_Norm: 49.41 (50.26)\n",
      "\n",
      "Test set: Average VAE loss: 112.0534, Average Metric loss: 0.2406, Metric Accuracy: 76.15%\n",
      "\n",
      "Train Epoch: 3 [42000/60000]\tVAE Loss: 108.5315 (112.0118) \tMetric Loss: 0.2286 (0.2414) \tMetric Acc: 78.33% (76.08%) \tEmb_Norm: 49.58 (50.19)\n",
      "\n",
      "Test set: Average VAE loss: 111.8830, Average Metric loss: 0.2406, Metric Accuracy: 76.13%\n",
      "\n",
      "Train Epoch: 3 [48000/60000]\tVAE Loss: 109.2453 (111.8493) \tMetric Loss: 0.2547 (0.2411) \tMetric Acc: 74.17% (76.05%) \tEmb_Norm: 49.56 (50.13)\n",
      "\n",
      "Test set: Average VAE loss: 111.7189, Average Metric loss: 0.2406, Metric Accuracy: 76.06%\n",
      "\n",
      "Train Epoch: 3 [54000/60000]\tVAE Loss: 107.7579 (111.6774) \tMetric Loss: 0.2447 (0.2407) \tMetric Acc: 76.67% (76.07%) \tEmb_Norm: 48.66 (50.05)\n",
      "\n",
      "Test set: Average VAE loss: 111.5676, Average Metric loss: 0.2401, Metric Accuracy: 76.11%\n",
      "\n",
      "Train Epoch: 4 [0/60000]\tVAE Loss: 110.8919 (110.8919) \tMetric Loss: 0.2510 (0.2510) \tMetric Acc: 75.83% (75.83%) \tEmb_Norm: 49.26 (49.26)\n",
      "\n",
      "Test set: Average VAE loss: 109.5670, Average Metric loss: 0.2333, Metric Accuracy: 76.37%\n",
      "\n",
      "Train Epoch: 4 [6000/60000]\tVAE Loss: 111.1338 (109.8467) \tMetric Loss: 0.2008 (0.2350) \tMetric Acc: 76.67% (76.22%) \tEmb_Norm: 48.86 (49.40)\n",
      "\n",
      "Test set: Average VAE loss: 109.6472, Average Metric loss: 0.2334, Metric Accuracy: 76.32%\n",
      "\n",
      "Train Epoch: 4 [12000/60000]\tVAE Loss: 109.4477 (109.8132) \tMetric Loss: 0.2821 (0.2378) \tMetric Acc: 70.00% (75.87%) \tEmb_Norm: 49.30 (49.36)\n",
      "\n",
      "Test set: Average VAE loss: 109.6727, Average Metric loss: 0.2371, Metric Accuracy: 75.93%\n",
      "\n",
      "Train Epoch: 4 [18000/60000]\tVAE Loss: 108.1011 (109.6897) \tMetric Loss: 0.2053 (0.2380) \tMetric Acc: 80.00% (75.94%) \tEmb_Norm: 49.29 (49.37)\n",
      "\n",
      "Test set: Average VAE loss: 109.5818, Average Metric loss: 0.2369, Metric Accuracy: 75.98%\n",
      "\n",
      "Train Epoch: 4 [24000/60000]\tVAE Loss: 109.5990 (109.6090) \tMetric Loss: 0.1840 (0.2372) \tMetric Acc: 81.67% (75.96%) \tEmb_Norm: 48.96 (49.32)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Test set: Average VAE loss: 109.5024, Average Metric loss: 0.2364, Metric Accuracy: 76.00%\n",
      "\n",
      "Train Epoch: 4 [30000/60000]\tVAE Loss: 109.6153 (109.5273) \tMetric Loss: 0.2379 (0.2374) \tMetric Acc: 72.50% (75.91%) \tEmb_Norm: 48.15 (49.28)\n",
      "\n",
      "Test set: Average VAE loss: 109.4446, Average Metric loss: 0.2367, Metric Accuracy: 75.92%\n",
      "\n",
      "Train Epoch: 4 [36000/60000]\tVAE Loss: 110.5205 (109.4312) \tMetric Loss: 0.2641 (0.2367) \tMetric Acc: 75.83% (75.90%) \tEmb_Norm: 48.55 (49.17)\n",
      "\n",
      "Test set: Average VAE loss: 109.3520, Average Metric loss: 0.2363, Metric Accuracy: 75.93%\n",
      "\n",
      "Train Epoch: 4 [42000/60000]\tVAE Loss: 107.1107 (109.3330) \tMetric Loss: 0.2947 (0.2365) \tMetric Acc: 70.83% (75.87%) \tEmb_Norm: 48.30 (49.11)\n",
      "\n",
      "Test set: Average VAE loss: 109.2531, Average Metric loss: 0.2363, Metric Accuracy: 75.88%\n",
      "\n",
      "Train Epoch: 4 [48000/60000]\tVAE Loss: 110.9476 (109.2356) \tMetric Loss: 0.2166 (0.2365) \tMetric Acc: 79.17% (75.88%) \tEmb_Norm: 48.92 (49.07)\n",
      "\n",
      "Test set: Average VAE loss: 109.1639, Average Metric loss: 0.2361, Metric Accuracy: 75.90%\n",
      "\n",
      "Train Epoch: 4 [54000/60000]\tVAE Loss: 108.5283 (109.1410) \tMetric Loss: 0.2062 (0.2367) \tMetric Acc: 77.50% (75.86%) \tEmb_Norm: 49.02 (49.05)\n",
      "\n",
      "Test set: Average VAE loss: 109.0817, Average Metric loss: 0.2363, Metric Accuracy: 75.89%\n",
      "\n",
      "Train Epoch: 5 [0/60000]\tVAE Loss: 111.3086 (111.3086) \tMetric Loss: 0.2466 (0.2466) \tMetric Acc: 73.33% (73.33%) \tEmb_Norm: 48.54 (48.54)\n",
      "\n",
      "Test set: Average VAE loss: 108.0519, Average Metric loss: 0.2289, Metric Accuracy: 76.20%\n",
      "\n",
      "Train Epoch: 5 [6000/60000]\tVAE Loss: 105.1769 (108.0958) \tMetric Loss: 0.1789 (0.2294) \tMetric Acc: 79.17% (76.28%) \tEmb_Norm: 48.30 (48.58)\n",
      "\n",
      "Test set: Average VAE loss: 108.0261, Average Metric loss: 0.2288, Metric Accuracy: 76.26%\n",
      "\n",
      "Train Epoch: 5 [12000/60000]\tVAE Loss: 107.6234 (108.0467) \tMetric Loss: 0.2473 (0.2309) \tMetric Acc: 73.33% (76.08%) \tEmb_Norm: 48.28 (48.62)\n",
      "\n",
      "Test set: Average VAE loss: 107.9667, Average Metric loss: 0.2310, Metric Accuracy: 76.06%\n",
      "\n",
      "Train Epoch: 5 [18000/60000]\tVAE Loss: 109.5254 (108.0181) \tMetric Loss: 0.2325 (0.2330) \tMetric Acc: 75.00% (75.88%) \tEmb_Norm: 48.16 (48.63)\n",
      "\n",
      "Test set: Average VAE loss: 107.9324, Average Metric loss: 0.2316, Metric Accuracy: 75.98%\n",
      "\n",
      "Train Epoch: 5 [24000/60000]\tVAE Loss: 108.0681 (107.9452) \tMetric Loss: 0.2707 (0.2324) \tMetric Acc: 69.17% (75.90%) \tEmb_Norm: 48.39 (48.57)\n",
      "\n",
      "Test set: Average VAE loss: 107.8866, Average Metric loss: 0.2317, Metric Accuracy: 75.94%\n",
      "\n",
      "Train Epoch: 5 [30000/60000]\tVAE Loss: 109.2265 (107.8783) \tMetric Loss: 0.2630 (0.2325) \tMetric Acc: 75.00% (75.86%) \tEmb_Norm: 48.81 (48.54)\n",
      "\n",
      "Test set: Average VAE loss: 107.8062, Average Metric loss: 0.2320, Metric Accuracy: 75.88%\n",
      "\n",
      "Train Epoch: 5 [36000/60000]\tVAE Loss: 107.3945 (107.8220) \tMetric Loss: 0.2208 (0.2321) \tMetric Acc: 76.67% (75.89%) \tEmb_Norm: 48.29 (48.51)\n",
      "\n",
      "Test set: Average VAE loss: 107.7618, Average Metric loss: 0.2320, Metric Accuracy: 75.90%\n",
      "\n",
      "Train Epoch: 5 [42000/60000]\tVAE Loss: 110.3224 (107.7781) \tMetric Loss: 0.2475 (0.2322) \tMetric Acc: 73.33% (75.87%) \tEmb_Norm: 49.32 (48.48)\n",
      "\n",
      "Test set: Average VAE loss: 107.7211, Average Metric loss: 0.2315, Metric Accuracy: 75.91%\n",
      "\n",
      "Train Epoch: 5 [48000/60000]\tVAE Loss: 106.3145 (107.7110) \tMetric Loss: 0.2115 (0.2322) \tMetric Acc: 77.50% (75.81%) \tEmb_Norm: 47.81 (48.44)\n",
      "\n",
      "Test set: Average VAE loss: 107.6599, Average Metric loss: 0.2319, Metric Accuracy: 75.82%\n",
      "\n",
      "Train Epoch: 5 [54000/60000]\tVAE Loss: 108.5648 (107.6739) \tMetric Loss: 0.2401 (0.2322) \tMetric Acc: 75.00% (75.79%) \tEmb_Norm: 48.03 (48.42)\n",
      "\n",
      "Test set: Average VAE loss: 107.6224, Average Metric loss: 0.2320, Metric Accuracy: 75.81%\n",
      "\n",
      "Train Epoch: 6 [0/60000]\tVAE Loss: 110.1471 (110.1471) \tMetric Loss: 0.3198 (0.3198) \tMetric Acc: 70.00% (70.00%) \tEmb_Norm: 48.42 (48.42)\n",
      "\n",
      "Test set: Average VAE loss: 106.9283, Average Metric loss: 0.2289, Metric Accuracy: 76.00%\n",
      "\n",
      "Train Epoch: 6 [6000/60000]\tVAE Loss: 105.3078 (107.0663) \tMetric Loss: 0.1457 (0.2301) \tMetric Acc: 83.33% (75.87%) \tEmb_Norm: 48.12 (48.20)\n",
      "\n",
      "Test set: Average VAE loss: 106.9644, Average Metric loss: 0.2291, Metric Accuracy: 75.99%\n",
      "\n",
      "Train Epoch: 6 [12000/60000]\tVAE Loss: 105.5832 (106.9573) \tMetric Loss: 0.3416 (0.2294) \tMetric Acc: 68.33% (75.93%) \tEmb_Norm: 48.48 (48.16)\n",
      "\n",
      "Test set: Average VAE loss: 106.9026, Average Metric loss: 0.2289, Metric Accuracy: 75.98%\n",
      "\n",
      "Train Epoch: 6 [18000/60000]\tVAE Loss: 105.1930 (106.9243) \tMetric Loss: 0.2774 (0.2307) \tMetric Acc: 71.67% (75.87%) \tEmb_Norm: 47.94 (48.12)\n",
      "\n",
      "Test set: Average VAE loss: 106.8684, Average Metric loss: 0.2303, Metric Accuracy: 75.87%\n",
      "\n",
      "Train Epoch: 6 [24000/60000]\tVAE Loss: 107.1943 (106.8603) \tMetric Loss: 0.2252 (0.2312) \tMetric Acc: 75.00% (75.83%) \tEmb_Norm: 47.97 (48.11)\n",
      "\n",
      "Test set: Average VAE loss: 106.8079, Average Metric loss: 0.2305, Metric Accuracy: 75.89%\n",
      "\n",
      "Train Epoch: 6 [30000/60000]\tVAE Loss: 107.3117 (106.8311) \tMetric Loss: 0.2243 (0.2314) \tMetric Acc: 75.00% (75.84%) \tEmb_Norm: 47.53 (48.12)\n",
      "\n",
      "Test set: Average VAE loss: 106.7902, Average Metric loss: 0.2312, Metric Accuracy: 75.85%\n",
      "\n",
      "Train Epoch: 6 [36000/60000]\tVAE Loss: 108.3233 (106.8092) \tMetric Loss: 0.3262 (0.2321) \tMetric Acc: 68.33% (75.77%) \tEmb_Norm: 48.15 (48.07)\n",
      "\n",
      "Test set: Average VAE loss: 106.7798, Average Metric loss: 0.2318, Metric Accuracy: 75.78%\n",
      "\n",
      "Train Epoch: 6 [42000/60000]\tVAE Loss: 107.7218 (106.7931) \tMetric Loss: 0.2961 (0.2321) \tMetric Acc: 67.50% (75.74%) \tEmb_Norm: 47.78 (48.07)\n",
      "\n",
      "Test set: Average VAE loss: 106.7564, Average Metric loss: 0.2318, Metric Accuracy: 75.77%\n",
      "\n",
      "Train Epoch: 6 [48000/60000]\tVAE Loss: 105.2670 (106.7514) \tMetric Loss: 0.2306 (0.2322) \tMetric Acc: 79.17% (75.75%) \tEmb_Norm: 47.83 (48.05)\n",
      "\n",
      "Test set: Average VAE loss: 106.7154, Average Metric loss: 0.2317, Metric Accuracy: 75.78%\n",
      "\n",
      "Train Epoch: 6 [54000/60000]\tVAE Loss: 107.1888 (106.7104) \tMetric Loss: 0.1725 (0.2318) \tMetric Acc: 82.50% (75.78%) \tEmb_Norm: 47.95 (48.03)\n",
      "\n",
      "Test set: Average VAE loss: 106.6758, Average Metric loss: 0.2317, Metric Accuracy: 75.77%\n",
      "\n",
      "Train Epoch: 7 [0/60000]\tVAE Loss: 107.5169 (107.5169) \tMetric Loss: 0.2715 (0.2715) \tMetric Acc: 73.33% (73.33%) \tEmb_Norm: 47.35 (47.35)\n",
      "\n",
      "Test set: Average VAE loss: 106.0055, Average Metric loss: 0.2280, Metric Accuracy: 75.92%\n",
      "\n",
      "Train Epoch: 7 [6000/60000]\tVAE Loss: 102.9215 (106.1778) \tMetric Loss: 0.2826 (0.2316) \tMetric Acc: 75.83% (75.99%) \tEmb_Norm: 47.32 (47.69)\n",
      "\n",
      "Test set: Average VAE loss: 106.1286, Average Metric loss: 0.2305, Metric Accuracy: 75.89%\n",
      "\n",
      "Train Epoch: 7 [12000/60000]\tVAE Loss: 105.9734 (106.1744) \tMetric Loss: 0.3200 (0.2332) \tMetric Acc: 69.17% (75.78%) \tEmb_Norm: 48.21 (47.60)\n",
      "\n",
      "Test set: Average VAE loss: 106.1943, Average Metric loss: 0.2316, Metric Accuracy: 75.87%\n",
      "\n",
      "Train Epoch: 7 [18000/60000]\tVAE Loss: 106.0126 (106.1977) \tMetric Loss: 0.2038 (0.2340) \tMetric Acc: 79.17% (75.63%) \tEmb_Norm: 47.78 (47.62)\n",
      "\n",
      "Test set: Average VAE loss: 106.1342, Average Metric loss: 0.2328, Metric Accuracy: 75.68%\n",
      "\n",
      "Train Epoch: 7 [24000/60000]\tVAE Loss: 105.4992 (106.1444) \tMetric Loss: 0.2147 (0.2332) \tMetric Acc: 75.83% (75.61%) \tEmb_Norm: 47.41 (47.62)\n",
      "\n",
      "Test set: Average VAE loss: 106.1042, Average Metric loss: 0.2327, Metric Accuracy: 75.63%\n",
      "\n",
      "Train Epoch: 7 [30000/60000]\tVAE Loss: 105.3056 (106.0991) \tMetric Loss: 0.2205 (0.2328) \tMetric Acc: 78.33% (75.60%) \tEmb_Norm: 47.50 (47.61)\n",
      "\n",
      "Test set: Average VAE loss: 106.0754, Average Metric loss: 0.2323, Metric Accuracy: 75.62%\n",
      "\n",
      "Train Epoch: 7 [36000/60000]\tVAE Loss: 107.4064 (106.0846) \tMetric Loss: 0.1736 (0.2323) \tMetric Acc: 77.50% (75.57%) \tEmb_Norm: 47.73 (47.61)\n",
      "\n",
      "Test set: Average VAE loss: 106.0504, Average Metric loss: 0.2318, Metric Accuracy: 75.62%\n",
      "\n",
      "Train Epoch: 7 [42000/60000]\tVAE Loss: 105.4324 (106.0665) \tMetric Loss: 0.2163 (0.2322) \tMetric Acc: 74.17% (75.59%) \tEmb_Norm: 47.84 (47.61)\n",
      "\n",
      "Test set: Average VAE loss: 106.0448, Average Metric loss: 0.2320, Metric Accuracy: 75.61%\n",
      "\n",
      "Train Epoch: 7 [48000/60000]\tVAE Loss: 107.0968 (106.0461) \tMetric Loss: 0.1900 (0.2323) \tMetric Acc: 74.17% (75.58%) \tEmb_Norm: 47.51 (47.60)\n",
      "\n",
      "Test set: Average VAE loss: 106.0192, Average Metric loss: 0.2320, Metric Accuracy: 75.60%\n",
      "\n",
      "Train Epoch: 7 [54000/60000]\tVAE Loss: 107.0962 (106.0147) \tMetric Loss: 0.2894 (0.2323) \tMetric Acc: 72.50% (75.55%) \tEmb_Norm: 47.65 (47.58)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Test set: Average VAE loss: 105.9968, Average Metric loss: 0.2321, Metric Accuracy: 75.55%\n",
      "\n",
      "Train Epoch: 8 [0/60000]\tVAE Loss: 104.1939 (104.1939) \tMetric Loss: 0.1934 (0.1934) \tMetric Acc: 78.33% (78.33%) \tEmb_Norm: 47.21 (47.21)\n",
      "\n",
      "Test set: Average VAE loss: 105.5562, Average Metric loss: 0.2273, Metric Accuracy: 75.77%\n",
      "\n",
      "Train Epoch: 8 [6000/60000]\tVAE Loss: 105.4516 (105.6203) \tMetric Loss: 0.2242 (0.2339) \tMetric Acc: 75.83% (75.53%) \tEmb_Norm: 47.71 (47.49)\n",
      "\n",
      "Test set: Average VAE loss: 105.6007, Average Metric loss: 0.2321, Metric Accuracy: 75.56%\n",
      "\n",
      "Train Epoch: 8 [12000/60000]\tVAE Loss: 105.4116 (105.6250) \tMetric Loss: 0.2086 (0.2302) \tMetric Acc: 75.83% (75.65%) \tEmb_Norm: 47.22 (47.46)\n",
      "\n",
      "Test set: Average VAE loss: 105.5955, Average Metric loss: 0.2298, Metric Accuracy: 75.66%\n",
      "\n",
      "Train Epoch: 8 [18000/60000]\tVAE Loss: 104.0175 (105.6186) \tMetric Loss: 0.2561 (0.2313) \tMetric Acc: 71.67% (75.56%) \tEmb_Norm: 47.65 (47.46)\n",
      "\n",
      "Test set: Average VAE loss: 105.5603, Average Metric loss: 0.2308, Metric Accuracy: 75.57%\n",
      "\n",
      "Train Epoch: 8 [24000/60000]\tVAE Loss: 106.6680 (105.5506) \tMetric Loss: 0.2591 (0.2325) \tMetric Acc: 74.17% (75.45%) \tEmb_Norm: 47.05 (47.45)\n",
      "\n",
      "Test set: Average VAE loss: 105.5244, Average Metric loss: 0.2320, Metric Accuracy: 75.46%\n",
      "\n",
      "Train Epoch: 8 [30000/60000]\tVAE Loss: 104.0885 (105.5367) \tMetric Loss: 0.1872 (0.2332) \tMetric Acc: 77.50% (75.38%) \tEmb_Norm: 47.20 (47.39)\n",
      "\n",
      "Test set: Average VAE loss: 105.5185, Average Metric loss: 0.2328, Metric Accuracy: 75.42%\n",
      "\n",
      "Train Epoch: 8 [36000/60000]\tVAE Loss: 104.6771 (105.5190) \tMetric Loss: 0.2165 (0.2343) \tMetric Acc: 78.33% (75.30%) \tEmb_Norm: 47.30 (47.40)\n",
      "\n",
      "Test set: Average VAE loss: 105.4870, Average Metric loss: 0.2337, Metric Accuracy: 75.34%\n",
      "\n",
      "Train Epoch: 8 [42000/60000]\tVAE Loss: 105.2462 (105.4954) \tMetric Loss: 0.2544 (0.2339) \tMetric Acc: 73.33% (75.33%) \tEmb_Norm: 47.50 (47.40)\n",
      "\n",
      "Test set: Average VAE loss: 105.4705, Average Metric loss: 0.2335, Metric Accuracy: 75.33%\n",
      "\n",
      "Train Epoch: 8 [48000/60000]\tVAE Loss: 102.6592 (105.4773) \tMetric Loss: 0.2219 (0.2336) \tMetric Acc: 79.17% (75.31%) \tEmb_Norm: 46.99 (47.37)\n",
      "\n",
      "Test set: Average VAE loss: 105.4587, Average Metric loss: 0.2333, Metric Accuracy: 75.31%\n",
      "\n",
      "Train Epoch: 8 [54000/60000]\tVAE Loss: 106.9129 (105.4556) \tMetric Loss: 0.2945 (0.2332) \tMetric Acc: 69.17% (75.30%) \tEmb_Norm: 47.57 (47.36)\n",
      "\n",
      "Test set: Average VAE loss: 105.4351, Average Metric loss: 0.2330, Metric Accuracy: 75.32%\n",
      "\n",
      "Train Epoch: 9 [0/60000]\tVAE Loss: 105.4779 (105.4779) \tMetric Loss: 0.2031 (0.2031) \tMetric Acc: 75.83% (75.83%) \tEmb_Norm: 46.81 (46.81)\n",
      "\n",
      "Test set: Average VAE loss: 105.0736, Average Metric loss: 0.2294, Metric Accuracy: 75.59%\n",
      "\n",
      "Train Epoch: 9 [6000/60000]\tVAE Loss: 107.6513 (105.0248) \tMetric Loss: 0.2333 (0.2347) \tMetric Acc: 76.67% (75.16%) \tEmb_Norm: 47.66 (47.20)\n",
      "\n",
      "Test set: Average VAE loss: 105.0234, Average Metric loss: 0.2323, Metric Accuracy: 75.39%\n",
      "\n",
      "Train Epoch: 9 [12000/60000]\tVAE Loss: 104.2797 (105.0543) \tMetric Loss: 0.1501 (0.2323) \tMetric Acc: 79.17% (75.42%) \tEmb_Norm: 47.59 (47.20)\n",
      "\n",
      "Test set: Average VAE loss: 105.0599, Average Metric loss: 0.2315, Metric Accuracy: 75.47%\n",
      "\n",
      "Train Epoch: 9 [18000/60000]\tVAE Loss: 105.2810 (105.0919) \tMetric Loss: 0.2750 (0.2330) \tMetric Acc: 71.67% (75.36%) \tEmb_Norm: 47.54 (47.23)\n",
      "\n",
      "Test set: Average VAE loss: 105.0741, Average Metric loss: 0.2329, Metric Accuracy: 75.36%\n",
      "\n",
      "Train Epoch: 9 [24000/60000]\tVAE Loss: 106.8982 (105.1038) \tMetric Loss: 0.1745 (0.2343) \tMetric Acc: 77.50% (75.25%) \tEmb_Norm: 47.19 (47.27)\n",
      "\n",
      "Test set: Average VAE loss: 105.0816, Average Metric loss: 0.2336, Metric Accuracy: 75.29%\n",
      "\n",
      "Train Epoch: 9 [30000/60000]\tVAE Loss: 104.9235 (105.1036) \tMetric Loss: 0.2337 (0.2344) \tMetric Acc: 80.83% (75.20%) \tEmb_Norm: 47.02 (47.24)\n",
      "\n",
      "Test set: Average VAE loss: 105.0985, Average Metric loss: 0.2339, Metric Accuracy: 75.24%\n",
      "\n",
      "Train Epoch: 9 [36000/60000]\tVAE Loss: 106.2310 (105.1038) \tMetric Loss: 0.3016 (0.2346) \tMetric Acc: 70.83% (75.20%) \tEmb_Norm: 47.18 (47.23)\n",
      "\n",
      "Test set: Average VAE loss: 105.0917, Average Metric loss: 0.2342, Metric Accuracy: 75.19%\n",
      "\n",
      "Train Epoch: 9 [42000/60000]\tVAE Loss: 104.2457 (105.1063) \tMetric Loss: 0.1989 (0.2345) \tMetric Acc: 77.50% (75.17%) \tEmb_Norm: 47.51 (47.21)\n",
      "\n",
      "Test set: Average VAE loss: 105.0846, Average Metric loss: 0.2339, Metric Accuracy: 75.20%\n",
      "\n",
      "Train Epoch: 9 [48000/60000]\tVAE Loss: 104.8508 (105.0761) \tMetric Loss: 0.2984 (0.2343) \tMetric Acc: 75.00% (75.16%) \tEmb_Norm: 47.31 (47.20)\n",
      "\n",
      "Test set: Average VAE loss: 105.0608, Average Metric loss: 0.2339, Metric Accuracy: 75.19%\n",
      "\n",
      "Train Epoch: 9 [54000/60000]\tVAE Loss: 106.8833 (105.0524) \tMetric Loss: 0.2574 (0.2343) \tMetric Acc: 73.33% (75.16%) \tEmb_Norm: 47.35 (47.20)\n",
      "\n",
      "Test set: Average VAE loss: 105.0387, Average Metric loss: 0.2340, Metric Accuracy: 75.18%\n",
      "\n",
      "Train Epoch: 10 [0/60000]\tVAE Loss: 104.8552 (104.8552) \tMetric Loss: 0.1739 (0.1739) \tMetric Acc: 81.67% (81.67%) \tEmb_Norm: 46.88 (46.88)\n",
      "\n",
      "Test set: Average VAE loss: 104.7929, Average Metric loss: 0.2285, Metric Accuracy: 75.59%\n",
      "\n",
      "Train Epoch: 10 [6000/60000]\tVAE Loss: 105.5561 (104.7840) \tMetric Loss: 0.2973 (0.2343) \tMetric Acc: 75.00% (75.00%) \tEmb_Norm: 47.17 (47.10)\n",
      "\n",
      "Test set: Average VAE loss: 104.8192, Average Metric loss: 0.2332, Metric Accuracy: 75.16%\n",
      "\n",
      "Train Epoch: 10 [12000/60000]\tVAE Loss: 105.3444 (104.7933) \tMetric Loss: 0.2900 (0.2356) \tMetric Acc: 72.50% (75.07%) \tEmb_Norm: 47.12 (47.08)\n",
      "\n",
      "Test set: Average VAE loss: 104.7799, Average Metric loss: 0.2345, Metric Accuracy: 75.16%\n",
      "\n",
      "Train Epoch: 10 [18000/60000]\tVAE Loss: 104.4806 (104.7667) \tMetric Loss: 0.2807 (0.2363) \tMetric Acc: 68.33% (75.06%) \tEmb_Norm: 46.95 (47.12)\n",
      "\n",
      "Test set: Average VAE loss: 104.7642, Average Metric loss: 0.2356, Metric Accuracy: 75.10%\n",
      "\n",
      "Train Epoch: 10 [24000/60000]\tVAE Loss: 104.3822 (104.7446) \tMetric Loss: 0.1867 (0.2361) \tMetric Acc: 76.67% (74.98%) \tEmb_Norm: 46.99 (47.14)\n",
      "\n",
      "Test set: Average VAE loss: 104.7107, Average Metric loss: 0.2355, Metric Accuracy: 75.02%\n",
      "\n",
      "Train Epoch: 10 [30000/60000]\tVAE Loss: 102.2304 (104.7159) \tMetric Loss: 0.1569 (0.2365) \tMetric Acc: 80.00% (74.96%) \tEmb_Norm: 47.17 (47.10)\n",
      "\n",
      "Test set: Average VAE loss: 104.7016, Average Metric loss: 0.2358, Metric Accuracy: 75.03%\n",
      "\n",
      "Train Epoch: 10 [36000/60000]\tVAE Loss: 104.9063 (104.7142) \tMetric Loss: 0.2637 (0.2367) \tMetric Acc: 70.83% (74.99%) \tEmb_Norm: 47.29 (47.07)\n",
      "\n",
      "Test set: Average VAE loss: 104.6987, Average Metric loss: 0.2362, Metric Accuracy: 75.02%\n",
      "\n",
      "Train Epoch: 10 [42000/60000]\tVAE Loss: 105.2107 (104.6867) \tMetric Loss: 0.1857 (0.2361) \tMetric Acc: 79.17% (75.06%) \tEmb_Norm: 47.31 (47.06)\n",
      "\n",
      "Test set: Average VAE loss: 104.6800, Average Metric loss: 0.2356, Metric Accuracy: 75.09%\n",
      "\n",
      "Train Epoch: 10 [48000/60000]\tVAE Loss: 106.9542 (104.6930) \tMetric Loss: 0.3421 (0.2360) \tMetric Acc: 70.00% (75.06%) \tEmb_Norm: 47.32 (47.06)\n",
      "\n",
      "Test set: Average VAE loss: 104.6774, Average Metric loss: 0.2355, Metric Accuracy: 75.09%\n",
      "\n",
      "Train Epoch: 10 [54000/60000]\tVAE Loss: 103.6959 (104.6751) \tMetric Loss: 0.2274 (0.2362) \tMetric Acc: 73.33% (75.05%) \tEmb_Norm: 47.15 (47.05)\n",
      "\n",
      "Test set: Average VAE loss: 104.6646, Average Metric loss: 0.2358, Metric Accuracy: 75.08%\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "from __future__ import print_function\n",
    "import argparse\n",
    "import torch\n",
    "import torch.utils.data\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.autograd import Variable\n",
    "from torchvision import datasets, transforms\n",
    "from triplet_mnist_loader import MNIST_t\n",
    "import torch.nn.functional as F\n",
    "\n",
    "batch_size = 120\n",
    "epochs = 10\n",
    "no_cuda = False\n",
    "seed = 1\n",
    "log_interval = 50\n",
    "margin = 0.2\n",
    "cuda = not no_cuda and torch.cuda.is_available()\n",
    "\n",
    "\n",
    "torch.manual_seed(seed)\n",
    "if cuda:\n",
    "    torch.cuda.manual_seed(seed)\n",
    "\n",
    "\n",
    "kwargs = {'num_workers': 1, 'pin_memory': True} if cuda else {}\n",
    "train_loader = torch.utils.data.DataLoader(\n",
    "        MNIST_t('../data', train=True, download=True,\n",
    "                       transform=transforms.ToTensor()),\n",
    "        batch_size=batch_size, shuffle=True, **kwargs)\n",
    "test_loader = torch.utils.data.DataLoader(\n",
    "        MNIST_t('../data', train=False, download=True,\n",
    "                       transform=transforms.ToTensor()),\n",
    "        batch_size=batch_size, shuffle=True, **kwargs)\n",
    "\n",
    "\n",
    "class VAE(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(VAE, self).__init__()\n",
    "\n",
    "        self.fc1 = nn.Linear(784, 400)\n",
    "        self.fc21 = nn.Linear(400, 20)\n",
    "        self.fc22 = nn.Linear(400, 20)\n",
    "        self.fc3 = nn.Linear(20, 400)\n",
    "        self.fc4 = nn.Linear(400, 784)\n",
    "\n",
    "        self.relu = nn.ReLU()\n",
    "        self.sigmoid = nn.Sigmoid()\n",
    "\n",
    "    def encode(self, x):\n",
    "        h1 = self.relu(self.fc1(x))\n",
    "        return self.fc21(h1), self.fc22(h1)\n",
    "\n",
    "    def reparametrize(self, mu, logvar):\n",
    "        std = logvar.mul(0.5).exp_()\n",
    "        if cuda:\n",
    "            eps = torch.cuda.FloatTensor(std.size()).normal_()\n",
    "        else:\n",
    "            eps = torch.FloatTensor(std.size()).normal_()\n",
    "        eps = Variable(eps)\n",
    "        return eps.mul(std).add_(mu)\n",
    "\n",
    "    def decode(self, z):\n",
    "        h3 = self.relu(self.fc3(z))\n",
    "        return self.sigmoid(self.fc4(h3))\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = x.view(-1, 784)\n",
    "        h1 = self.relu(self.fc1(x))\n",
    "        mu = self.fc21(h1)\n",
    "        logvar = self.fc22(h1)\n",
    "        z = self.reparametrize(mu, logvar)\n",
    "        return self.decode(z), mu, logvar\n",
    "\n",
    "\n",
    "model = VAE()\n",
    "if cuda:\n",
    "    model.cuda()\n",
    "\n",
    "reconstruction_function = nn.BCELoss()\n",
    "reconstruction_function.size_average = False\n",
    "\n",
    "\n",
    "def loss_function(recon_x, x, mu, logvar):\n",
    "    BCE = reconstruction_function(recon_x, x)\n",
    "\n",
    "    # see Appendix B from VAE paper:\n",
    "    # Kingma and Welling. Auto-Encoding Variational Bayes. ICLR, 2014\n",
    "    # https://arxiv.org/abs/1312.6114\n",
    "    # 0.5 * sum(1 + log(sigma^2) - mu^2 - sigma^2)\n",
    "    KLD_element = mu.pow(2).add_(logvar.exp()).mul_(-1).add_(1).add_(logvar)\n",
    "    KLD = torch.sum(KLD_element).mul_(-0.5)\n",
    "\n",
    "    return BCE + KLD\n",
    "\n",
    "\n",
    "optimizer = optim.Adam(model.parameters(), lr=1e-3)\n",
    "criterion = torch.nn.MarginRankingLoss(margin = margin)\n",
    "\n",
    "\n",
    "class AverageMeter(object):\n",
    "    \"\"\"Computes and stores the average and current value\"\"\"\n",
    "    def __init__(self):\n",
    "        self.reset()\n",
    "\n",
    "    def reset(self):\n",
    "        self.val = 0\n",
    "        self.avg = 0\n",
    "        self.sum = 0\n",
    "        self.count = 0\n",
    "\n",
    "    def update(self, val, n=1):\n",
    "        self.val = val\n",
    "        self.sum += val * n\n",
    "        self.count += n\n",
    "        self.avg = self.sum / self.count\n",
    "        \n",
    "def accuracy(dista, distb):\n",
    "    margin = 0\n",
    "    pred = (dista - distb - margin).cpu().data\n",
    "    return (pred > 0).sum()*1.0/dista.size()[0]\n",
    "\n",
    "train_loss_metric = []\n",
    "train_loss_VAE = []\n",
    "train_acc_metric = []\n",
    "test_loss_metric = []\n",
    "test_loss_VAE = []\n",
    "test_acc_metric = []\n",
    "\n",
    "for epoch in range(1, epochs + 1):\n",
    "    losses_metric = AverageMeter()\n",
    "    losses_VAE = AverageMeter()\n",
    "    accs = AverageMeter()\n",
    "    emb_norms = AverageMeter()\n",
    "\n",
    "    model.train()\n",
    "    for batch_idx, (data1, data2, data3) in enumerate(train_loader):\n",
    "        if cuda:\n",
    "            data1, data2, data3 = data1.cuda(), data2.cuda(), data3.cuda()\n",
    "        data1, data2, data3 = Variable(data1), Variable(data2), Variable(data3)\n",
    "\n",
    "        recon_batch1, mu1, logvar1 = model(data1)\n",
    "        recon_batch2, mu2, logvar2 = model(data2)\n",
    "        recon_batch3, mu3, logvar3 = model(data3)\n",
    "\n",
    "        loss_vae = loss_function(recon_batch1, data1, mu1, logvar1)     \n",
    "        loss_vae += loss_function(recon_batch2, data2, mu2, logvar2)  \n",
    "        loss_vae += loss_function(recon_batch3, data3, mu3, logvar3)  \n",
    "        loss_vae = loss_vae/(3*len(data1))\n",
    "\n",
    "        dista = F.pairwise_distance(mu1, mu2, 2)\n",
    "        distb = F.pairwise_distance(mu1, mu3, 2)\n",
    "        target = torch.FloatTensor(dista.size()).fill_(1)\n",
    "        if cuda:\n",
    "            target = target.cuda()\n",
    "        target = Variable(target)\n",
    "        loss_triplet = criterion(dista, distb, target)\n",
    "\n",
    "        loss_embedd = mu1.norm(2) + mu2.norm(2) + mu3.norm(2)\n",
    "\n",
    "        loss = 0.01*loss_vae # + loss_triplet + 0.001*loss_embedd\n",
    "\n",
    "        # measure accuracy and record loss\n",
    "        acc = accuracy(dista, distb)\n",
    "        losses_metric.update(loss_triplet.data[0], data1.size(0))\n",
    "        losses_VAE.update(loss_vae.data[0], data1.size(0))\n",
    "        accs.update(acc, data1.size(0))\n",
    "        emb_norms.update(loss_embedd.data[0]/3, data1.size(0))\n",
    "\n",
    "        # train\n",
    "        optimizer.zero_grad()          \n",
    "        loss.backward()\n",
    "        optimizer.step()  \n",
    "        if batch_idx % log_interval == 0:\n",
    "            print('Train Epoch: {} [{}/{}]\\t'\n",
    "                  'VAE Loss: {:.4f} ({:.4f}) \\t'\n",
    "                  'Metric Loss: {:.4f} ({:.4f}) \\t'\n",
    "                  'Metric Acc: {:.2f}% ({:.2f}%) \\t'\n",
    "                  'Emb_Norm: {:.2f} ({:.2f})'.format(\n",
    "                epoch, batch_idx * len(data1), len(train_loader.dataset),\n",
    "                losses_VAE.val, losses_VAE.avg,\n",
    "                losses_metric.val, losses_metric.avg, \n",
    "                100. * accs.val, 100. * accs.avg, emb_norms.val, emb_norms.avg))\n",
    "\n",
    "            train_loss_metric.append(losses_metric.val)\n",
    "            train_loss_VAE.append(losses_VAE.val)\n",
    "            train_acc_metric.append(accs.val)\n",
    "\n",
    "           # plot(data1, recon_batch1, recon_batch1)\n",
    "\n",
    "            # Test\n",
    "            model.eval()\n",
    "            for batch_idx, (data1, data2, data3) in enumerate(test_loader):\n",
    "                if cuda:\n",
    "                    data1, data2, data3 = data1.cuda(), data2.cuda(), data3.cuda()\n",
    "                data1, data2, data3 = Variable(data1), Variable(data2), Variable(data3)\n",
    "\n",
    "                recon_batch1, mu1, logvar1 = model(data1)\n",
    "                recon_batch2, mu2, logvar2 = model(data2)\n",
    "                recon_batch3, mu3, logvar3 = model(data3)\n",
    "\n",
    "                loss_vae = loss_function(recon_batch1, data1, mu1, logvar1)     \n",
    "                loss_vae += loss_function(recon_batch2, data2, mu2, logvar2)  \n",
    "                loss_vae += loss_function(recon_batch3, data3, mu3, logvar3)  \n",
    "                loss_vae = loss_vae/(3*len(data1))\n",
    "\n",
    "                dista = F.pairwise_distance(mu1, mu2, 2)\n",
    "                distb = F.pairwise_distance(mu1, mu3, 2)\n",
    "                target = torch.FloatTensor(dista.size()).fill_(1)\n",
    "                if cuda:\n",
    "                    target = target.cuda()\n",
    "                target = Variable(target)\n",
    "                loss_triplet = criterion(dista, distb, target)\n",
    "\n",
    "                loss_embedd = mu1.norm(2) + mu2.norm(2) + mu3.norm(2)\n",
    "\n",
    "                loss = 0.01*loss_vae + loss_triplet + 0.001*loss_embedd\n",
    "\n",
    "                # measure accuracy and record loss\n",
    "                acc = accuracy(dista, distb)\n",
    "                losses_metric.update(loss_triplet.data[0], data1.size(0))\n",
    "                losses_VAE.update(loss_vae.data[0], data1.size(0))\n",
    "                accs.update(acc, data1.size(0))\n",
    "                emb_norms.update(loss_embedd.data[0]/3, data1.size(0))\n",
    "\n",
    "            print('\\nTest set: Average VAE loss: {:.4f}, Average Metric loss: {:.4f}, Metric Accuracy: {:.2f}%\\n'.format(\n",
    "            losses_VAE.avg, losses_metric.avg, 100. * accs.avg))\n",
    "            test_loss_metric.append(losses_metric.avg)\n",
    "            test_loss_VAE.append(losses_VAE.avg)\n",
    "            test_acc_metric.append(accs.avg)\n",
    "\n",
    "           # plot(data1, recon_batch1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Basic functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABHEAAAFaCAYAAACZu8ETAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3XnclfP2//HVMZxEVAqlNCkhhAwpESkhUzLPs595Fpln\njnwdQ+cgUyHzcJA58xQRhUIZSqWoRJn7/eFheX+We2+7u30P196v519rn8+nfV/2ta9hX+ez1qqz\nYMECAwAAAAAAQO32j5reAAAAAAAAAPw9HuIAAAAAAABkAA9xAAAAAAAAMoCHOAAAAAAAABnAQxwA\nAAAAAIAM4CEOAAAAAABABvAQBwAAAAAAIAN4iAMAAAAAAJABPMQBAAAAAADIgMUXZnKdOnUWVNWG\nIL8FCxbUKcb7sA9r1MwFCxY0KcYbsR9rDsdiSeBYLAEciyWBY7EEcCyWBI7FEsCxWBIKOhZZiQNU\nn89qegMAmBnHIlBbcCwCtQPHIlA7FHQs8hAHAAAAAAAgA3iIAwAAAAAAkAE8xAEAAAAAAMgAHuIA\nAAAAAABkAA9xAAAAAAAAMmChWoxn2dJLL528vvzyyz0+/PDDk7Gdd97Z4wcffLBqNwwAAAAAAKAA\nrMQBAAAAAADIAB7iAAAAAAAAZEBJp1M1bNjQ44EDByZjBx98sMezZs1Kxvr16+cx6VRV7/jjj09e\nX3bZZR7fcccdHu+3337Vtk0AAAAAsmnq1KkeP/3008nYPvvsU92bAxQVK3EAAAAAAAAygIc4AAAA\nAAAAGcBDHAAAAAAAgAwo6Zo4t9xyi8fbbbddMjZq1CiPtT6OWZpDiarRp08fj88777xkbPHF//xa\n/vzzz9W2TQBy23XXXT1u1apVznlbbLFF8rpXr14e16lTx+MFCxbkfI9bb701eX3AAQcUupkAUGss\nscQSyesLL7zQY70P2mSTTZJ5c+fOrdoNA0rUYYcd5vGKK67o8Z577pnMe+655zweMmRIlW8XUGys\nxAEAAAAAAMgAHuIAAAAAAABkQMmlUzVr1szjTp065Zx37rnnejx27Ngq3Sb81U033eTxMsssk4y9\n9dZbHg8YMKDatgkoR2ussYbHugzZzGyPPfbweNlll/U4pgjko2lT+VKoVL7Wn6RWAcgKPb+amZ1w\nwgkVzltllVWS1+PGjauybcLC6dy5c/J6/PjxHhcj7U1/t0yZMiUZ23nnnT1+4IEHFvlvlYN//KPi\n9Qmazm2WpnqTToUsYiUOAAAAAABABvAQBwAAAAAAIANKLp1Kl9o3b97cY+1GZWY2YsSIatsm/K5H\njx4eN2nSJOe8m2++2eMZM2ZU6TZh0ay66qoeP/nkk8lYy5YtK/w3canrI4884vHjjz+ejF1//fUe\n06ms8uJycO3+1KhRI49XWGGFatumfOKy53333ddj0qlQ7jbaaCOPNd3CzGyXXXbxOKYqH3jggR4/\n+uijVbR1qAw9x5mZnXrqqTW0JTAz6927t8d6j2Jmdt1113l83HHHeVxounA0cOBAj3/77bdkrG/f\nvh6TTlWYN954w+PvvvvO43g+BLKOlTgAAAAAAAAZwEMcAAAAAACADOAhDgAAAAAAQAZkvibOJpts\nkrw+//zzPf7hhx881pbiqBk77rijx4sttljOecOGDauOzUElaR0crS0VW6Tmyg+POd99+vSpMDZL\n61qdddZZHlMf5+/puVBrYZiZrbTSSkX9W5999pnH06ZNyznv1Vdf9VhrCaBqdezY0eOddtop57wt\nttjC42effTbn2MEHH+zxJ598UqltqlevnsdPP/10MtalSxePzzzzzGTsggsuqNTfywKtGxeP2X79\n+nm8xBJLeLz44ultXL66HN27d/eYmjjVY8KECclrPQfq9zy2IkfN2n777T2OdfyOPvpoj0888USP\nf/nll4LfX2vRxbpWKn5/8Pc+/PBDj+fNm+dxrInTunVrj3V/fPPNN1W4dagq+ttkjz32KPjf7bXX\nXh4vv/zyHv/73/9O5v3vf//z+J133qnMJhYdK3EAAAAAAAAygIc4AAAAAAAAGZDJdCpdMnXbbbcl\nY7qUWJcj01K8+i299NLJ66233rrCeXfeeWfyeu7cuVW2TVh0F110kce6HDWaOnWqx2PHjvU4to/W\nY1aXMpqZnXzyyR7rkmbar/691Vdf3eNC06e0HaeZ2VdffeXxlClTPL7qqquSee+++67H+dJrDj30\n0IK2AwuvU6dOHp922mnJmC7Xz5fKqjT1Jho6dKjHMaU5n/r163v82GOPeawts83SlMvKtu3NCk2j\n0WvhCiuskPPfjBs3LufYmmuumXNsv/328/hf//qXxzNmzPjb7UTlzJ8/P3n9/fffVzgvphKj+h1/\n/PEea8v3eM9y++23e/zrr79W6m9pSmq+Y3348OGVev9ypsdYTN9XnTt39rhZs2Yek05V89Zee22P\n9dx47LHHJvP0N4P+RliYNONcYhkWLekQ75W32morj998882F/luVxUocAAAAAACADOAhDgAAAAAA\nQAbwEAcAAAAAACADMlkT57DDDvM41uS4//77PaaFZs3SFrJmZu3bt69w3i233JK8zpfDiuqndRTM\n0la3mmeqNXDMzHbYYQePR48eXdDf6tWrV/Ja62ZoK89YCytfjYhydf3113scW2vOnj3b45EjR3oc\n25nqWGXpPr3ssssK+jcxfznWzcLvjjjiCI+vuOIKj//5z39W6d/dYIMNPI61PF5//XWPjznmmGRM\nj+F4fVBaZ2LUqFGV3s7aKP5333XXXR7nq40xYMAAj2NNKqWt4TfeeONkTN+/ZcuWHlMTp/poy+NY\na0VpTcFcdXRQXFqHQz//yZMnJ/OOO+44jwuttbHEEkskr7fbbrsK502bNi15TX2WqvP44497/NFH\nH9XgliB66KGHPNZrVaGqopae1hNcbrnlkrGuXbt6TE0cAAAAAAAAJHiIAwAAAAAAkAGZSafStpma\n3jFr1qxknrYEo1V1zYrLzZBNTZo0KWjeUUcdlbwuNIVKPf/888nrDz74wGNtmX3mmWcm83bfffeF\n/lul7sknn6wwrm733HOPxzGtK5eZM2cmr/fZZ5+iblOp6Natm8dVnUKltJXn//3f/yVj2pa+0Pbj\nP//8c/J611139bgmv7tV4YYbbkhe672Nfg7xc7366qs9/uGHH3K+v6bV6bEX9e7d2+PqXP5d7jSl\nVNvSLrXUUsk8TZXUdvAonptuuil5vffee3us6RixNMPXX3+90H+re/fuyWtNv1DXXntt8vrbb79d\n6L+Fwuj1qUWLFh5//PHHNbE5EIMGDfI4X/pwLg888EDyOqYpKr1nueaaazwePHhwMm+XXXbJ+R5b\nbrmlx5XZ3spiJQ4AAAAAAEAG8BAHAAAAAAAgAzKTTqXpE40bN/Z4zJgxybyxY8dW2zYhv/79++cc\nmz59usel1n2k1GhHGbM0lUKXDWo1+crabLPNktdrrLFGhfO0Ow5qnlbtj2l1sStHLtqFY8cddyzO\nhpUY7Z5iZrbTTjsV9O/uvvtujy+88EKPY1pAu3btPNauSWa5OyetuuqqeV/n8sorr3isS6fNzB5+\n+OGC3iOLcp3TzMxuvfVWj0899dRKvb/eE8W0q7p163q89dZbe6zfCVSt1157zWNN21hrrbWSedoF\nknSq4tE0/2222SYZ03sbPRaPPvroRf676667bs4xPQ9rV0lUrc8//9xjTVuLJQQ6d+6c8z00jeaZ\nZ57x+O23307m5UvnwV8NGTLEY+1ONWXKlGTe8OHDK/z38d7mp59+KujvNmjQwGNNsfs7uu+rEytx\nAAAAAAAAMoCHOAAAAAAAABnAQxwAAAAAAIAMqLU1cWK7xS222KLCeZXN5dYaDtq62CxtYf7+++97\nfO+99ybzaGFeeT/++KPHsU08ahdttWlm9ttvv3lc1a1p49/+Q2z9h+pVr1695LW2as1XC0tpW0ez\ntE281o3An04++eTkda624l26dElejx492uNffvkl5/vXqVPH4/vvvz8ZO/zwwwvezj989913yev9\n99/f46efftrjcrqWHnnkkcnrkSNHeqz1FZo3b57Mmzx5ckHvr3VW5s+fn4xpTZypU6cW9H6oGVoH\nApW39NJLJ69fffVVj2PtkyeeeMLjc8891+N858xCbbjhhjnHtF7HjBkzFvlvoTAdO3b0uG/fvh7r\nvYiZWc+ePT3OdU9qZnbCCSd4PGDAgGTs0ksvrfR2lqN58+Z5fNJJJ1Xp32rYsKHHd955p8cbbbRR\nzn+jv2HNzD788MPib1gBWIkDAAAAAACQATzEAQAAAAAAyIBam061yy67JK+XX355j7U9dUxxyiW2\nRz3ggAM8vuiiiwp6D10qZ/bXlpBIadpNMTRt2jR5rW2IO3TokIzlWtqmS+XMzGbPnl2krUMxaFtV\n1C56jN1zzz3JWL62yUpTqIYNG5aMPf/884uwdeUhpv7mom10zXKnA/Tu3Tt5rSkEG2ywwUJu3e80\nNUrTp8zMHnzwwUq9ZynR1upmZldddZXHumx8q622SubdfPPNBb1/t27dPK5fv37OeaNGjSro/YAs\ni2lM8V5RaVvxzz77rKD3198m2223XTK20koredynT5+c71FTqRj40w033JBzLF8KVS5nnXVW8nrm\nzJkea/tsVL/u3bsnrzX1rVevXjn/3Q8//ODxQQcdlIxpKmZ1YiUOAAAAAABABvAQBwAAAAAAIANq\nbTpVq1atco4VmkKlzjzzzOS1doiInTFOPPFEj/fYYw+PN99882Sedgq5/PLLF3qbSp12rDEzu+SS\nSzxu1KiRx1oh3sxs7NixHq+44ooef/HFF8k87TBWqCuuuCJ5rcvodFk7/qRLjM3MTjnlFI+POeYY\nj5955plkXq7uJ7HznC4t3WabbSq9nSg+7digx0qh6VPRXXfd5XHstIS/d/fddyev11133QrnxbTR\ntdde22PtQBW7vBXaEUfTUGPXqjFjxng8YcKEgt6vnJ1zzjkeb7vtthX+72ZpaoamM2666abJPE0V\nX3zx9Bbv8ccf9/jaa6+t1PaieAYNGuTxLbfckozp/U28ZsauY8hN0wujmCYzdOhQj2+77bac/07P\noSrek+q8fCk5+tsidr3R3yfbb799MhZTM1E18nVPzJWyGo/ZM844w2PSqapf165dPb7yyiuTsVz3\nUdrV0yy9t4n3WDWFlTgAAAAAAAAZwEMcAAAAAACADOAhDgAAAAAAQAbU2po4+VoNF9r6T8WW5doq\nbN99903GHnroIY/vu+8+jzWnMb7WeWZmEydOXOhtLCfLLLOMx82aNUvGtCaO1l+J+cbawjzmOPbv\n39/jxo0be1yvXr1knuYiUxOnYlpHwSzdJ+utt57H7733XjLvp59+qvD94n7UFp2Foj1u8eix2KNH\nj2RM6zQ0aNCgoPeL+ePHHnusx/E8iYUT6zRorTc9tzVs2DCZN2LEiArn5auB88033ySvtRadnitp\nj7to5s2b5/Fuu+3mcawxduGFF1YYL4ynnnqqwr+Lmhdrpuh9S7wfHjZsWLVsU1ZpLZrVV1+94H9X\naJ3FQmvdFErfL9ax0vqR8fpMTZyqEWuhaO3HSI9TrVt26aWXJvO0vmfPnj2TsaeffrpS24nctJ6t\nmdl///tfj/WeN9J9Ed/j66+/LtLWFQ8rcQAAAAAAADKAhzgAAAAAAAAZUGvTqYpBl7PpkjeztFWf\nLjWPZs2a5XHz5s2TsWWXXdbjTTbZJBkjnapwe+65Z/J6/PjxHu+33345/92TTz7p8UknnZSM6Wtd\n4qipWmZmG264occdOnRIxkgT+N3zzz+fvNb0qq233trjmMKRyz/+kT471rS4ym4T8ostUZdcckmP\nNX1U0wsXhqan6nfCzOy1116r1Hvir6ZNm5a81iXCxx9/fM5/17Fjx4LeX1OoYppxvuskikOvT7r8\n3sxsyy239FjvX+LxtdZaa3l82mmnFXsTgVpP06Lq1q2bc16uVuELo9D3iPP0t8W4ceM8julZt99+\nu8cPP/xwZTYRBdDr28UXX5yMVSZtLaZMaUmPk08+ORkjnarymjRp4vG1117rce/evZN5mkIVU/4f\neeQRjzX9vzamT0WsxAEAAAAAAMgAHuIAAAAAAABkAA9xAAAAAAAAMqDW1sQZNGhQ8lpb3eZrD6a0\nNsP111+fjL300ksexxbHWqNjr7328njXXXfNuU20fPyr2KJWaxSdeuqpHse6N3369PE4X/vp//u/\n/8s5tsIKK3i8//77exzbJGvudGy3S02cimlu7+GHH+7xPvvsU9C/f+GFF5LX06dP9/j888/P+e9m\nzJhR6CYiqF+/fvJa8/ErI+YU67k2Xw0crTvVv3//ZOzOO+/0+OOPP16k7SsXkydPXqR//8477ySv\nBw4c6DE1cGoXbTke248rve5SE6d201oM8X5J730Wpk02zH755ReP995772Rsq6228rhFixbJ2Cqr\nrOLxpEmTPH722WcL+rtLL7108vrNN9/0ONa6GTJkiMennHJKQe+PqqO/T/T34cJYZ511PN5oo41y\nzvvggw8q9f746+/FE0880eN8tf/0nvWwww5LxoYPH16krat+rMQBAAAAAADIAB7iAAAAAAAAZECt\nTaeKy9nmzJnjsS49jMvBH3zwQY91+aKmfZilrXS1BbVZ2i5Xx7766qtk3oEHHphz+2H266+/Jq81\nRe7QQw/1OLam1lSofDRFLrYO7969u8f50u809WPkyJEF/d1yN3/+fI+vvPLKCuOFoe1y87ntttsq\n9f7lao899vD4P//5T1HfOx5TTzzxREH/bvHF/7zkxNavBxxwgMeaUmlmNn78+IXdxLLQrFmzhf43\nmkq32267JWOksQHVR4/Fa665Jhk7++yzPd53332TsRtuuMHjTz/9tGo2rkTMmzcvef3QQw9V2d+K\n6eAxhUq98cYbVbYdWHh6f6klA8zM3n33XY9/+umnZExTqLQFfEzVU2PGjKn0dpYj/YzPPffcZExT\nINWjjz6avL7gggs8fv3114u4dTWLlTgAAAAAAAAZwEMcAAAAAACADKi16VQTJ05MXl944YUeX375\n5R7feuutyTxNj2ndunXO919jjTU81m5U+RSa9oGKaTpap06dPB46dGgyb9NNN/W4Tp06Od9Pl9Hl\nWlIXxarwBx98sMdxmSSqhx6L+Ht9+/b1WJeZahc2M7MVV1zR43r16hV1G+JxWWjHwHy0O9whhxyS\njJ100kmL/P6lYKeddkpeH3fccQv9Hk8//bTHpE8BtcNjjz2WvNa0gZg2qed90qlqj2222SZ5rdfJ\n559/PhmL+xtVQ1OctIxDpL8hYqqblunQbqpmZttuu63HmkIVU+n0d6aW7EDFGjVq5LEeK02bNk3m\nffvttx4/9dRTHscOVLH7X6lgJQ4AAAAAAEAG8BAHAAAAAAAgA3iIAwAAAAAAkAG1tiZO9K9//cvj\nf/zjz2dPF198cTKvZ8+eBb1fvtZ/X375pcfa2nH06NEFvTf+3ueff+7xZpttlox169bNY62FscMO\nO1Tqb5133nkex+8Luak1b8SIER5rG8DozTffrI7NqXX22Wef5PWNN97osbbsri302DZLazY0btzY\n43y1kPbee+/kNTVxfjdw4MDk9WKLLVZDWwKgmOL1TWtoxPvVww8/3OOqbJmNv9egQQOPl1tuuWRM\n99uoUaOSsdj6HFXj6quv9rhPnz7JWJMmTTyuW7duzvfYcccdC/pb+X5XDh482OM777yzoPcrJ717\n905e62+BWAdHHXHEER6X4+fKShwAAAAAAIAM4CEOAAAAAABABtS+tfgFuOKKKzyOLcZ32WUXj7fe\nemuPYxvG+fPnexxb/b3zzjsez549e5G2FQvvpZde8lhbxt93333JvEJbvo8dO9Zj0qdqnxNPPNHj\nfC3l43LkcjFgwIDkdbFTqLT14rRp03LOGzJkiMcfffRRznmxbfX48eM91mWxw4cPT+ZpGmVclq4p\nZAcffHDOv12KGjZs6HHHjh1rcEtQSurXr+/x3Llza3BLgOxq27atx23atMk5L9+9DarOuHHjPG7Z\nsmUy1qFDB49PPvlkj/fff/9knpbwyJcypft46NChyZimU+F3+hv93HPPTcbWX3/9Cv/N66+/nrzW\ncgzliJU4AAAAAAAAGcBDHAAAAAAAgAzgIQ4AAAAAAEAGZLImzq+//urx9OnTk7Frr722whjZ9Msv\nv3i85557JmP9+/f3uF+/fsmYtqubNGlSFW0dikFzjPPlG2srwVNPPbVKt6k2iTnA7du391hzsPVY\nMTObNWuWxz/++GMydvbZZ3v8/vvve/zGG28s2sb+jalTp3q82267JWOaHx2Va3t5M7OjjjrK48rW\nQxozZozHWtsI5eW4447z+Mgjj/T46KOPTuaVe52B2kDPy6uvvnoNbgnyyXfPUpl5qD4ffvihxwcd\ndJDHr7zySjJPa/kdcMAByVjr1q091pqCF110Uc6/hd9pvccNNtgg57wnn3zS40svvTQZK/e6tazE\nAQAAAAAAyAAe4gAAAAAAAGRAJtOpUJ6+//775PUtt9xSYYxs+fTTTwuat8suu3hcTulUcfmuphYt\ntdRSHmuqkpnZ7bffXrUbtohiO3OO4Yptt912i/wemk43atSoRX4/ZFOLFi0q/N8333zz5DXpVDWv\nZ8+eHj/99NM1uCXIp9DW4a+++moVbwmKJV/K8QUXXFCNW1JatMyFmVmXLl1yztVznpbOmDt3bvE3\nLMNYiQMAAAAAAJABPMQBAAAAAADIANKpANSo+++/3+Mzzjgj57znnnuuGram9qPrXnn56quvFvk9\nxo4d63G5d3ModW+99ZbHcV83aNCgwn/TqVOnKt0mLDztvLrWWmvV4JYgn3zdNe+++26PH3/88Wrb\nJqA2itcZ7bb58ssvJ2M77bSTx7GUBv7EShwAAAAAAIAM4CEOAAAAAABABvAQBwAAAAAAIAOoiQOg\nRr3//vseX3HFFcnYiSee6DE55ShHr7zyisfbbLNNQf/mtddeS16fdNJJRd0m1F4zZ870WM+fZmbX\nXHONx2+++abHxx57bNVvGFCCRo8e7fFiiy1Wg1sC1G6//vprzrG33347eU0dnMKwEgcAAAAAACAD\neIgDAAAAAACQAXViS7y8k+vUKXwyimrBggV1ivE+7MMa9daCBQs6F+ON2I81h2OxJGTyWPzhhx+S\n16NGjfJY0w3//e9/J/Pmzp1btRtWQzgWS0Imj0WkOBZLAsdiCcjKsfjEE094vOOOOyZj8+fPr8o/\nnQUFHYusxAEAAAAAAMgAHuIAAAAAAABkAA9xAAAAAAAAMoAW4wAAZEDdunVrehMAAAAWSe/evWt6\nEzKPlTgAAAAAAAAZwEMcAAAAAACADFjYdKqZZvZZVWwI8mpZxPdiH9Yc9mP2sQ9LA/sx+9iHpYH9\nmH3sw9LAfsw+9mFpKGg/1lmwoErbwAMAAAAAAKAISKcCAAAAAADIAB7iAAAAAAAAZAAPcQAAAAAA\nADKAhzgAAAAAAAAZwEMcAAAAAACADOAhDgAAAAAAQAbwEAcAAAAAACADeIgDAAAAAACQATzEAQAA\nAAAAyAAe4gAAAAAAAGQAD3EAAAAAAAAygIc4AAAAAAAAGcBDHAAAAAAAgAzgIQ4AAAAAAEAG8BAH\nAAAAAAAgA3iIAwAAAAAAkAE8xAEAAAAAAMgAHuIAAAAAAABkAA9xAAAAAAAAMmDxhZlcp06dBVW1\nIchvwYIFdYrxPuzDGjVzwYIFTYrxRuzHmsOxWBI4FksAx2JJ4FgsARyLJYFjsQRwLJaEgo5FVuIA\n1eezmt4AAGbGsQjUFhyLQO3AsQjUDgUdiwu1Eidr6tT582HkkksumYwtvfTSHv/yyy/J2Hfffefx\nb7/9VkVbBwAAAAAAUDhW4gAAAAAAAGQAD3EAAAAAAAAygIc4AAAAAAAAGVByNXH+8Y8/n0ttsMEG\nHv/nP/9J5jVt2tTjOXPmJGO77babx2PGjPF4wQIKdReL1iuqW7duMqb1i5ZaaimP586dm8ybN2+e\nx+wbAAAAoHzp74t//vOfFcZmZssss4zHM2bM8Pinn36qwq0DioeVOAAAAAAAABnAQxwAAAAAAIAM\nKLl0Kk2r0dbhkydPTub9/PPPHg8ePDgZGzduXIXvh+JZbLHFPNZ272ZmrVq18njjjTf2eOTIkcm8\niRMnevzDDz8kY+w3oGbosW2Wpkdq3LBhw2Tet99+6/Hs2bOTsd9++62YmwgANU7T/+M9C/cwQGEW\nXzz9Kau/Ifr37++xltgwM3vllVc8vuuuuzz+4osviryFQNVgJQ4AAAAAAEAG8BAHAAAAAAAgA3iI\nAwAAAAAAkAElVxNHW8j16tXL49jG+vnnn/f4gQceSMa0Xg6KR9v+rbjiih7vv//+ybz111/f4xdf\nfNHjWbNmJfO05pG+d0RuOVCYeBwttdRSHjdr1szjFi1aJPP69u3r8dprr52MtWnTxmOtfxVr56hY\nw+zAAw/0ePTo0Tn/HQDUNK11s+mmmyZjd955p8d6vj3mmGOSeffff7/Hv/76a7E3ERmh3xHuZf+k\n9w96j2Fmdu6553rco0cPj+P9TYcOHTx+9dVXPf7yyy+TeRx/qK1YiQMAAAAAAJABPMQBAAAAAADI\ngJJLp2revLnHXbp08fi7775L5g0ZMsRjbW2LqqPLH7fYYguPNe3NLN0fusRxxowZyTzS3mqe7lNN\nvYmv586d6/GPP/6YzGOJcPXT/bbccst53Lt372TewQcf7PGqq67qcaNGjZJ52jo80tQCjfOlQOo2\nmZkNGDDA43322cfjH374Ied7AEBN0HNbw4YNk7FlllnGY03z79mzZzLvoYce8ph0juqn10hNAzZL\nr3dz5szxWFP8zdJ7m3i909f6PWjatGkyT9/z66+/TsbmzZvn8W+//VbBf0Xp0s/2+++/T8b0t4HO\niync+nnqe3BPiqxgJQ4AAAAAAEAG8BAHAAAAAAAgAzKfTqXdqMzS5f+aWvXyyy8n86ZOnepxuS1D\nrCmaIrHa1Pv7AAAgAElEQVTrrrt63Lp162TeO++84/HEiRM9jktVC0V1/+Jq3Lixx6effrrH/fr1\nS+bVr1+/wn8/c+bM5LXu75tvvjkZ0y5yunQYf0+/9zHdqV27dh5ffPHFHm+++ebJPP13udKizNLj\nKqbL6ZJlXZYeU6Z0qXN8f+14FcdQeflS2qJinzv1b8d9uvjif96axPN+uaSWxM9E73WWX355j/U+\nxyxNJRg3blwyRvph9Vt22WWT1/r91X3cqlWrZJ6ee3/66aeq2bgyF89/en3q2rWrx5r+b2b2zTff\neDx06FCPY7pTvg6qmkK1xhpreLzeeusl87RTo5YXMCvveyL93RY/h/Hjx3vctm3bnO/xzDPPePzR\nRx9V+N5YePzmqj7cDQMAAAAAAGQAD3EAAAAAAAAygIc4AAAAAAAAGZD5mjgdO3ZMXm+55ZYea8u4\nYcOGJfNiy3FUPW35rjUuYl2jkSNHeqztGyubW0lO5qKJ9VQOOeQQj/fbbz+PYw0c/dy1DkBsoamv\nt9pqq2Tsrbfe8vjwww/3+JNPPsn5t8pVbJ+p+f177LFHMqa1w/QcusQSSyTzNDdca23Mnz8/mTdp\n0iSPn3jiiWTs7bff9ni11Vbz+MADD0zmxZoQubCvK6b1NbTegpnZSiut5HGzZs08/vbbb5N5sX6H\n0n0+bdo0j2MdCP3OxHOHfic7derk8QorrJDM0zoQeg4wM5s7d66Zlc73QOsH6Offq1evZN5ee+3l\n8VprreVx/Iz13ubhhx9OxgYNGuTxV1995XGpfJa10YwZM5LXun/q1avncTx/6zEcWyizv4pDa2+Z\npTXhLrjgAo/jPcvo0aM9fvTRRz2O9f50P8UaV1rXauONN/Z4lVVWSeZpXchY04raLb/745rwhxdf\nfNFjrRmmn7lZWjunXGqtFYt+n+P5SMcq+7nqeyy11FIex+udvtb926hRo2TeBhts4PGmm26ajOl9\niR5jWq/TzOzjjz/2eNSoUTnHKlu/tTJYiQMAAAAAAJABPMQBAAAAAADIgEymU+nyqdNOOy0Z0yVU\n559/vsdxWRTLEKteXD664447etykSROPdVm3Wbo8ldaaNS8uJd5zzz091hSquKRV24M/+eSTHsdl\n47vvvrvHMT2yc+fOHl911VUeH3TQQck8Te8oJ5qKscwyyyRjq6++usfdunVLxjR9RdOkYnqNpkkN\nHz7c42effTaZp8vIY7tP3UZdGr7TTjsl81q2bOlxXI76yiuveBxbmJczTUXdZJNNPO7fv38yT1Nz\n9N/E5fn6XYjLoGfPnu3xa6+9VmFslqaLxPTInj17eqwt5uN3RlvpajtfM7N3333Xsiye/3QJ+Omn\nn+5x9+7dk3kNGzb0WI/TqVOnJvNWXnllj/Waa2bWunVrjwcMGOBxTE9F8cRzql4ndd83btw4mafp\npXrsmVXvcv1SpvehZmannnqqxx06dPA4/l7QNP8pU6Z4HPeLppnE1K11113X46233jrnvHvuucdj\nrn0Vi/tHrxmaihPTjDVNkd+EC0e/2/GaVmgKld4bxlR+TYPTsg2ahm1mtuaaa3qs6ci6383SlCn9\nu5H+d/Xp0ycZ0+Pv/fffT8aOPPJIj7WEQFWn6bESBwAAAAAAIAN4iAMAAAAAAJABPMQBAAAAAADI\ngEzWxNFc8S222CIZ09zuxx57zONYW4UWjVUvtoJbZ511PNacRG3NZmb25ZdfelzZ/aTvz75eePr5\nxdolmquq7YW1BbhZ2mpaa23E3NexY8fmfI+uXbt6rLmvO+ywQzLvpptuqvBvlTrdTzHPV3OA9Zgy\nM3vjjTc81poad999dzJPW6lq/ZR4TOXbDq1Tduihh3q8xhpr5HyP2ML8/vvvz/m3y4nuU7O0PtUJ\nJ5zgcWxTq8ec1m2INRw0fzvWZlAtWrTwuH379smYvtZzRdx+3Y+xboi2A9W6WGbZrImj3+3Y5vbM\nM8/0WFscx1z6++67z+Mbb7zR4/jZbbbZZh7H86nWxrrmmms87tevXzIv1ijCwtHvdqwVp6/1nBpb\n4uoxptdIM2riLAqt1bjNNtskY+utt57Hev7T+xwzs5tvvtlj3Z+xrooe91o/0Mxs33339VjrfGg9\nDTOzL774wmP2e8XiPYHeK2r79hVXXDGZp/v1qaeeWuTtyNd2O+v3LfnqyFT23+nnpTXyzNLjY+ed\nd/ZYa4WZpTX+8t2H6ucf94Vea/UYjvP0N+1KK62UjOm1W1vXa43Ait5zUbESBwAAAAAAIAN4iAMA\nAAAAAJABmUmn0qWN2lY8LvkePHiwx9r2NutL2bKoXr16yWtdLqfLQjXtxqzwNor5ls7p63zL41Ax\nbRmoywTN0s9TW4drbPbX9sV/iGkC2k5YW3eamR122GEeaxtObVVsZvbAAw94HFvWl4uYMvrOO+94\nHFsIa7qSfubxPSrTdjO28dQ0nwMOOMDjmFanf+vDDz9Mxl588UWPy+0Y1iXHukzczOz444/3WNtH\nx5afer7V9JuYNqNt6uO1VbdDz+0bbbRRMk9bYcd9rOdlPc9Pnjw5maffXW2xa5bN/a/7o2/fvsmY\npoTrvGuvvTaZd/vtt3scl2grPQ/vsssuyZimi2iaWlwaPnHixJzvj4Wj96FmaSqrthWPLXG1VXxM\nTc91bcXf03PXXnvtlYzp56z3Kc8++2wy76233vI43zVS3y/+LU1t1JSQadOmJfO0XTYqptcmszSd\nWI8j/ZzN0tQcvVbF3yB63YrXRT1na4vrmEap7cyzKF539TPP9/sr3/Vax+I5bdSoUR5rWYUmTZok\n8/RznjBhgsdTpkxJ5uk9sP7miO+x2267edy/f/9knqZExuNevz+5fuua/bVUwKJiJQ4AAAAAAEAG\n8BAHAAAAAAAgAzKTTtWxY0ePdRlwXHY/fPhwjwtNBYhL8VShS8HwV3Hpoi4X1n0Tl7bFdJtc8lX+\n1zFNGYjpIqiYLg1s27ZtMqZLhDU1o9A0uEi7ScW0Ck3N0GXQsbORpgPE5euVSQnKCv1vi6kxumwz\nLsle1HNXXD6ry4gvvvjiZOyQQw7xOF/HI/0uaZqQmdns2bMrtZ2lQNPTjj322GRMj009ZuM5VJfo\naxcO7T5mll5bN9xww2RMlzHrNTOee/PtYz3/asrUkCFDknl6TYjdl7JI942mNJmlKTXjxo3z+I47\n7kjmxeX5uejnFZeo6zVZz9d0o6o6s2bNSl5rWpx27GzXrl0yL9/xTPfNwsVrlZ7jVl111WRMUx80\nLfv6669P5mkKsn7+8W81bdrU4wMPPDAZ09RV7ZIUywuUU7fNyoqfu3ZFjL9DlF67unTp4rF2FzJL\n03nidybXfdYLL7yQzNMuvKVwT6ppZPG/J18nqFzzYrqZpjBOmjTJ4/j5awqVdlqN75fvd6Wea/V7\nENPBVbxm6v2MnvMr+7uoUKzEAQAAAAAAyAAe4gAAAAAAAGQAD3EAAAAAAAAyoNbWxIl1ai666CKP\ntbbKgw8+mMzLlX8Wcya11obmO5qZrbbaah5rW7KYk671eL788stkrBRyHhfV0ksvnbzWWgmaMxhr\nmCjdb/E7ofnGe++9dzKmdVvuvfdejz/66KNkHvnGFdM84mbNmuUci+2kK0P31XnnnZeMbbvtthXO\ni8d5bM+K4tdK0OOvffv2yZien7fffvtkLLa7/kOsTzVgwACPtQ1vuYnXKj2PrrvuuslYrpzt2Mby\nueee8/jf//63x7FmytixYz2OdZR69uzp8fLLL+9xrDmg177YXvPtt9/2+PLLL/c41kXLejvWSD+T\neG+gx5XeYxRaGy7Sc7LW/zBLr8Fa84M2xsWl5954ntN7EL330fo4ZmabbbaZx7feemsyRg2jwsV7\nlIMOOshjrbNnltZeGzFihMdak8MsPWY1jq3gd9ppJ4+1TotZ+h354osvPNYaH2bpvU08X+c6r8Rr\nSKnXTYrnys8//9xjvVeM10ttP37ZZZd5HO95teZfPH/rsag1zfT3iZnZFVdcUeG/ySr9TlX2WqXi\n56r3MPpbO9YrylV/J993Ph4fq6++usdHHHGEx1q3yiw9/mLNI90u/c5V9bMAVuIAAAAAAABkAA9x\nAAAAAAAAMqDWplNpupNZmvKkS6FefPHFZJ6O6dLhTp06JfO0ZaCmT5mlyyN1+VRcAjdq1CiPjzrq\nqGQstkouRzH9SffN9OnTPZ4xY0YyL9cyuLi07ZxzzvG4X79+yZgu79t88809/u9//5vM03a7sRVo\nqS9BzUeXncaUiFwpbnGJYi4xvebss8/2OLbh1FQN3aefffZZMm/ixIkel/N+KzY9h/bo0cPjG264\nIZmnS8VzpU+ZpemLgwcPTsZuvvlmj8s5HTV+f/UYy5e+qP9Oz69maTqV7tPYDrxFixYex+uitu/U\nlJDY/l2v3do618xs+PDhHr/88ssex+XYpXYM6/f+zTffTMZ23XVXj7V96tZbb53M09bDulw7Hm/7\n77+/x61bt07G9Bz96KOPehxTfrBo9Psbz2V6T6nHTkwJbtOmjceazmFG+tvf0e9527ZtkzH9LRDP\nO5repiUSYltjTXHVc7K2JzZLfxfE8gJ6DGur63gPpGmP8feO3gfpvJhuXox0l9osXi80LVhTcxs0\naJDM098UjRo18jimxeWj519Nd27Xrl0y78knn/T49ddfL/j9a6v4u6AqFeN+UO91ttxyy2RM7z0b\nN27scfxv1PIqw4YNS8b0d2x13r+yEgcAAAAAACADeIgDAAAAAACQAbU2nWqTTTZJXutSxNh5I9c8\nXc6mlcHN0qXiMQ3k22+/rTCOS8+1m4B2EjBLl42X+lLGXOKSRE0L0OVmcZ4uT9R4ww03TOb17dvX\nY12OapYug+vQoYPHxx13XDJPvyPXXXddMlbOS5Z1CaqmUZiZLbfcch7rkm9dhmhmNmfOHI9XWWUV\nj4cOHZrM0yWo8RjTZbK6TYMGDUrm6b4qtVSMqqbHZVzSr6kZmr6o3YnM8qfSafqApnDo+5nRKS4X\n/VxiZwY95vRcqdcts/Qcq/turbXWSubtsMMOHq+00krJmP5tvb7pEuP4t2LnQU1RqM7l2DVNr3d6\nDJilx9iaa67p8ZVXXpnM0xQBTeWOKQK77767x7Ebi37mMSUS1UOPU72Wxn3VsmVLj2O3HE2j4Xr3\nV3ofsdFGGyVjK6ywgscxPVXPp4cccojHsVyCXifzpQ/ruTBeI/Vv677W49csPf/rPa9ZmgbyyCOP\neDxt2rRknv7tcvi+aDmLQveV3gfFz0hfx7IaOqbp/5qeZWZ27rnneqxdV83K9zdiVYq/K7WD6jHH\nHJOM6flCUxHj/dbFF1/ssXYiM0ufS1TnMcZKHAAAAAAAgAzgIQ4AAAAAAEAG8BAHAAAAAAAgA2pt\nTZxYY0ZzOjWvPran1rzGpk2behzbR2t++QMPPJCMaTtWzac89thjk3naqlDzbM3S3MtyzXeMLRWV\nfj4rrrhiMvbdd995vPLKK3t80EEHJfM0jzHuX21tq3Gs87DLLrt4/OqrryZj+j0ot32o+Z333Xdf\nMnbSSSd5vP7663usx5RZmvNdr149j2Prea0XET9nHRsxYoTH2q6xon+HVMzH19ztVq1aeXzkkUcm\n8/baay+PtX5DlKt2kVl6XF1yySUexzaoqJjWt4m13bQeiraTjvn422+/vcfaTjrWEtOaELGezaRJ\nkzwePXq0x7H+gr5/vmO9XMXPdbfddvNY2wv37t07mac1U9Zbbz2PtS28WXqujZ/3hx9+6PGnn366\nEFuNyornXn390UcfeRyvYXr9jO3mtSYStcR+p5+rHit6PJiltTJijRQ9n+p75Kulote+uA91LJ4L\n9bVue6wVpvevevyapbWR9J4tHvflUAdH6W8I/Tzjsaifk35+sR6m1nF89tlnk7E99tjD41133dVj\nbV9ultY7W3bZZZOx+PsFlaPHqd67mqV1rWL9Mb1nuffeez2Oden0Hki/Y2bpsU9NHAAAAAAAACR4\niAMAAAAAAJABtTadKi5L1CVxurwwLmnSZUzaZk7b75mZvf/++x6/9957yZgusdN0ntjaXFvnxrQu\nlo3/lX4muty/R48eyTxdEte9e3ePV1999WSeLul/6qmnkjHdN82bN/e4V69eObcpLrvV5a7llq6j\nx9FNN92UjB1xxBEeaxphXD6aq+10XGo4e/Zsj/MtR37iiSc8jsci8i/X1tRSszQlrkuXLh5rK3iz\nv+7TP8RUKF3WrUv9zcxef/31CrcptjPXYzGmCJTz+VSPCb1umZl9/PHHHuu+i2lSXbt29Vj3gaYm\nm6XtwvV4MzN78803PdYUqnxpceV23qyMr776yuPLLrvM46uuuiqZp63hV111VY/322+/ZJ5ex+Lx\ne/3113tcTi3ea1K83uk9q7aI1pbWZmYNGzb0eMcdd0zG9Luh98P4nZ539NpkZvbBBx943L59+2RM\nr0n50iN0TO/9Y1qMnpM1hdksTavU1P2HH344maetjGN7a/3b+dKpyo2mIuZLhdPUby0b8J///CeZ\np6UC4ndBz99rrLGGxxtssEEyT1P1Nt9882QslvRA4XT/6ueqqftmafkN/c1hZnbNNdd4rClUc+fO\nTeblOyfUVMoiK3EAAAAAAAAygIc4AAAAAAAAGcBDHAAAAAAAgAyotTVxnnnmmeT1YYcd5rG2B4vt\nqTXHUeuixFoCWgsgthvTXGRtH7fOOusk87T2Q2x5TL75X2sNaTvNtm3beqxtvs3S1uS6f99+++1k\nntY5mjhxYjKWq6V1zHHUXNfYYpx9+LspU6Ykr/fff3+Pb7nlFo8159cszR/V1nya/22W1s7Ze++9\nc45pzne5tcxU+plonq/WzDAz69Spk8ex3WKfPn081roZMX88V12o6dOnJ/Nee+01j2PrYq0F0L9/\nf49jG8+33nrL488//zwZ09fl9j3Q/0ZtN26WXuP0fBWPRW2rq2LdqjvuuMPjeD7Ua6a25ETx5Gp5\na2Y2depUj/U+57bbbkvm6fdFa+eYpfUbctUsQ3Hlq6ei9Vri/dImm2zicZMmTZKxVq1aeUxNnN/p\n56w11Z5//vlkntaYiTXgGjdu7LEeb7EWjdYE02NWr29mZmeeeWaF22Rmdu2113o8ePBgj+M9aqHX\nuHK4FhZKz3Nasy3WCNPr52OPPeax/lYxy1/bTa+LDz74oMdrrbVWMk9/Z6600ko53w+/03tPjWPt\n0u23397jCy64wOPYxl3vS48//vhkTGuq6r1NFmpLsRIHAAAAAAAgA3iIAwAAAAAAkAG1Np0qLi3V\ntoC6lPTII49M5g0dOtRjXb4Wl9Fpu+rOnTsnY9oarkWLFh7H5f/ahi62GMdfl6BqC83rrrvO45Yt\nWybztthiC48//PBDj3W5o1naXrdjx47JmL6nLkvW5ZNmaWu/2B6S5akV09RBTdmJaYnaSlWX/8dl\n/N26dfP40EMPTcZ0OWMWljZWBU2ZMkvPZXou1DbSZmZ9+/b1OKaCarqNplDF73yuzz+mXW288cYe\na5tNs/S8qS1dY0v0L774wuOYwnfjjTd6/OKLL3qcr711KYqf2ejRoz1u06aNxzGdSlvn6n7Uc6iZ\n2ZgxYzyO+yCmA6B66XlTjz9NVTUze/TRRz0+9thjk7FevXp5rOnIen5G9dHz13HHHZeMaZpcTL9o\n1qyZx5pmXq7XyEg/hzlz5iRj+vqTTz5Z5L+l6R1bbbVVMqYpHbGt8QsvvOCxpsly37no9LfHO++8\n43GPHj2SeXqd1LE333wzmafHabx/1XTxdu3aeRzvh/XaHfexvmcp7H/979H/7vjZ6XVMP0eztKxG\n9+7dPT7mmGOSeauttlqF7xFLbJx00kkejxw5MhnT/Zu1z5+VOAAAAAAAABnAQxwAAAAAAIAMqLXp\nVFr93czs3nvv9fi0007zeOedd07madcVXSIcl//rMrqYaqWVyHWJf1ya/Morr3ictSVYNWH8+PEe\na9rM2WefnczTVLd8HTT69evn8b777puM6RJXTXW78847k3m6pDVfBXpUTLsAVJamaeTb33qMldPx\n1rRp0+S1piudcMIJHnfo0CGZ16hRI49jRf+YlpOLfs56fMT303Sd+vXrJ2MrrLCCx9pBK6aJaVdA\nTbsyS8/DmkJUDulUekwst9xyyZgu0deuJjGtQvejdj2KXSD1c6Y7X+2i+1DTBeJ1S48JTbsxS49b\nPRZJp6oZuk/1/sgsTSXXtH4zsy5dung8YsQIj0mnqn6awqGlGMzSlJp4zY2dBlE8ek95ySWXeKzp\n/2bpPZJ2ItZSAGbp74ZY/kHTsLQ7md4TxffUrnSlKFeaVPwdXrduXY+bN2+ejO22224e77777h5r\nBzmz9D5Fu4qde+65ybzXX3/d49hdM8u/J1iJAwAAAAAAkAE8xAEAAAAAAMgAHuIAAAAAAABkQK2t\niRNz1K688kqPNR/8zDPPTOatuOKKHuer+5CvBeH//vc/jzWvLrYsw8LRfaqtbU899dRk3oEHHuix\n1i7abrvtknnbbLONx5rfb5bmSb711lseDx48OJkXW46j+mndhnjc6+vp06dX2zbVNK2DsuOOOyZj\nu+yyi8frrbeexzEHO1ebx3xiHRTN4541a5bH+epHff7558lrzT9eddVVPY61yDR3OtYL0NpLWnNH\nt8ks27nNuei+i/tHa5xoi/H4XVDaOlzbtZultVFK8bPMMr1n0ZoPsQ6K1kbS2CytO6WtWbUWkhn7\nvjbQWmKxloTWRdMx6lhVPz3valtks/Q4inVW9NrF8VZcek58+eWXPb7pppuSeUcddZTH+hsitrHW\n3ySxdbjex2iNl7hP9b5owoQJyVgp7/8mTZp4HOvZ6L3NZpttloxpfSE9F8Zz3Lhx4zw+5ZRTPNbW\n8mbpc4Oqrh1WnS3jWYkDAAAAAACQATzEAQAAAAAAyIBam04V6fInTYkZPnx4Mq979+4eb7zxxh7r\nMmIzswcffNDjt99+OxmbOXOmx7Sdrnpffvll8vryyy/3WFvqaktxM7PevXvnfE9tvXvWWWd5/PXX\nX1d6O1EcsY1427ZtPY5pP9ouV5dDljr9HGJb6bXXXttjTZvJlzIVl3Tq+VSPibvvvjuZd++993qs\nqRmxjbjup5jipC2t9d/FlpLa/jOmzo0dO9ZjTa0q5WXIf9D9qtc0M7ODDz7YY219Gr8Leuw89thj\nHn/66afJPFoU1176Xc+XNqPHzuTJk5OxVVZZxWNNY43fF+57qp+2OzYza9eunccx9VTP++yrmtW0\naVOP4/VIj9N4LMZUR1QNTee++OKLkzH9rbfzzjt7rMeeWVrWYfHF05/N+lr3v/4GMTM744wzPC71\nFuN6j6/3FJpOb2bWvn17j/v27ZuMaXqblr0YM2ZMMm/gwIEejxo1yuOabCNenX+LlTgAAAAAAAAZ\nwEMcAAAAAACADOAhDgAAAAAAQAZkpiZOLrH+wkMPPVRhjOzQ+hpa/2LYsGHJPM1F7dWrVzKmrQS1\nxXg51NCo7WKLxu23397jWC9H95d+F0qd5hHr99cszaXXttyxDa2+x4wZM5IxPTdqDapYn6rQlrX5\nWirGffqH8ePHJ6+fffZZj/PV6Ci3GhCaj7/XXnslYx07dvRY62TEz1xbto8cOdLjOXPmFG07UX30\nGIvHwzfffOOx1pIyM2vVqpXHWp8gnpPL7RirKXqe0/oQZmk73ng8f/LJJx6zr6qf7rfVVlvN47gv\nfv75Z48fffTRZExrxaF66HXQzGzQoEEe33bbbR5r3UEzs2233dbjDh06JGOrr766x1pj5/zzz0/m\nPfHEEx4Xel+VVbmuT/r5mJn16NHD41hn8bvvvvP43Xff9fiSSy5J5o0ePdpjrYNTLr/1WIkDAAAA\nAACQATzEAQAAAAAAyIDMp1OhfMSWjDfccIPH2jbXLG2zynLj2kVb25qZtW7d2uN8KUHaZrDU6VLQ\n+N/91FNPVTjv+++/T+Zp2syLL76YjGkaajHaSudbupprrNSXFBfLCius4HGbNm2Ssbp163qsKRfx\nnDdu3DiPNT2PfZB98fjSJeVLLrlkMqYpd5oGEM/JkyZNyvn+qBrz5s1LXk+ZMsXj2NZYWyWzf6qf\nnmv1HBrvUfVYnDBhQjLGfqt5ug805fyZZ55J5um9VL169ZIxTQPS9OR431aM+6ys0OND43j/oimj\n06ZNS8Y0/UxT3WIavn7O5XhMsRIHAAAAAAAgA3iIAwAAAAAAkAGkUyGzdBmddmtA7aNLKpdffvlk\nTJf8xyWn+rpJkyYel1OnKl3KW9FrlLZ8aVLaJUXjmCalXYpiR0dkW1xCrufMmKKjS/+7du1aYWxm\n9sUXX3isHXZQXLqvXn/99WRMu1XFbn2apoPqp8ecph5++OGHyTzthvTRRx8lY6T5Z4cep9o1qaLX\nSI+PZZZZJuc8TY2Kx8PQoUM9/vzzzz3W7sXxb5UjVuIAAAAAAABkAA9xAAAAAAAAMoCHOAAAAAAA\nABlATRwAVU7zVmfOnJmMaX5/zPWfPHmyx9TyQDmI7YQbNGjg8XLLLZeM6XGlcayJo23FqadR2ubP\nn++x1kIyS+sJ6Pdlww03TOZpi93Y+rWcWuXWpHgMo/bQY+Cpp57yONbEUZ9++mnyutxreaB0aR2/\nZZdd1uOOHTvm/DcPPPBA8vrLL7/0OLZrx59YiQMAAAAAAJABPMQBAAAAAADIANKpAFSrzz77LHnd\nrFmznHNZuo9yE9MoxowZ4/Edd9yRjG222WYeT58+3eNhw4Yl85577jmPaW1b2rQl+IgRI5KxevXq\nedypUyePBw0alMzTpewActP0xQkTJtTglgC1Q926dT1eYoklPNaW4mZmiy22mMeTJk1KxvS4Qm6s\nxAEAAAAAAMgAHuIAAAAAAABkAA9xAAAAAAAAMqDOwrS5q1OnDj3xasiCBQvq/P2sv8c+rFFvLViw\noNzUY7wAACAASURBVHMx3oj9WHM4FksCx2IJ4FgsCRyLJYBjsSRwLJaA2nQsartxjc2oefk3CjoW\nWYkDAAAAAACQATzEAQAAAAAAyICFbTE+08w++9tZKLaWRXwv9mHNYT9mH/uwNLAfs499WBrYj9nH\nPiwN7Mfsq1X7UEu2LEz5FhS2HxeqJg4AAAAAAABqBulUAAAAAAAAGcBDHAAAAAAAgAzgIQ4AAAAA\nAEAG8BAHAAAAAAAgA3iIAwAAAAAAkAE8xAEAAAAAAMgAHuIAAAAAAABkAA9xAAAAAAAAMoCHOAAA\nAAAAABnAQxwAAAAAAIAM4CEOAAAAAABABvAQBwAAAAAAIAN4iAMAAAAAAJABPMQBAAAAAADIAB7i\nAAAAAAAAZAAPcQAAAAAAADKAhzgAAAAAAAAZwEMcAAAAAACADOAhDgAAAAAAQAYsvjCT69Sps6Cq\nNgT5LViwoE4x3od9WKNmLliwoEkx3oj9WHM4FksCx2IJ4FgsCRyLJYBjsSRwLJYAjsWSUNCxyEoc\noPp8VtMbAMDMOBaB2oJjEagdOBaB2qGgY5GHOAAAAAAAABnAQxwAAAAAAIAM4CEOAAAAAABABvAQ\nBwAAAAAAIAMWqjtV1tSrV8/j3r17J2MDBw70uGHDhsnYGWec4fEDDzzg8Q8//FDsTSxb//znPz1u\n165dMla3bl2P69ev7/E333yTzPv66689nj59ejL2888/F2U7AQAAANR+der82ZxJf0/o74449uOP\nP3o8e/bsZN6CBTRpQu3EShwAAAAAAIAM4CEOAAAAAABABpRcOpUuo1t33XU9PvXUU5N57du39zim\n6UycONHjn376qdibWLZ037Ro0cLjvffeO5mnY02aNPH4vffeS+aNGjXK4yeeeCIZ0+WQLIUEqs8/\n/pH+fwOLLbaYx0suuaTHcWmzLmeeP39+Mvbbb78VcxMBAEBGLbHEEh6vtNJKyVjHjh093nrrrT1e\nc801k3l6D3LHHXd4PGTIkGQevwNRW7ESBwAAAAAAIAN4iAMAAAAAAJABPMQBAAAAAADIgJKribPM\nMst4rG3FV1lllWTe3LlzPb766quTsXHjxnlMLYbiWXzxP79ubdq08bhz587JvFatWnk8efJkj2OL\n948++sjjefPmFWszgbKix2WsU9O4cWOP27Vr57HmnJuZtWzZ0mM9ts3SGlcNGjTw+JdffknmzZw5\n0+N77rknGbv11ls9njVrlsfUuwJQ22jr4vXWWy8ZO+KIIzz++OOPPb799tuTeVqbkfvQmqX1HKN8\n1yD9d7FW3NJLL+2xXmf1emxmNmXKFI+///77v9/YEhX3gdbEWW211ZKxPfbYw+M+ffp4vOyyyybz\n9PNs1qyZx/o70ox7DtRerMQBAAAAAADIAB7iAAAAAAAAZEDm06niEjtdutqzZ0+PteW0mdmgQYM8\nvvPOO5Oxcl6yWJV0+ajum4YNGybzdPnoI4884vGDDz6YzPv88889/vXXX4u2nSgObS2taTrxmNXW\n0nE/snS1OOIS7ebNm3vco0cPj/W4NDPr0qWLx7oUOS4N16XNGse5GsfvQevWrT1eddVVk7FGjRp5\nfPHFF3tMGiXKhR4vuc6tcV5MQY4pjCge/dw1bXS33XZL5mnL459//tnjb7/9Npl33XXXeazXSFQd\n3YfLLbecx926dUvmfffddx6/++67HmuZBrP0/iVeF/UaN2DAAI9XXnnlZN4VV1zhcbwHLqf73ngv\nqG2/4+87vb/R1MaYlqgp3BMmTPA4/gbkPhS1FStxAAAAAAAAMoCHOAAAAAAAABmQ+XSq2HXqlFNO\n8ViXQ8bK/9r9hPSpqhHTJbSCvKZw1K9fP5mnKVS33HKLx1oh3ix/xwaWP1YPTdOJy8YvuOACj5s2\nbepx3Ddz5szx+LXXXkvGBg4c6PH777/vMd06/ioeb5pmseaaayZjJ5xwgsdbbbWVx5oGUNF7/iGm\nZejrmMKhKQO6pDx2gNDvkqZPmZntueeeHuu5W5eyI7dcqTia4mpm1qRJE4/jd0H/naYNfPPNN8k8\nTTXI9z3RYzieE8r1/K37Ke6b9ddf3+O99trL4w022CCZV69ePY//97//JWPnn3++x3reRXHpOVBT\nNszS773eo+o9kVnakY90quqh16RDDjnE4379+iXzRo0a5fGnn37qcUyJy3eO0w6PG2+8sccxPVLT\nrvJ1ySo3mkoW06m0c612zozpZyNGjPD4ueee81jvWZBN8VjRVP5YXiDX/VGULx1Zx6ozzZGVOAAA\nAAAAABnAQxwAAAAAAIAM4CEOAAAAAABABmSyJo7WVTj++OOTsXbt2nk8ZswYj2+77bZkXmwFiOKL\neYdHHHGEx5qn+t577yXzdF9pvYVYB4X84Jqhx9+gQYM8Puyww3LOUzE3fPnll/d4yy23TMbatm3r\n8emnn+6x5jKblW/rXD0G4uetrUoPP/zwZKx3794ea+vwfPnA2tLzq6++SsbGjRvn8dSpU5MxPdfq\ncb/ZZpsl87Q+RDy2tUZOmzZtPI7njnKtpWKWnm+1to1ZWjdF932sp6Jt3rU1q1m6/7XmwJNPPpnM\ne/vttz2O3wX93uj3ItZR0tbx8bxfavtY95vWv9hnn32Sebrf9BiItaX0PKDXXLP0s9PzKTUgFp1+\ntvr9HTlyZDJv55139rhDhw4er7HGGsm8hg0behxrflATrmro/cbuu+/usdb0MzN79dVXPdZajbEW\nhn4n4pj+Lb32xfpHWguwnFqK/x39bOP9iNY51ZpUX3zxRTLvnXfe8Vg/d62fEv9WqV1/aiu9B9Rr\nZKwZpTXg9D5xnXXWSeb179/f406dOiVj+p76ffnggw+SeXqfq8elWXovqvc9ei0wK/73h5U4AAAA\nAAAAGcBDHAAAAAAAgAzIZDqVLpPq1atXMqbpN1dccYXHX375ZTKP5ahVb4UVVkheawtNTdt4/PHH\nk3mTJ0/2ON9+ypdOxZLH4okpNhdeeKHHmkIV03l0ib4uY33ppZeSec2aNfNY2+iambVo0cLjyy67\nzOPYbv7ll1/2uFz3fdxPmqamS/PN0tQYXXocUyB1Kaimzdxyyy3JPF12Gt9D96G2M4/LXevXr2+5\naOtW/Vvluq//oEuJNU0jphnr0v2lllqqoPeOn60u5dflx6uttloyT9PztP2uWfp9mjFjhseanmWW\nni9i6nPW93lMU9t000091hSnlVZaKZmnrds1nXT+/Pk5369Vq1bJmC4pHzJkiMcffvhhIZuOAml6\n78cff5yMjR8/3mNNn1tyySWTeflSW1Ec8TM+8MADPW7fvr3H8Zyjx6Ke0/Kdm+Lf6t69u8d67xTv\nbfTcmPVzX3XR40/vdeK9id5z6Hk5ziONLb/4WyzX9zRf2+94fOh9xMYbb+xxTIXq0qWLx6ussorH\nMQVSUxbj/tXt1d+tmv5vZta1a1ePp0yZkow99thjHg8dOtTjmMJHOhUAAAAAAEAZ4iEOAAAAAABA\nBvAQBwAAAAAAIAMyUxNH8+XOPPNMjzXPzczsuuuu81hbjFMDp/ppLQyzNP9Uc4pjG7dC8081n5Jc\n4aqjeaBmaR0cPS61xoWZ2b777uuxtlmN+0rrAgwcODAZ69mzp8daO2fAgAHJvF133dVjrfFSTuJx\nM336dI8feuihZGzSpEkea72cWDtM/53Wdsj3GccWkNr2UesMNGjQIJmn34tY5+OBBx7wOOYYl5Ol\nl146eX3ooYd6fNppp3ms9ZDM0u+G1nCI+3vOnDkeT5s2LRnTWg16/o61jDQvPdZk0Zx4bemqLXvN\n0vPFJ598kozFGjlZoOfJzTbbLBk766yzPNZ8fG1/a5bW+NMaNvF4O+qoozw+6KCDkjHdV7odWqfF\njOvpotLPT+t5maV1ovS+NNZpWHnllT2OxwCKQ+8pzNK24lpvTM93ZmlNKq2/kk+TJk2S11oTTs8P\nsSZOPA/j7+l5Tu8z4nUxton+A+e/v8pXzyYf/SzjOU7vZ/R8Z2bWt29fj7fbbjuPmzdvnnM7tK5R\n3D49TrVep1l6L6LHerzf0jp18dmD3ks98sgjHlf1/SorcQAAAAAAADKAhzgAAAAAAAAZkJl0qnXX\nXdfjbt26eRyXAd99990e//DDDx7H5XH52lMrltVVXmxrrJ+57puJEycm8wpNfWPfVB1teXnBBRck\nY7rEUPfjHnvskcx79tlnPc63r/QYvvTSS5Ox1Vdf3WNNu+rcuXMyr3Xr1h6PHTs2598qNfq5xmXd\nU6dO9fjBBx9MxrQdora2jcvGNa0p33Gpy8HjMlNdCrvFFlt4HFst63fprbfeSsZuvPHGCueVA90/\n2j7aLE1t1M89pqO9/PLLHmtq2oQJE5J5upQ/LjnW87e23txyyy2TeXqtjtcA/Z7oEmZNIzJLl8Bf\neeWVlnWa8qSt4M3M2rRp4/Fnn33m8YUXXpjM0/OapsfpudrM7Pnnn/dYl6Sbpa3mNY7L3OO+x8LR\n87K2ODZLW0brOXWppZZK5unxXOj9Kv6efpaHH354MhbPV3944403ktd6fSr0PnS99dZLXmu6lvr8\n88+T1+WaHr4w4j7Q64nGcZ6mXelYbHet9yrxPkjPnflSdrJe0iNfOlVMO9J7NP0s43deU6g01d7M\nrEOHDh5ru/F4b6PHi7b9juUFdPu11ICZ2VdffeWxlgHRMg1m6X1J3J/69+I2ViVW4gAAAAAAAGQA\nD3EAAAAAAAAyIDPpVNqRSpe2DR8+PJn39ddfe6zL4/ItBYvL+nXpsy6Vi8vjtOtAoR2Vykm+Zaa6\n3Cx2Gyl0eWq+/auvSbtaeFqFXVOazNLPVqv7v/jii8m8Qj93PXa0c4dZ2vFqjTXWyPkeugSynNKp\nVFzeqcv447kr17GTb5/pvJjCoV0+NMXHzGy//fbzWJejxm3S1J4TTjghGdMOdllflvx34lJu7fD0\n//7f/0vG9HPXfTd58uRk3uDBgz1+7bXXPI6pHrqP49LnFVdc0WNNAWrXrl0yT88dyyyzTDKWq0vW\n22+/nczTrmgxxS+L9HiJaTO6vzVtI3Yk0s9O93W899BzaNy/SvcT18iqE89X2gEuX5pG/J6gOPS8\nFlMbc6X8n3POOcm8fMeV0t8ZvXv3zvm3NA1HO9uYlf71rhji+Uu7UOn+jp+lXtN69Ojhsf6ONEvT\neTQlPL6Hfi9uv/32ZJ7u1yymq8bPTr/bMcVdv9v63xpT/mfPnu2xdlw0S/epdmrU1CeztLOlpmjH\nv6XX2XiPpSl3+++/v8e6381y/3eZpdduTeuq6msrK3EAAAAAAAAygIc4AAAAAAAAGcBDHAAAAAAA\ngAyotTVxYj6+tlb95ptvPNbcebPc+aOav2dm1qRJE4932mmnZGzzzTf3WPMdNc/NzOyJJ57w+J57\n7knGqrPFWFZobuC0adM8ztfKNt//rrWLdD+Zpbmpmq8Z6yto3iS5x3/SOjixfaB+TtqqOuagFkr3\na2zxqW0Gta4E+2rh5MvLLTRnV+uDabt3s7SGTawzoN8f/Y7E+kfnnXeex1pryay8ao5pS3Ezs65d\nu3oc61PpOVCPiS+++CKZp7nnur/135ulNYu0VbhZWgugc+fOFf6bfNtkluazP/fccx7fddddyby4\n/7NOP4dYJ0jH9PMptD5VpK2ptYWuWXoMa20WzqdVJ+5HbRmt9z6xzbsef/fff38yVtlrLcy22247\nj5s2bZqM6XEwatQoj7Umh1nuYzMel3r8ac0Vs/Q3idaFjLWwdF6+47Sc61rF+4Mvv/zS47XXXtvj\neL3T62njxo09jt8LrT0Xf5vmqikYW8rrdTLWyym0xlJtoueg+PlrzRn9zmodWbP0d7LWvzQzmzhx\nYoV/V+vexPfXfRHr3ujvB60HZ2a21VZbebzDDjt4HOuS6fk61vC59957PdZ6f9TEAQAAAAAAAA9x\nAAAAAAAAsqDWplPFdsK6FOqjjz7yeNasWTnfQ5dT6RJjM7Njjz3W4169eiVjupxdl261bt06mdev\nXz+PR48enYzpcvByXaoc2xBrKpMuH9RljGbpkjtdshffT5en9u/fPxnT1nAff/yxx2+++WYyT9ti\nT58+PRkrpxSOSJcD5vscdJ/GlMVCv/eaXnD66acnY3HZ4x/isszYohjFoefQ1VZbzeNLL700mde9\ne3ePtW2uWXoMjx8/3uOrrroqmTdy5EiPs9iCs1jicaTXrphqpceYfs4xxUn3T6NGjTyOqT3bbrut\nx+uss04ypsvIdRvj8mZNX9U0AbM0Bfm///2vxzH9S/d/KVw/9TPR+xczs44dO3qsy/vXX3/9ZN57\n773nsS5Dj9dFTUGN6QPq6aef9rgUPuOs0H2n1894H7Tmmmt6HPdjPOaQW0x/OeqoozyO6RL6uT7/\n/PMe57umaRpcTD3fc889PW7ZsmUypqkfej/ctm3bZN6ECRMq3L74WuOYnlPqx3e8X3j22Wc93nLL\nLT2O50q9Tup3IV+KcEyPidfrP2jJDjOzffbZx2P93WGWptBlMS0ubrP+Zsj331Ps76Xui5jaqOfX\no48+OhnbZpttPNaSDvF4mzp1qsf/+te/kjFN4avO442VOAAAAAAAABnAQxwAAAAAAIAMqLXpVH36\n9Ele6zJyXaKdb2mbLoGMKVNa+f/rr79Oxl566SWPtUJ2t27dknnaMUu7dZilSyCzWHm8GDR1wixN\nfdPq79oNLM7TZei67Nws7YijS8jN0u+BpuZtvPHGybxOnTp5HJfH6XaU+nLUaNKkSR7H1CVdaqpV\n3WOnuM8++8xjXcYaOxsdc8wxHm+//fbJmC5V1mNdlzqb/bWyPSonVvTXfXXOOed4rOk5ZumS9Zh+\nN3nyZI9vuOEGj+P3Rbu2ZHFJcbHEzjPaxSueU/X40CXfmjJlZrbJJpt43K5dO49jinC+VBy9TurS\nYe0WaZYe67ED2fDhwz3WYzZ2cyy1VFbdp0OHDk3GNG1KO5pcfvnlyTzt5jVmzBiPY1ej3r17exzT\nRbQjVUwtRtWI5zI9hvX4iB02W7Ro4XG+tDjkF39LrLXWWh7H652edzS1NHYaat68uceauh/vEzUd\nPKZkKU1r1TINcXu1NIBZmoaj19nY3a/Uu7DG68XYsWM91v/2mI6s/+6DDz7wWPepWe5ut2ZmK6+8\nssf6uyam82h6VfwtqffKpZBKXuj9WzHu8/Rz1t99yy+/fDJvwIABHsdjTK+TWkoi3r9cc801Hj/5\n5JPJWE11DGQlDgAAAAAAQAbwEAcAAAAAACADeIgDAAAAAACQAbWqJo7mtmnOqVma66Y597FlnOaH\na+7/TjvtlPP9rr766mRM621oDqW22DVLW5G1b98+5/uXq9iyW+seaL53zFnWeiy6f3fYYYdknuai\nxla2mvuv7xG/V5qbes899+R8j1LMI85H61+88cYbyVjPnj091noON954YzJPPz+trRJzVXX/xBoO\nSnORhw0bloyVWg2NqqbnWv3MtcWxmdkRRxzhcdeuXT2OueWaD/zVV18lYyNGjPBYW39q7jH+lC+/\n/+67707G9PylrWnjcaT1GLTGQjwf6ryYD651Fl544YX/3969hlhVd3EcX8KDVGZTeak0fSy7avfs\nMiKZFU2SaUlMTUH5oosR0cWwoBdFDVgERZH0wgKtRIoINc2uFKZliqUzKV5KfZocxwuN0RTUG58X\n4Xp+//V4duMwZ+bsfb6fV+uwt2eO+3/+e++z+a+1PNZ8frO0tlGsp/XTTz8dclvR56/m/sd6FbNm\nzfJ45syZHp966qnJfnqd1PNuvDZpK+PY8njt2rUex1qA6BlaQ0rHI9br0/oaQ4YMSbZRAy6b3lPE\nOnt6jot1S/S8qbUUzz///GS/Uq2M41zManmsdJvW0TFLz41x3HV+6/U0fo6i37/G2ip6r6j3KnEM\n9Fql9Ya+/fbbZL+mpiaP4xgMHjzYYz1H19fXl9xP6y2Zmb3//vsec17uOp0Pjz/+eLKtoaHB4/jc\nQO9FmpubPZ4/f36y34cffuhx/M3ZW3UcedIAAAAAAACQAzzEAQAAAAAAyIGKSqdSMRVHl+truzdt\nbRtpCkdNTU2yTZf469Jws3RZoi6v1HbUcVtsVVj05YudoUsVzdIx1aWq2u7dLE3X6ejo8DguM9Ul\njlu3bk226RJUTXWLS9T1M8bvSDWnxP35558ev/jii8k2HS9tixqXg+uc0GWssRWfzresY67fhU2b\nNpXcr+j0WGYt0S7VftosTb2ZOnWqx5oyFffT94spU5oqqSkzZmnKgH6v4pJWHfv4HdHzadHPrfH/\n197e7vF3332XbNOl15pGE5f/19bWeqxpqLE9taZQxZTFLVu2eKzLzeOyYj33xnap+lrHuJpaysfx\n1RRDvY7dfffdyX7HHnusx3oPFK9pmi4S2xpr6/k49ugZOge+/PJLj6dPn57sp/cjjzzySLJt2rRp\nHlfT3OksvbbEtFBNnejfv3+yrdTvjHge02Ou2+KciveUSs+T2ipczwdmZl999ZXH8b5n165dHldT\neurh0LGKc0Xbsut9ys6dO5P99Duk9zpm6XHXsRs7dmyyn94r67XaLG0xX03pVFn3r539N9oOfu7c\nuR7X1dUl++k5QX9LmKX3Nq+//rrHWlrFLB2bSplj1fsrFQAAAAAAIEd4iAMAAAAAAJADPMQBAAAA\nAADIgYpKitZ8xc8++yzZdu+993qsrdq0jbhZ2tZY6zlo7qhZmmcac9s0x+7BBx/0ePz48SX/1ief\nfJJsizm01SjWtdBW7ldddZXHOp5mZtddd53H2l5XWwCamW3YsMFjrRthluY6ay5qpP+OMfsfnYsr\nVqxIts2YMcPjxsZGj2PNIq2Jo3Vvtm3bVvJvXXTRRSU/0/79+z3WNq1FF3OAtWWm1smI+feXXnqp\nx9qS2Cydf5qfHevUKK19EuugaI5xrIWl82/ChAket7W1JftpnZ3YxnPfvn0ea/vQotfHMUvPo/G4\n63HS78Vll12W7KctijX/Xq9hZmbPPfecx59//nnJv6XXzFhnQF/Ha2tWfYJqpcdB60k9/fTTyX5a\n30ZbqcZafdqmXNtUm5kNHDjQY/0exBoBKB89Z2ntjXj/oTUXr7zyymSbnuv1uoi/ae21l19+Odmm\n57VTTjkl2abXIK1/odccs/SYax2c+Le0Lke8juv97GOPPeax3vOapde++B2phutfV7S2tnrc0tLi\nsc4ps/R6qrVtYl2arN8Gpa5jWfdS8XPEmoVFFWte6ut4vHReaXzmmWcm+82bN89jrYMb6ZjG3zSz\nZ8/2WOtOxfvQSqmDo1iJAwAAAAAAkAM8xAEAAAAAAMiBikqnUtruzSxd9qZLpm699dZkv1WrVnms\ny4W3b9+e7KcpPLrU3Mzsnnvu8XjcuHElP+PixYs91laRZiwVPxQ9RrrE8eyzz072Gz58uMea7rRx\n48ZkP11iF5fRnXzyyR5rWl1W+96YqsBS1b/F47Bw4UKPdZ6eddZZJd9DWznGpck33HCDx9oO3ixN\ni8tK5ykyTZMxM7vkkks8vvbaaz3WeWOWLhUfMWJEsm3QoEEe61LeeN7SNqv6OQYMGJDsp8v7NWXD\nzGzUqFEe6zJ0PQeYpelVTU1NyTZt3anpeNU2R+P/V+eSLk0ePXp0st9RRx11yPdbvXp18lpTDWKb\n1a4sJeY62HVZ7dm1ze26deuS/Zqbmz0eM2ZMsk2X8eucjamNKB9Nq9F0xpiGqulzMdXgnHPO8Tim\nBiA9V8XUGG0bvHz58mRbV9I9dZzidVHHOs7nZ5991uMvvvjCY00FQ9fofYaWuoj3lzqPamtrPY7f\nmdg6XunvTH2PmKqn97K7d+9OtsXSE0WS1UZc71m0jImZ2bnnnuvxlClTPG5oaEj2i3PuoPibf86c\nOR4vWLAg2ablGfTaGselEu9nWIkDAAAAAACQAzzEAQAAAAAAyIGKTaeKS0u1W5WmUN1yyy3Jfpdf\nfrnHuiwqmjRpksdDhw5Nth133HEe67K8pUuXJvs9+uijHrME8p/pMbr55ps91qWkZunxHzZsmMfX\nXHNNsp+mFsSleLp0UcWuRmvWrPE4Lr+rxErklUCPy44dOzzWzipZ4tLwrVu3ehxTrXRcdbyrKY1G\nU5/M0vOOdp2KHQ50qap2CovbOrvcVZeNx/fTZaZxLuo2XfoaU7x0CbPOe7M0HU/TfIq4DDmOh45r\nTIvSjnDaPVHTSeN7areTt956K9lPl3mX+9jqZ6rEZcqVTI+XpjyapelVsfuYXhf1O7J58+bu/ojo\nBJ2LW7ZsSbbpuT1eMzWlduXKlR4zj/5Zd3fI0w6Rmjpslp7jtPuRmdmyZcs85vdD99LUtddee83j\n66+/PtlP70EmT57scby/ybr30RSqadOmeRx/V+oYx067MX2rWuixjJ0Ub7/9do9vvPFGj3W+RT/+\n+KPHOhZmZuvXr/c4/s4o9dsiD+dTVuIAAAAAAADkAA9xAAAAAAAAcoCHOAAAAAAAADlQsTVxYi7a\nk08+6bG2Mo4tNLW1asxdVJpHHtsVf/PNNx43NjZ6HGu3UDOl67Rd+BVXXJFs0zoNWhtD2wGapXni\nWrvDLM2J1VxkbTdoZrZkyRKPOzo6km15yIesJFl1ajQ3PM4bHauY+69joPtl1XEpGm21aGY2duxY\nj7V+VJwDeuzi2Oi+WbWG9LXmEceaZXqujWOjnyOrzkpWLrLW8ujs38oT/X/EdqaaA3766acn2+rq\n6jyeMGGCx7E+kh53zQ3Xdrtm/19fpbuVmrdFGcdy0mOk8zfW/tMaY3v37k229e3b12Ntgcvx7zl6\nrLVORlNTU7Lfeeed57GOm1laz0jPh0WsEVaJdAzvvPNOj2PNMrVnz57ktdbbRPfS85fWanzvvfeS\n/R544AGPR44c6fF9992X7HfHHXd4HK/P/fr181hr58R7Ka159cwzzyTbYr2kaqHnrlgH8YILN+14\nLwAAB21JREFULvBYj7H+DjBLr3daR2fDhg3Jflm/17u7TlZPYiUOAAAAAABADvAQBwAAAAAAIAcq\nNp0qamtr87ihocHjp556KtlvypQpHuvSxvb29mS/r7/+2uPZs2cn21avXu1xbEWG7rdp06bk9dSp\nUz2ur6/3OLaT1+Xgcemifl8WLVrk8SuvvJLsp639qqltdU/LWqKoS8NjGkhWSlCR6XLt2FJRU86y\nUotUZ9MNteWtmVlLS4vHmgL5xx9/lHy/eM7Uv63pOrpE1iz9f+rfMkvTQoqYxqpjF+fAGWec4fGk\nSZOSbZpOPGTIEI9jWuIvv/zi8YIFCzyOS/zLvZS41PtXU3pkV3U2LVHHOqZsDBo0yGNNzYup56Tl\nlI+Oo57LYptpncMxhUNbjOt+jFvP0HO0/h6J5zG9Z1m3bl2yLaaFoDz0nuOll15KtmkK8sUXX+xx\nvDc5+uijS75/qfms905mZg8//LDHMdUnbyk8h0PnRLzODBw40OPx48cn2/R+UFOGd+7cmeynqW7N\nzc0eV8vvBVbiAAAAAAAA5AAPcQAAAAAAAHKAhzgAAAAAAAA5kJuaOKq1tdXj+++/P9k2c+ZMjzX/\nLrbE1TzJasmdywttCfjCCy94/Oabbyb7XXjhhR7H/GLNm9Tc1FjLo8i5qJVK2zCamU2cONHj2EpV\n7dixw+NqmrMxj3j37t0ea+52Vl2RX3/9NXmtNcHmz5/v8apVq0r+La23EGvsqPg5tGaD5ozHsdYa\nZrFVa6m88yKKNXG0xbzWwDEzGzFihMd6zGJtDG1vumLFipL7lZt+N/Lc1rO3ZR27448/3uP4XdI5\nN3jwYI9jDSVqq5SPjpfet8Q6HDom8Xw7fPhwjwcMGODxzz//3G2fE6Vp/TGNI51Hy5cvT7Zxzut5\nWi/MzOymm27yWNt+19XVJfsdc8wxHsdx07qaOsb628XM7IcffvC46PcwSq/58d5f6xBdffXVyTY9\n5lqrccaMGcl+TU1NHnf2uMZ71DzPRVbiAAAAAAAA5AAPcQAAAAAAAHIgl+lUKqbRxJaayDdNm9HU\nDjOzjz/+uOS/Y6l+ZdHli0OHDk22abpIXNavbVd12WrR6Xf2008/TbaddNJJHp922mke19TUJPut\nWbPG42XLliXbtm3b5rEe43LMldhy/KCY2qitzrNSw/ScUMS5Hf/vmiYVU+v0tR6X3377Ldnv3Xff\n9VhTTWPaTG+1GEfXxbREPZ+eeOKJyTY9v+p5JKYvaktXlI+m9a9cuTLZNnnyZI/j+Gzfvt3jjo6O\nMn06HBTPySNHjvRYz7s6nmbpPWtMVeZc2Pv0nlJLc2iKolna7jqOm97faLpWvO+ppvHW+xJNE41t\nxO+66y6P47Wqvb3d48bGRo/j/XB8BtAZXR2LUungvYmVOAAAAAAAADnAQxwAAAAAAIAcyH06FapX\nNXUoyjvtrtG/f/9km3ZOitXlNdVH021i15Uid1Npa2tLXj///PO99EnKi/n8t3gcYtqZ0u+97hc7\noSxZssRjTZUpx5LgSlxyXGRZKXYxPVXHQ8+hJ5xwQrKfpgUwhuWj17s33ngj2fbRRx95HFOQNZ1K\nr4sojzgH1q9f7/HcuXM9Hj16dLLfO++84/HGjRvL8+HQLXSMtRvSoV4j25FHHunxqFGjPI7dpGtr\naz2O9z0ffPCBxwsXLvS4J+/1YxplVpp/b10nWYkDAAAAAACQAzzEAQAAAAAAyAEe4gAAAAAAAOQA\nNXEAlJ3m/n///ffJtlmzZnn80EMPJdu0NsPSpUs97kpbQSAPYl2olpYWj2NuvraX3rt3r8dz5sxJ\n9mttbfW43LWHqKHSs37//ffk9dtvv+1xHIv6+nqPN2/e7HGsM0Bdo54X5+WuXbsOGaP37dmzx+Mn\nnnjC41gzo8i1+oCDtOalmVlNTY3H48aN83jYsGEl30PvUczMXn31VY87Ojo87snrUZzP+rcr5brI\nShwAAAAAAIAc4CEOAAAAAABADvQ5nCVBffr0qYz1Q1XowIEDpXubHQbGsFetPXDgwJjueKOijmPW\n8sVKwVwshIqdi3FpsraCjq2G+/Xr57Eu8dfUKrPiLutnLmbLaj+u34lyp9j9g4qdi+g85mIhMBcL\noLfn4hFHHOHx9OnTPb7tttuS/TQVePHixcm2efPmebx//36Py3GtKtU6vJfTqTo1F1mJAwAAAAAA\nkAM8xAEAAAAAAMgBHuIAAAAAAADkADVxcqK3cxzRLcg3LgDmYiEwFwuAuVgIzMUCYC4WAnOxACp1\nLvbt2zd5rc8fYt2+SqyH2cOoiQMAAAAAAFAUPMQBAAAAAADIgX8d5v77zOw/5fggyPTvbnwvxrD3\nMI75xxgWA+OYf4xhMTCO+ccYFgPjmH8VO4Z//fVXd71VNejUOB5WTRwAAAAAAAD0DtKpAAAAAAAA\ncoCHOAAAAAAAADnAQxwAAAAAAIAc4CEOAAAAAABADvAQBwAAAAAAIAd4iAMAAAAAAJADPMQBAAAA\nAADIAR7iAAAAAAAA5AAPcQAAAAAAAHLgv4+556BjaJvrAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f4071473a50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "def plot(data1, recon_batch1, recon_batch2):\n",
    "    n = 10  # how many digits we will display\n",
    "    plt.figure(figsize=(20, 6))\n",
    "    data = data1.data.numpy()\n",
    "    predict = recon_batch1.data.numpy()\n",
    "    predict2 = recon_batch2.data.numpy()\n",
    "    for i in range(n):\n",
    "        # display original\n",
    "        ax = plt.subplot(3, n, i + 1)\n",
    "        plt.imshow(data[i][0])\n",
    "        plt.gray()\n",
    "        ax.get_xaxis().set_visible(False)\n",
    "        ax.get_yaxis().set_visible(False)\n",
    "\n",
    "        # display reconstruction\n",
    "        ax = plt.subplot(3, n, i + 1 + n)\n",
    "        plt.imshow(predict[i].reshape(28, 28))\n",
    "        plt.gray()\n",
    "        ax.get_xaxis().set_visible(False)\n",
    "        ax.get_yaxis().set_visible(False)\n",
    "        \n",
    "        # display reconstruction\n",
    "        ax = plt.subplot(3, n, i + 1 + 2*n)\n",
    "        plt.imshow(predict2[i].reshape(28, 28))\n",
    "        plt.gray()\n",
    "        ax.get_xaxis().set_visible(False)\n",
    "        ax.get_yaxis().set_visible(False)        \n",
    "    plt.show()\n",
    "\n",
    "model.eval()\n",
    "temp = model(data_metric_test)\n",
    "plot(data_metric_test, temp[0], recon_batch_metric_test)\n",
    "np.save('origin', data_metric_test.data.numpy())\n",
    "np.save('vanilla', temp[0].data.numpy())\n",
    "np.save('metric', recon_batch_metric_test.data.numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}