SVM_pyramids_Adam.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import cv2\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle as cPickle\n",
    "from matplotlib import pyplot as plt\n",
    "from sklearn.cluster import MiniBatchKMeans\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.metrics import precision_score,confusion_matrix,multilabel_confusion_matrix,recall_score\n",
    "from sklearn.preprocessing import normalize\n",
    "from sklearn import svm\n",
    "from datetime import datetime\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.preprocessing import StandardScaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_images_filenames = cPickle.load(open('resources/train_images_filenames.dat','rb'))\n",
    "test_images_filenames = cPickle.load(open('resources/test_images_filenames.dat','rb'))\n",
    "train_labels = cPickle.load(open('resources/train_labels.dat','rb'))\n",
    "test_labels = cPickle.load(open('resources/test_labels.dat','rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "SIFTdetector = cv2.SIFT_create(nfeatures=300)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_dense_sift(gray,sift,step):\n",
    "    step_size = step\n",
    "    kp = [cv2.KeyPoint(x, y, step_size) for y in range(0, gray.shape[0], step_size) \n",
    "                                        for x in range(0, gray.shape[1], step_size)]\n",
    "    dense_feat = sift.compute(gray, kp)\n",
    "    dense_feat_des = dense_feat[1]\n",
    "    return dense_feat_des"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "step=5\n",
    "\n",
    "today = datetime.now()\n",
    "dt_string = today.strftime(\"%H:%M:%S\")\n",
    "print(f\"{dt_string} started doing step={step}\")\n",
    "\n",
    "Train_descriptors = []\n",
    "Train_label_per_descriptor = []\n",
    "\n",
    "for filename,labels in zip(train_images_filenames,train_labels):\n",
    "    ima=cv2.imread(filename)\n",
    "    gray=cv2.cvtColor(ima,cv2.COLOR_BGR2GRAY)\n",
    "#     kpt,des=SIFTdetector.detectAndCompute(gray,None)\n",
    "    des=compute_dense_sift(gray,SIFTdetector, step)\n",
    "    Train_descriptors.append(des)\n",
    "    Train_label_per_descriptor.append(labels)\n",
    "\n",
    "D=np.vstack(Train_descriptors)\n",
    "\n",
    "k=128\n",
    "codebook = MiniBatchKMeans(n_clusters=k, verbose=False, batch_size=k * 20,compute_labels=False,reassignment_ratio=10**-4,random_state=42)\n",
    "codebook.fit(D)\n",
    "\n",
    "visual_words=np.zeros((len(Train_descriptors),k),dtype=np.float32)\n",
    "for i in range(len(Train_descriptors)):\n",
    "    words=codebook.predict(Train_descriptors[i])\n",
    "    visual_words[i,:]=normalize(np.bincount(words,minlength=k).reshape(1,-1))\n",
    "\n",
    "# knn = KNeighborsClassifier(n_neighbors=37,n_jobs=-1,metric='manhattan')\n",
    "# knn.fit(visual_words, train_labels)\n",
    "\n",
    "today = datetime.now()\n",
    "dt_string = today.strftime(\"%H:%M:%S\")\n",
    "print(f\"{dt_string} started doing step={step}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "fig, ax = plt.subplots(figsize=(10,4))\n",
    "ax.bar(range(0,len(visual_words[0])),visual_words[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scaler = StandardScaler()\n",
    "scaler.fit_transform(visual_words)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Linear Kernel with scaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lin_clf = svm.LinearSVC(max_iter=2000, C=10)\n",
    "lin_clf.fit(visual_words, train_labels)\n",
    "\n",
    "visual_words_test=np.zeros((len(test_images_filenames),k),dtype=np.float32)\n",
    "for i in range(len(test_images_filenames)):\n",
    "    filename=test_images_filenames[i]\n",
    "    ima=cv2.imread(filename)\n",
    "    gray=cv2.cvtColor(ima,cv2.COLOR_BGR2GRAY)\n",
    "#     kpt,des=SIFTdetector.detectAndCompute(gray,None)\n",
    "    des=compute_dense_sift(gray,SIFTdetector, step)\n",
    "    words=codebook.predict(des)\n",
    "    visual_words_test[i,:]=normalize(np.bincount(words,minlength=k).reshape(1,-1))\n",
    "\n",
    "visual_words_test=scaler.transform(visual_words_test)\n",
    "scores = cross_val_score(lin_clf, visual_words_test, test_labels, cv=5)\n",
    "accuracy=scores.mean()*100\n",
    "\n",
    "print(accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RBF Kernel with scaler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rbf_svc = svm.SVC(kernel='rbf', C=1000, gamma=0.001)\n",
    "rbf_svc.fit(visual_words, train_labels)\n",
    "\n",
    "visual_words_test=np.zeros((len(test_images_filenames),k),dtype=np.float32)\n",
    "for i in range(len(test_images_filenames)):\n",
    "    filename=test_images_filenames[i]\n",
    "    ima=cv2.imread(filename)\n",
    "    gray=cv2.cvtColor(ima,cv2.COLOR_BGR2GRAY)\n",
    "#     kpt,des=SIFTdetector.detectAndCompute(gray,None)\n",
    "    des=compute_dense_sift(gray,SIFTdetector, step)\n",
    "    words=codebook.predict(des)\n",
    "    visual_words_test[i,:]=normalize(np.bincount(words,minlength=k).reshape(1,-1))\n",
    "\n",
    "visual_words_test=scaler.transform(visual_words_test)\n",
    "scores = cross_val_score(rbf_svc, visual_words_test, test_labels, cv=5)\n",
    "\n",
    "accuracy=scores.mean()*100\n",
    "\n",
    "print(accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nfor filename,labels in zip(train_images_filenames,train_labels):\\n    ima=cv2.imread(filename)\\n    gray=cv2.cvtColor(ima,cv2.COLOR_BGR2GRAY)\\n    break\\n\\ntiles = get_pyramid_image_cells(gray,2)\\n'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_pyramid_image_cells(image,level):\n",
    "    # CURRENTLY WORKING ONLY FOR LEVELS 0,1,2 (1,4, and 16 cells)\n",
    "    M = (image.shape[0])//(level+1)\n",
    "    N = (image.shape[1])//(level+1)\n",
    "    tiles = [image[x:x+M,y:y+N] for x in range(0,image.shape[0],M) for y in range(0,image.shape[1],N)]\n",
    "#     print(image.shape)\n",
    "#     print(\"level=\", level)\n",
    "#     print(\"returning \" + str(len(tiles)) + \" tiles\")\n",
    "    return tiles\n",
    "\n",
    "\n",
    "'''\n",
    "for filename,labels in zip(train_images_filenames,train_labels):\n",
    "    ima=cv2.imread(filename)\n",
    "    gray=cv2.cvtColor(ima,cv2.COLOR_BGR2GRAY)\n",
    "    break\n",
    "\n",
    "tiles = get_pyramid_image_cells(gray,2)\n",
    "'''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calculating a histogram per cell and concatenating them\n",
    "Compute a histogram for each part of the image and put these histograms together in a weighted concatenation \n",
    "(since there will be less words overall in a smaller cell).\n",
    "The weighted concatenation can be a simple normalization of every histogram so the area of the histogram adds up to 1.\n",
    "A pyramid of l levels will yield in 2^(2l) histograms n_words = k*(1+2**(2*pyramid_levels)).\n",
    "the final image histogram is a concatenated vector  of  the  bin  values  of  all  the  histograms in  the  pyramid.\n",
    "concatenation method used in:\n",
    "http://lear.imag.fr/pub/203-bosch-civr07.pdf\n",
    "https://core.ac.uk/download/pdf/82695491.pdf\n",
    "\n",
    "TODO: Initialisation of concatenated_histogram. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "19:20:54 started doing step=5\n",
      "(256, 256)\n",
      "level= 0\n",
      "returning 1 tiles\n",
      "(256, 256)\n",
      "level= 1\n",
      "returning 4 tiles\n",
      "(256, 256)\n",
      "level= 2\n",
      "returning 16 tiles\n",
      "19:21:09 started doing step=5\n"
     ]
    }
   ],
   "source": [
    "step=5\n",
    "\n",
    "today = datetime.now()\n",
    "dt_string = today.strftime(\"%H:%M:%S\")\n",
    "print(f\"{dt_string} started doing step={step}\")\n",
    "\n",
    "Train_descriptors = []\n",
    "Train_label_per_descriptor = []\n",
    "pyramid_levels = 2 # level zero has 1 tile, level 1 has 4 tiles => 5 histograms that will be concatenated\n",
    "for filename,labels in zip(train_images_filenames,train_labels):\n",
    "    ima=cv2.imread(filename)\n",
    "    gray=cv2.cvtColor(ima,cv2.COLOR_BGR2GRAY)\n",
    "    # break the image into pieces\n",
    "    image_cells = []\n",
    "    for i in range(pyramid_levels+1):\n",
    "\n",
    "        level_cells = get_pyramid_image_cells(gray,level=i)\n",
    "        image_cells = image_cells + level_cells\n",
    "        # compute descriptors for each tile\n",
    "    Train_descriptors_cell = []\n",
    "    for cell in image_cells:\n",
    "        des=compute_dense_sift(cell,SIFTdetector, step)\n",
    "        Train_descriptors_cell.append(des)\n",
    "    \n",
    "    Train_descriptors.append(Train_descriptors_cell)\n",
    "    Train_label_per_descriptor.append(labels)\n",
    "    break\n",
    "    \n",
    "today = datetime.now()\n",
    "dt_string = today.strftime(\"%H:%M:%S\")\n",
    "print(f\"{dt_string} started doing step={step}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "21"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(Train_descriptors[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2704"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(Train_descriptors[0][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "128"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(Train_descriptors[0][0][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "         0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "         0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "         0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   1.,   2.,\n",
       "         0.,   0.,   0.,   0.,   2.,   2.,   4.,  26.,   4.,   0.,   0.,\n",
       "         0.,  26.,   5.,   3.,   5.,   0.,   0.,   0.,   8.,   0.,   0.,\n",
       "         0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,  57.,  31.,   1.,\n",
       "         0.,   0.,   0.,  96.,  82., 156., 156.,   8.,   0.,   0.,  17.,\n",
       "       156.,  41.,  24.,  19.,   0.,   0.,   0., 125.,   0.,   0.,   0.,\n",
       "         0.,   0.,   0.,   0.,   0.,   0.,   0.,  25.,  29.,   0.,   0.,\n",
       "        23.,   7., 156., 116., 124.,  76.,   0.,   0., 105., 156., 156.,\n",
       "        57.,   4.,   0.,   0.,   0.,   5., 156.], dtype=float32)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Train_descriptors[0][0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train all cells as separate images\n",
    "D=np.vstack([des for descriptors_cells in Train_descriptors for des in descriptors_cells])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "D.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "k=128\n",
    "codebook = MiniBatchKMeans(n_clusters=k, verbose=False, batch_size=k * 20,compute_labels=False,reassignment_ratio=10**-4,random_state=42)\n",
    "codebook.fit(D)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(k*(1+2**(2*pyramid_levels)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(Train_descriptors[0][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "visual_words=np.zeros((len(Train_descriptors),k*(1+2**(2*pyramid_levels))),dtype=np.float32)\n",
    "for idx,cells in enumerate(Train_descriptors):\n",
    "\n",
    "    image_histograms = []\n",
    "    for id2,image_cell in enumerate(cells):\n",
    "        cell_words = codebook.predict(image_cell)\n",
    "        image_histograms.append(normalize(np.bincount(cell_words,minlength=k).reshape(1,-1)))\n",
    "        # concatenated histogram has k*(1+2**(2*pyramid_levels)) bins\n",
    "        # normalize(np.bincount(words,minlength=k).reshape(1,-1))\n",
    "\n",
    "    concatenated_histogram=image_histograms[0].copy()\n",
    "    for cell_idx in range(1,len(image_histograms)):\n",
    "        # concatenate histograms\n",
    "        concatenated_histogram = np.concatenate((concatenated_histogram,image_histograms[cell_idx]))\n",
    "    visual_words[idx,:]=concatenated_histogram.flatten()\n",
    "    \n",
    "    #visual_words[idx,:]=np.bincount(words,minlength=k*(2**(2*pyramid_levels))) # this will just be longer as levels \n",
    "                                                                                # increase \n",
    "                                                                                # see: k*(2**(2*pyramid_levels)) long\n",
    "\n",
    "# knn = KNeighborsClassifier(n_neighbors=37,n_jobs=-1,metric='manhattan')\n",
    "# knn.fit(visual_words, train_labels) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "visual_words[0,:].shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "TRAINED Visual Words tryin further worked until now"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "scaler = StandardScaler()\n",
    "scaler.fit_transform(visual_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lin_clf = svm.LinearSVC(max_iter=2000,C=10)\n",
    "lin_clf.fit(visual_words, train_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rbf_svc = svm.SVC(kernel='rbf',C=1000,gamma=0.001)\n",
    "rbf_svc.fit(visual_words, train_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "visual_words[1].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "inter_svc = svm.SVC(kernel=intersection_kernel_cmp)\n",
    "inter_svc.fit(visual_words,train_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "today = datetime.now()\n",
    "dt_string = today.strftime(\"%H:%M:%S\")\n",
    "print(f\"{dt_string} started doing step={step}\")\n",
    "\n",
    "Test_descriptors = []\n",
    "Test_label_per_descriptor = []\n",
    "pyramid_levels = 1 # level zero has 1 tile, level 1 has 4 tiles => 5 histograms that will be concatenated\n",
    "for filename,labels in zip(test_images_filenames,test_labels):\n",
    "    ima=cv2.imread(filename)\n",
    "    gray=cv2.cvtColor(ima,cv2.COLOR_BGR2GRAY)\n",
    "    # break the image into pieces\n",
    "    image_cells = []\n",
    "    for i in range(pyramid_levels+1):\n",
    "\n",
    "        level_cells = get_pyramid_image_cells(gray,level=i)\n",
    "        image_cells = image_cells + level_cells\n",
    "        # compute descriptors for each tile\n",
    "    Test_descriptors_cell = []\n",
    "    for cell in image_cells:\n",
    "        des=compute_dense_sift(cell,SIFTdetector, step)\n",
    "        Test_descriptors_cell.append(des)\n",
    "        \n",
    "    Test_descriptors.append(Test_descriptors_cell)\n",
    "    Test_label_per_descriptor.append(labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "visual_words_test=np.zeros((len(Test_descriptors),k*(1+2**(2*pyramid_levels))),dtype=np.float32)\n",
    "for idx,cells in enumerate(Test_descriptors):\n",
    "\n",
    "    image_histograms = []\n",
    "    for id2,image_cell in enumerate(cells):\n",
    "        cell_words = codebook.predict(image_cell)\n",
    "        image_histograms.append(normalize(np.bincount(cell_words,minlength=k).reshape(1,-1)))\n",
    "        # concatenated histogram has k*(1+2**(2*pyramid_levels)) bins\n",
    "        # normalize(np.bincount(words,minlength=k).reshape(1,-1))\n",
    "\n",
    "    concatenated_histogram=image_histograms[0].copy()\n",
    "    for cell_idx in range(1,len(image_histograms)):\n",
    "        # concatenate histograms\n",
    "        concatenated_histogram = np.concatenate((concatenated_histogram,image_histograms[cell_idx]))\n",
    "    visual_words_test[idx,:]=concatenated_histogram.flatten()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "visual_words_test=scaler.transform(visual_words_test)\n",
    "scores = cross_val_score(lin_clf, visual_words_test, test_labels, cv=5)\n",
    "accuracy=scores.mean()*100\n",
    "\n",
    "print(accuracy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# visual_words_test=scaler.transform(visual_words_test)\n",
    "scores = cross_val_score(rbf_svc, visual_words_test, test_labels, cv=5)\n",
    "accuracy=scores.mean()*100\n",
    "\n",
    "print(accuracy)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## creating kernel for intersection of histograms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def intersection_kernel_cmp(X, Y):\n",
    "    return cv2.compareHist(X, Y, method=cv2.HISTCMP_INTERSECT)\n",
    "\n",
    "# clf = svm.SVC(kernel=my_kernel)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "intersection_kernel_cmp(visual_words[0],visual_words[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## GridSearchCV cross validation finding best parameters for RBF and Linear"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# Set the parameters by cross-validation\n",
    "tuned_parameters = [{'kernel': ['rbf'], 'gamma': [1e-3, 1e-4],\n",
    "                     'C': [1, 10, 100, 1000]},\n",
    "                    {'kernel': ['linear'], 'C': [1, 10, 100, 1000]}]\n",
    "\n",
    "scores = ['precision_macro', 'recall_macro','accuracy']\n",
    "\n",
    "today = datetime.now()\n",
    "dt_string = today.strftime(\"%H:%M:%S\")\n",
    "print(f\"{dt_string} started doing step={step}\")\n",
    "\n",
    "data_results=[]\n",
    "\n",
    "for score in scores:\n",
    "    print(\"# Tuning hyper-parameters for %s\" % score)\n",
    "    print()\n",
    "\n",
    "    clf = GridSearchCV(\n",
    "        SVC(), tuned_parameters, scoring=score\n",
    "    )\n",
    "    clf.fit(visual_words, train_labels)\n",
    "\n",
    "    print(\"Best parameters set found on development set:\")\n",
    "    print()\n",
    "    print(clf.best_params_)\n",
    "    print()\n",
    "    print(\"Grid scores on development set:\")\n",
    "    print()\n",
    "    means = clf.cv_results_['mean_test_score']\n",
    "    stds = clf.cv_results_['std_test_score']\n",
    "    for mean, std, params in zip(means, stds, clf.cv_results_['params']):\n",
    "        c=params['C']\n",
    "        ker=params['kernel']\n",
    "        if ker=='rbf': \n",
    "            gamma=params['gamma']\n",
    "        else: gamma=None\n",
    "            \n",
    "        data_results.append([mean,std,c,gamma,ker,score])\n",
    "        \n",
    "        print(\"%0.3f (+/-%0.03f) for %r\"\n",
    "              % (mean, std * 2, params))\n",
    "    print()\n",
    "\n",
    "#     print(\"Detailed classification report:\")\n",
    "#     print()\n",
    "#     print(\"The model is trained on the full development set.\")\n",
    "#     print(\"The scores are computed on the full evaluation set.\")\n",
    "#     print()\n",
    "#     y_true, y_pred = y_test, clf.predict(X_test)\n",
    "#     print(classification_report(y_true, y_pred))\n",
    "#     print()\n",
    "\n",
    "today = datetime.now()\n",
    "dt_string = today.strftime(\"%H:%M:%S\")\n",
    "print(f\"{dt_string} started doing step={step}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df=pd.DataFrame(data_results, columns=['mean_val','std_val','c','gamma','ker','score'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import metrics\n",
    "# sorted(metrics.SCORERS.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('GridSearchCV_rbf_linear_params.pkl','wb') as f:\n",
    "    cPickle.dump(df,f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(figsize=(10,8))\n",
    "ax.set(title='Accuracy vs C', xlim=(0,1200), ylim=(0,1))\n",
    "df2=df[df.score=='accuracy']\n",
    "df2_rbf=df2[(df2.ker=='rbf')]\n",
    "df2_linear=df2[(df2.ker=='linear')]\n",
    "print(\"max for rbf\\n\",df2_rbf[df2_rbf.mean_val==df2_rbf.mean_val.max()])\n",
    "print(\"max for linear\\n\",df2_linear.loc[df2_linear.mean_val==df2_linear.mean_val.max()])\n",
    "# print(df2[df2.recall==df2.recall.max()])\n",
    "\n",
    "ax.plot(df2_linear.c, df2_linear.mean_val, 'ro', label='linear')\n",
    "ax.plot(df2_rbf.c, df2_rbf.mean_val, 'go', label='rbf')\n",
    "ax.legend()\n",
    "\n",
    "max_val = max(df2.mean_val)\n",
    "max_idx = df2.c[df2.mean_val.idxmax()]\n",
    "# ax.annotate(\"Max {:.2f}\".format(max_val), xy=(max_idx, max_val),\n",
    "#            weight='bold', size=14)\n",
    "\n",
    "# arrowprops is a mpl.patches.FancyArrowPatch\n",
    "_ = ax.annotate(\"Max accuracy {:.2f}\".format(max_val), xy=(max_idx, max_val),\n",
    "                weight='bold', size=14, \n",
    "                xytext=(.5, .9),\n",
    "                textcoords='axes fraction',\n",
    "                family='comic sans ms',\n",
    "                arrowprops={})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## My Analysis for different DENSE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_vis(step):\n",
    "    step=step\n",
    "\n",
    "    today = datetime.now()\n",
    "    dt_string = today.strftime(\"%H:%M:%S\")\n",
    "    print(f\"{dt_string} started doing step={step}\")\n",
    "\n",
    "    Train_descriptors = []\n",
    "    Train_label_per_descriptor = []\n",
    "\n",
    "    for filename,labels in zip(train_images_filenames,train_labels):\n",
    "        ima=cv2.imread(filename)\n",
    "        gray=cv2.cvtColor(ima,cv2.COLOR_BGR2GRAY)\n",
    "    #     kpt,des=SIFTdetector.detectAndCompute(gray,None)\n",
    "        des=compute_dense_sift(gray,SIFTdetector, step)\n",
    "        Train_descriptors.append(des)\n",
    "        Train_label_per_descriptor.append(labels)\n",
    "\n",
    "    D=np.vstack(Train_descriptors)\n",
    "\n",
    "    k=128\n",
    "    codebook = MiniBatchKMeans(n_clusters=k, verbose=False, batch_size=k * 20,compute_labels=False,reassignment_ratio=10**-4,random_state=42)\n",
    "    codebook.fit(D)\n",
    "\n",
    "    visual_words=np.zeros((len(Train_descriptors),k),dtype=np.float32)\n",
    "    for i in range(len(Train_descriptors)):\n",
    "        words=codebook.predict(Train_descriptors[i])\n",
    "        visual_words[i,:]=normalize(np.bincount(words,minlength=k).reshape(1,-1))\n",
    "\n",
    "    # knn = KNeighborsClassifier(n_neighbors=37,n_jobs=-1,metric='manhattan')\n",
    "    # knn.fit(visual_words, train_labels)\n",
    "\n",
    "    today = datetime.now()\n",
    "    dt_string = today.strftime(\"%H:%M:%S\")\n",
    "    print(f\"{dt_string} started doing step={step}\")\n",
    "    \n",
    "    scaler = StandardScaler()\n",
    "    scaler.fit_transform(visual_words)\n",
    "    \n",
    "    lin_clf = svm.LinearSVC(max_iter=2000, C=10)\n",
    "    lin_clf.fit(visual_words, train_labels)\n",
    "    \n",
    "    rbf_svc = svm.SVC(kernel='rbf', C=1000, gamma=0.001)\n",
    "    rbf_svc.fit(visual_words, train_labels)\n",
    "\n",
    "    visual_words_test=np.zeros((len(test_images_filenames),k),dtype=np.float32)\n",
    "    for i in range(len(test_images_filenames)):\n",
    "        filename=test_images_filenames[i]\n",
    "        ima=cv2.imread(filename)\n",
    "        gray=cv2.cvtColor(ima,cv2.COLOR_BGR2GRAY)\n",
    "    #     kpt,des=SIFTdetector.detectAndCompute(gray,None)\n",
    "        des=compute_dense_sift(gray,SIFTdetector, step)\n",
    "        words=codebook.predict(des)\n",
    "        visual_words_test[i,:]=normalize(np.bincount(words,minlength=k).reshape(1,-1))\n",
    "\n",
    "    visual_words_test=scaler.transform(visual_words_test)\n",
    "    \n",
    "    scores = cross_val_score(lin_clf, visual_words_test, test_labels, cv=5)\n",
    "    accuracy_linear=scores.mean()*100\n",
    "\n",
    "    print(\"linear=\",accuracy_linear)\n",
    "\n",
    "    scores = cross_val_score(rbf_svc, visual_words_test, test_labels, cv=5)\n",
    "    accuracy_rbf=scores.mean()*100\n",
    "\n",
    "    print(\"rbf=\",accuracy_rbf)\n",
    "    \n",
    "    return accuracy_linear,accuracy_rbf\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "17:16:52 started doing step=5\n",
      "17:19:42 started doing step=5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\adama\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:976: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\"Liblinear failed to converge, increase \"\n",
      "C:\\Users\\adama\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:976: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\"Liblinear failed to converge, increase \"\n",
      "C:\\Users\\adama\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:976: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\"Liblinear failed to converge, increase \"\n",
      "C:\\Users\\adama\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:976: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\"Liblinear failed to converge, increase \"\n",
      "C:\\Users\\adama\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:976: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\"Liblinear failed to converge, increase \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "linear= 65.54865424430642\n",
      "rbf= 78.93719806763283\n",
      "17:20:34 started doing step=20\n",
      "17:22:23 started doing step=20\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\adama\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:976: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\"Liblinear failed to converge, increase \"\n",
      "C:\\Users\\adama\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:976: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\"Liblinear failed to converge, increase \"\n",
      "C:\\Users\\adama\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:976: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\"Liblinear failed to converge, increase \"\n",
      "C:\\Users\\adama\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:976: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\"Liblinear failed to converge, increase \"\n",
      "C:\\Users\\adama\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:976: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\"Liblinear failed to converge, increase \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "linear= 63.44605475040257\n",
      "rbf= 72.61406333870102\n",
      "17:23:14 started doing step=50\n",
      "17:25:00 started doing step=50\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\adama\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:976: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\"Liblinear failed to converge, increase \"\n",
      "C:\\Users\\adama\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:976: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\"Liblinear failed to converge, increase \"\n",
      "C:\\Users\\adama\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:976: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\"Liblinear failed to converge, increase \"\n",
      "C:\\Users\\adama\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:976: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\"Liblinear failed to converge, increase \"\n",
      "C:\\Users\\adama\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:976: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\"Liblinear failed to converge, increase \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "linear= 52.41315850011502\n",
      "rbf= 62.94762671574266\n",
      "17:25:47 started doing step=100\n",
      "17:27:01 started doing step=100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\adama\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:976: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\"Liblinear failed to converge, increase \"\n",
      "C:\\Users\\adama\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:976: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\"Liblinear failed to converge, increase \"\n",
      "C:\\Users\\adama\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:976: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\"Liblinear failed to converge, increase \"\n",
      "C:\\Users\\adama\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:976: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\"Liblinear failed to converge, increase \"\n",
      "C:\\Users\\adama\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:976: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\"Liblinear failed to converge, increase \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "linear= 53.275055593896184\n",
      "rbf= 59.352810367303135\n",
      "17:27:41 started doing step=300\n",
      "17:28:17 started doing step=300\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\adama\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:976: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\"Liblinear failed to converge, increase \"\n",
      "C:\\Users\\adama\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:976: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\"Liblinear failed to converge, increase \"\n",
      "C:\\Users\\adama\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:976: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\"Liblinear failed to converge, increase \"\n",
      "C:\\Users\\adama\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:976: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\"Liblinear failed to converge, increase \"\n",
      "C:\\Users\\adama\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:976: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\"Liblinear failed to converge, increase \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "linear= 54.897630549804454\n",
      "rbf= 57.61904761904761\n",
      "17:29:23 started doing step=500\n",
      "17:29:58 started doing step=500\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\adama\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:976: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\"Liblinear failed to converge, increase \"\n",
      "C:\\Users\\adama\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:976: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\"Liblinear failed to converge, increase \"\n",
      "C:\\Users\\adama\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:976: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\"Liblinear failed to converge, increase \"\n",
      "C:\\Users\\adama\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:976: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\"Liblinear failed to converge, increase \"\n",
      "C:\\Users\\adama\\Anaconda3\\lib\\site-packages\\sklearn\\svm\\_base.py:976: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\"Liblinear failed to converge, increase \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "linear= 53.77195000383408\n",
      "rbf= 57.11908595966566\n"
     ]
    }
   ],
   "source": [
    "data_results_step =[]\n",
    "steps_to_do=[5,20,50,100,300,500]\n",
    "for step in steps_to_do:\n",
    "    accuracy_linear,accuracy_rbf=calculate_vis(step)\n",
    "    data_results_step.append([step,accuracy_linear,accuracy_rbf,1,1])\n",
    "    \n",
    "df=pd.DataFrame(data_results_step, columns=['step','accuracy_linear','accuracy_rbf','normalized','scaler'])    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "max for rbf\n",
      "    step  accuracy_linear  accuracy_rbf  normalized  scaler\n",
      "0     5        65.548654     78.937198           1       1\n",
      "max for linear\n",
      "    step  accuracy_linear  accuracy_rbf  normalized  scaler\n",
      "0     5        65.548654     78.937198           1       1\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x1d206583220>"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAnEAAAHwCAYAAADJiTnYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deZRdZZnv8e+TgcxkZjIMQQIhYkMg2FFRiDhAGLWNoKLoVWm1HfA6Yy9Em27HVrS52CIItDIjF7mACCKK0qAEAzYQZhlCAoRAmEJChuf+sXeZIlaSk+ScOvVWfT9rnXXOnp/9lov8fN89RGYiSZKksvRrdwGSJEnacIY4SZKkAhniJEmSCmSIkyRJKpAhTpIkqUCGOEmSpAIZ4iRJkgpkiJPUFBHxm4h4KiIGtbuWEkXECRHx03bXIakchjhJmywidgBeByRwaDcfe0B3Hk+SegpDnKRmeC9wI3AmcHTnBRGxbURcHBELI2JRRJzcadmHImJuRDwbEXdExJ71/IyInTqtd2ZEnFj/3i8i5kXE5yPiUeCMiBgdEZfVx3iq/j2h0/ZjIuKMiJhfL7+knn9bRBzSab2BEfFEROyx5gnWdR7caXpAve6eETE4In5an9/iiLgpIrbsqqHquh+pz/muiNg/Ig4AjgOOiIjnIuLWet2REXF6RCyotzkxIvrXy94XEddHxH9ExNMRcWdE7N/pOO+LiPvr4/wlIt7dwN9RUkEMcZKa4b3A2fXnLR0Bpg4clwEPAjsALwPOq5fNAk6ot92cqgdvUYPH2woYA2wPHEP137Iz6untgBeAkzut/xNgKPAKYAvgu/X8/wKO6rTeTGBBZt7SxTHPBd7ZafotwBOZ+Seq4DoS2BYYC3y4ruElImIX4GPA3pk5ot7HA5l5JfBvwPmZOTwzd683OQtYAewETAXeDHyw0y7/HrgfGAd8Gbi4DqzDgO8DB9bHeQ3Q1TlJKpghTtImiYh9qMLTBZl5M3Af8K568auAbYDPZubzmbk0M39fL/sg8M3MvCkr92bmgw0edhXw5cxclpkvZOaizPxZZi7JzGeBfwX2revbGjgQ+HBmPpWZyzPzt/V+fgrMjIjN6+n3UAW+rpwDHBoRQ+vpd9XzAJZThbedMnNlZt6cmc90sY+VwCBgSkQMzMwHMvO+rg5WB+EDgWPrtnucKnwe2Wm1x4GT6nM6H7gLOKhTG+0WEUMyc0Fm3r6W85JUKEOcpE11NHBVZj5RT5/D6iHVbYEHM3NFF9ttSxX4NsbCzFzaMRERQyPihxHxYEQ8A1wHjKp7ArcFnszMp9bcSWbOB64H/iEiRlGFprO7OmBm3gvMBQ6pg9yhrA5xPwF+CZxXD9l+MyIGrmUfx1L1QD4eEedFxDZrOcftgYHAgnqIdjHwQ6qexA6PZGZ2mn4Q2CYznweOoOoRXBARl0fE5LUcR1KhDHGSNlpEDAHeAewbEY/W16h9Ctg9InYHHga2W8vNBw8DL1/LrpdQDX922GqN5bnG9KeBXYC/z8zNgdd3lFgfZ0wd0rpyFtWQ6izghsx8ZC3rweoh1cOAO+pQRt0T9pXMnEI1dHkw1TDx38jMczKzo/cygW+s5ZweBpYB4zJzVP3ZPDNf0Wmdl0VEdJreDphfH+eXmfkmYGvgTuBH6zgvSQUyxEnaFIdTDRFOAfaoP7sCv6MKMX8EFgBfj4hh9Q0Ar623PQ34TETsFZWdImL7etktwLsion990f++66ljBNU1aIsjYgzV9WEAZOYC4BfAKfUNEAMj4vWdtr0E2BP4JNU1cutyHtV1aR9hdS8cETEjIl5Z9/w9QzW8unLNjSNil4h4Q1SPYVla19yx3mPADhHRr1PdVwH/HhGbR0S/iHh5RHRuiy2AT9TnNIuq7a+IiC0j4tD62rhlwHNd1SOpbIY4SZviaOCMzHwoMx/t+FDdVPBuqp6wQ6guzH8ImEc1zEdmXkh17do5wLNUYWpMvd9P1tstrvdzyXrqOAkYAjxBdZfslWssfw9VsLqT6jqyYzsWZOYLwM+AicDF6zpIHaxuoOptO7/Toq2Ai6gC3Fzgt1TX261pEPD1us5HqULYcfWyC+vvRRHxp/r3e4HNgDuAp+pjbN1pf38AJtX7+1fg7Zm5iOq/7Z+m6pV7kioEf3Rd5yapPPHSyykkqe+JiOOBnTPzqPWu3ENExPuAD9ZDs5L6IB+SKalPq4dfP0DVWydJxWjZcGpE/DgiHo+I2zrNGxMRV0fEPfX36E7LvhgR99YPv3xLq+qSpA4R8SGqGwh+kZnXtbseSdoQLRtOrS8cfg74r8zcrZ73Tapb/b8eEV8ARmfm5yNiCtVdXx3PlPoV1dCGF+JKkiR1oWU9cfX/q31yjdmHUd3OT/19eKf559UP7vwLcC9VoJMkSVIXuvvu1C3ru7s67vLqeGjly6iGNDrMq+dJkiSpCz3lxoboYl6X47wRcQzVuxIZNmzYXpMn+xBySZLU8918881PZOb4Zu2vu0PcYxGxdWYuqN9n+Hg9fx7Vq3E6TKB+6viaMvNU4FSAadOm5ezZs1tZryRJUlNERKPvh25Idw+nXsrqdyoeDfy80/wjI2JQREykenjlH7u5NkmSpGK0rCcuIs4F9gPGRcQ8qtfgfB24ICI+QPX09lkAmXl7RFxA9VTyFcA/eWeqJEnS2rUsxGXmO9eyaP+1rP+vVK+NkSRJ0nr0lBsbJElSL7J8+XLmzZvH0qVL211Ktxs8eDATJkxg4MCBLT2OIU6SJDXdvHnzGDFiBDvssAMRXT2EonfKTBYtWsS8efOYOHFiS4/V3Tc2SJKkPmDp0qWMHTu2TwU4gIhg7Nix3dIDaYiTJEkt0dcCXIfuOm9DnCRJ6pWGDx8OwPz583n729/e5mqazxAnSZJ6tW222YaLLrqopcdYsWJFS/ffFUOcJEnq1R544AF22203AM4880ze9ra3ccABBzBp0iQ+97nP/XW9q666ile/+tXsueeezJo1i+eeew6Ar371q+y9997stttuHHPMMWRWbwbdb7/9OO6449h333353ve+1+3n5d2pkiSptY49Fm65pbn73GMPOOmkjdr0lltuYc6cOQwaNIhddtmFj3/84wwZMoQTTzyRX/3qVwwbNoxvfOMbfOc73+H444/nYx/7GMcffzwA73nPe7jssss45JBDAFi8eDG//e1vm3ZaG8IQJ0mS+pT999+fkSNHAjBlyhQefPBBFi9ezB133MFrX/taAF588UVe/epXA3DttdfyzW9+kyVLlvDkk0/yile84q8h7ogjjmjPSWCIkyRJrbaRPWatMmjQoL/+7t+/PytWrCAzedOb3sS55577knWXLl3KRz/6UWbPns22227LCSec8JLHhwwbNqzb6l6T18RJkqQ+b/r06Vx//fXce++9ACxZsoS77777r4Ft3LhxPPfccy2/QWJD2BMnSZL6vPHjx3PmmWfyzne+k2XLlgFw4oknsvPOO/OhD32IV77yleywww7svffeba50tei4w6JE06ZNy9mzZ7e7DEmStIa5c+ey6667truMtunq/CPi5syc1qxjOJwqSZJUIEOcJElSgQxxkiRJBTLESZIkFcgQJ0mSVCBDnCRJUoEMcZIkqU8YPnx4l/PvvPNO9thjD6ZOncp9993XzVVtPEOcJEnq9TKTVatWdbnskksu4bDDDmPOnDm8/OUv7+bKNp5vbJAkSb3SAw88wIEHHsiMGTO44YYbeOGFF/j0pz/Ntddey+jRoznvvPO46aabOOmkk+jfvz/XXXcd1157bbvLbpghTpIktdSxVx7LLY/e0tR97rHVHpx0wEnrXe+uu+7ijDPO4JRTTiEi2HPPPfn3f/93vvrVr/KVr3yFk08+mQ9/+MMMHz6cz3zmM02tsdUcTpUkSb3W9ttvz/Tp0wHo168fRxxxBABHHXUUv//979tZ2iazJ06SJLVUIz1mrTJs2LC1LouIbqyk+eyJkyRJfcKqVau46KKLADjnnHPYZ5992lzRprEnTpIk9QnDhg3j9ttvZ6+99mLkyJGcf/757S5pk0RmtruGjTZt2rScPXt2u8uQJElrmDt3Lrvuumu7y2ibrs4/Im7OzGnNOobDqZIkSQUyxEmSJBXIECdJklQgQ5wkSWqJkq+73xTddd6GOEmS1HSDBw9m0aJFfS7IZSaLFi1i8ODBLT+WjxiRJElNN2HCBObNm8fChQvbXUq3Gzx4MBMmTGj5cQxxkiSp6QYOHMjEiRPbXUav5nCqJElSgQxxkiRJBTLESZIkFcgQJ0mSVCBDnCRJUoEMcZIkSQUyxEmSJBXIECdJklQgQ5wkSVKBDHGSJEkFMsRJkiQVyBAnSZJUIEOcJElSgQxxkiRJBTLESZIkFcgQJ0mSVCBDnCRJUoEMcZIkSQUyxEmSJBXIECdJklQgQ5wkSVKBDHGSJEkFMsRJkiQVyBAnSZJUIEOcJElSgQxxkiRJBTLESZIkFcgQJ0mSVCBDnCRJUoEMcZIkSQUyxEmSJBXIECdJklQgQ5wkSVKBDHGSJEkFMsRJkiQVyBAnSZJUIEOcJElSgQxxkiRJBTLESZIkFcgQJ0mSVCBDnCRJUoEMcZIkSQUyxEmSJBWoLSEuIj4VEbdHxG0RcW5EDI6IMRFxdUTcU3+PbkdtkiRJJej2EBcRLwM+AUzLzN2A/sCRwBeAazJzEnBNPS1JkqQutGs4dQAwJCIGAEOB+cBhwFn18rOAw9tUmyRJUo/X7SEuMx8Bvg08BCwAns7Mq4AtM3NBvc4CYIvurk2SJKkU7RhOHU3V6zYR2AYYFhFHbcD2x0TE7IiYvXDhwlaVKUmS1KO1Yzj1jcBfMnNhZi4HLgZeAzwWEVsD1N+Pd7VxZp6amdMyc9r48eO7rWhJkqSepB0h7iFgekQMjYgA9gfmApcCR9frHA38vA21SZIkFWFAdx8wM/8QERcBfwJWAHOAU4HhwAUR8QGqoDerkf2tWLWCmx65iYmjJ7LV8K1aVbYkSVKP0pa7UzPzy5k5OTN3y8z3ZOayzFyUmftn5qT6+8lG9vXQ0w/xmh+/hp/d8bNWly1JktRjFP/GhomjJrLdyO249oFr212KJElStyk+xEUEM3aYwW8e+A2rclW7y5EkSeoWxYc4gBk7zGDRC4u47fHb2l2KJElSt+gdIW7iDACu/YtDqpIkqW/oFSFuu5HbsePoHb0uTpIk9Rm9IsRBNaT62wd/y8pVK9tdiiRJUsv1qhC3eOlibn3s1naXIkmS1HK9J8TV18X95oHftLcQSZKkbtBrQtw2I7Zh57E7e12cJEnqE3pNiINqSPW6B69jxaoV7S5FkiSppXpdiHtm2TPMWTCn3aVIkiS1VK8KcfvusC+AQ6qSJKnX61UhbqvhW7HruF0NcZIkqdfrVSEOqiHV3z34O5avXN7uUiRJklqm94W4iTN4fvnzzJ4/u92lSJIktUyvC3H77bAf4HVxkiSpd+t1IW7c0HG8cotXGuIkSVKv1utCHFTXxV3/0PUsW7Gs3aVIkiS1RO8McRNn8MKKF/jjI39sdymSJEkt0StD3L7b70sQDqlKkqReq1eGuNFDRrPHVnsY4iRJUq/VK0McVNfF3fDwDSxdsbTdpUiSJDVd7w1xE2ewbOUybpx3Y7tLkSRJarpeG+Jet93r6Bf9uPYvDqlKkqTep9eGuJGDR7LX1nt5XZwkSeqVem2Ig+q6uBvn3ciS5UvaXYokSVJT9e4QN3EGy1ct5/qHrm93KZIkSU3Vq0Pc67Z7HUMGDOHSuy5tdymSJElN1atD3LDNhnHgpAP52dyfsSpXtbscSZKkpunVIQ5g1pRZLHhugUOqkiSpV+n1Ie7gnQ9m8IDBXHjHhe0uRZIkqWl6fYgbvtlwDtzJIVVJktS79PoQB9WQ6vxn5/PfD/93u0uRJElqij4R4g7e+WAG9R/Ehbc7pCpJknqHPhHiRgwawYGTDuSiuRc5pCpJknqFPhHiYPWQ6g0P39DuUiRJkjZZnwlxh+x8SDWk6l2qkiSpF+gzIW7EoBEcsNMBXHSHQ6qSJKl8fSbEQTWk+sizj3DjvBvbXYokSdIm6VMh7pBdDvEuVUmS1Cv0qRC3+aDNectOb/EuVUmSVLw+FeIA3r7r25n3zDz+MO8P7S5FkiRpo/W5EHfoLoeyWf/NvEtVkiQVrc+FuJGDR/Lml7/Zu1QlSVLR+lyIg+ou1YefeZibHrmp3aVIkiRtlD4Z4g7d5VAG9hvokKokSSpWnwxxowaP+uuQama2uxxJkqQN1idDHFRDqg8+/SA3zXdIVZIklafPhrjDJh9WDan64F9JklSgPhviRg0exZte/iYuvONCh1QlSVJx+myIg9VDqrPnz253KZIkSRukT4e4w3Y5zLtUJUlSkfp0iBs9ZDRv3PGNDqlKkqTi9OkQB9WQ6gOLH+DmBTe3uxRJkqSG9fkQd9jkwxjQb4B3qUqSpKL0+RA3ZsgYh1QlSVJxekeImz8fNiGAzZoyi78s/gt/WvCnJhYlSZLUOuWHuDvugMmT4T//c6N3cfjkw6shVe9SlSRJhSg/xE2eDPvsA8ceC7M37nlvY4aMYf+J+zukKkmSilF+iOvXD37yE9hyS5g1C556aqN28/Ypb+f+p+5nzqNzmlygJElS85Uf4gDGjoULLoBHHoGjj4ZVqzZ4F4dPPpz+0d+7VCVJUhF6R4gDmD4dvv1t+H//r/reQOOGjuMNE9/gkKokSSpC7wlxAB//eDWketxxcN11G7z5rCmzuO+p+7jl0VtaUJwkSVLz9K4QFwGnnQY77ghHHgmPPbZBm79117dWQ6repSpJknq43hXiADbfHC66qLrB4V3vgpUrG9503NBxzJg4wyFVSZLU4/W+EAfwd38Hp5wCv/41nHDCBm06a8os7n3yXm597NbW1CZJktQEvTPEAbz//dXnxBPh+usb3uytk9/qXaqSJKnH670hDuDkk2HYMDjnnIY3GT9sPPvtsJ9DqpIkqUfr3SFu6FB44xvh8ss36N2qs6bM4p4n7+EX9/6ihcVJkiRtvN4d4gBmzoQHH4S5cxve5B2veAc7jdmJg885mE9d+SmWLF/SwgIlSZI2XN8IcQBXXNHwJqOHjGbOP87ho3t/lJP+cBJTfziVGx6+oUUFSpIkbbjeH+ImTKjuVr388g3abPhmwzl55slc895rWLZiGfucsQ+fu/pzLF2xtEWFSpIkNa73hzioeuN+/3t4+ukN3vQNE9/Anz/yZz449YN867+/xdQfTuWPj/yxBUVKkiQ1rm+EuIMOghUr4OqrN2rzzQdtzg8P+SG/POqXPPfic7z69Fdz3DXHsWzFsiYXKkmS1Ji+EeKmT4fRozfouriuvPnlb+a2j9zG+3Z/H1/7/dfY69S9uHn+zU0qUpIkqXF9I8QNGABveUsV4lat2qRdjRw8ktMPO53L33U5Ty19ir8/7e85/trjeXHli00qVpIkaf36RoiD6rq4xx6DOXOas7tJM7ntI7fx7r97N/9y3b/wqh+9ilsf9VVdkiSpe/SdEHfAARCxyUOqnY0eMpqzDj+Lnx/5cx597lGm/Wga//Lbf2H5yuVNO4YkSVJX2hLiImJURFwUEXdGxNyIeHVEjImIqyPinvp7dFMPOn48vOpVG/yokUYcusuh3P7R23nHK97B8b85numnT+e2x29r+nEkSZI6tKsn7nvAlZk5GdgdmAt8AbgmMycB19TTzXXQQfDHP8LChU3f9dihYzn7bWdz0ayLePjph9nr1L342u++xopVK5p+LEmSpG4PcRGxOfB64HSAzHwxMxcDhwFn1audBRze9IPPnFm9Q/XKK5u+6w7/MOUfuP2jt3PoLody3K+P4zWnv4a5Cxt/5ZckSVIj2tETtyOwEDgjIuZExGkRMQzYMjMXANTfWzT9yFOnwpZbNvW6uK6MHzaeC2ddyPlvP5/7n7qfqT+cyreu/xYrV61s6XElSVLf0Y4QNwDYE/hBZk4FnmcDhk4j4piImB0Rsxdu6LBov35Vb9yVV1YP/22xd7ziHdz+0ds5cNKBfO5Xn+N1Z7yOu564q+XHlSRJvV87Qtw8YF5m/qGevogq1D0WEVsD1N+Pd7VxZp6amdMyc9r48eM3/OgzZ8LixXDjjRtV/IbacviWXPyOi/npW3/KnU/cyR4/3IPv3vBdnysnSZI2SWRm9x804nfABzPzrog4ARhWL1qUmV+PiC8AYzLzc+vaz7Rp03L27NkbdvCnn4Zx4+Czn4V/+7eNqH7jzX92Pv942T9y2d2XATBq8CjGDR3H+KHjGT9sPOOHju96uv49bLNh6zmCJEnqqSLi5syc1rT9tSnE7QGcBmwG3A+8n6pX8AJgO+AhYFZmPrmu/WxUiAOYMQOefBJu7f6H82Yml951Kbc+disLn1/IwiULeWLJEyxcspCFz1e/l6/q+jlzQwYMeUmoGz9sPOOGrDHdKQSOGjyKftF3HgUoSVJP1itCXLNsdIj79rernriHH4YJE5pf2CbITJ5Z9sxfQ91fQ179u3PY6/j9/PLnu9xX/+jPuKHjXhr81ujd6xz8xg0dx8D+A7v5jCVJ6huaHeIGNGtHRZk5swpxV1wBxxzT7mpeIiIYOXgkIwePZKcxOzW0zQvLX3hJqOsq+D2x5An+/NifWbhkIU++sPYOTod4JUkqQ9/sicuEiRNhjz3gkkuaX1gPt2LVCp584cnVIW+Nnr01h3gXLlm41ocWDxkw5G+GcdcV/BzilST1VfbENUNE9faGs86CZctg0KB2V9StBvQbwBbDtmCLYY09ii8zeXrZ0y/t3esc/DpN37XoLod4JUnqBn0zxEE1pHrKKXDddfCmN7W7mh4tIhg1eBSjBo/qMUO86wp+DvFKkvqCvhviZsyAwYOr6+IMcU03ZOAQth25LduO3Lah9VesWsGiJYvWG/weWPwANz1y00YN8Q4bOIzN+m/W9M+AfgOIiGY2nyRJ69V3Q9zQoVWQu/xy+O53211Nnzeg3wC2HL4lWw7fsqH1mznE2wybHAb7bdr2A/sPbGi9/tHfwClJvUTfDXFQXRf3sY/BPffApEntrkYbYGOGeDOTlbmSF1e+uEmf5SuXr3+9VWtf9tyLzzV0nJa0G9GS3si1fSaPm8y0bZp2Da8kqZO+HeIOPLD6vuIK+OQn21uLWi4iGBADGNBvAEMHDm13OeuUmaxYtWKTA2dDny4C5/KVy1m6YinPLHtm/aF2LQ+nBvjY3h8zxElSi/TtELfjjjB5siFOPU5EMLD/QAb2H8gwevaNGpnJ8lVd906O2GxEu8uTpF6rb4c4qIZU/+M/4LnnYPjwdlcjFSdi9RCtJKn7+NTVmTPhxRfh179udyWSJEkNM8Ttsw+MGFHdpSpJklQIQ9xmm1XPibviiup1XJIkSQUwxEF1Xdy8eXDbbe2uRJIkqSGGOIADDqi+HVKVJEmFMMQBbLMNTJ1aDalKkiQVwEeMdDjoIPja1+Cpp2D06HZXow2RCU8/DU88AYsWVZ+O353n9e9f3cSy+eaNfY8YAYMGtfvsJEnqkiGuw8yZcOKJcNVVcMQR7a6m71q5sgrSawawrkJZ598rV3a9v/79YcwYGDsWVq2CZ5+FZ56B5xt8j+pmm21Y8DMQSpK6SUMhLiJ+BvwY+EVmrmptSW3yqldV/9BffrkhrlmWL4cnn1x3GFtz2VNPrf0u4YEDYdy46u80bhxMmbJ6umPemt+bbw79urhqYOXK6gHPHaGuke+O3wsXwn33GQglSW3VaE/cD4D3A9+PiAuBMzPzztaV1Qb9+1c3OPziF1WPTVf/8PdlS5d23QO2rh6yp59e+/4GD65CVkfg2m67dYexsWOrN2pENOd8+veHkSOrz6bamEDY8d2uQNj5t4FQkorUUIjLzF8Bv4qIkcA7gasj4mHgR8BPM3Ptb8AuyUEHwdlnw+zZVc9cb5QJS5asP4ytuWxd4WL48JcGrp12WncYGzsWhvbsF9BvkJ4SCB9/vAqEHdMGQknq1Rq+Ji4ixgJHAe8B5gBnA/sARwP7taK4bvfmN1c9cJdfXkaIy6z+sd6QMLZoUdWrtjajRq0OXFttBbvttvYeso6P/3A3T28LhEcdBZ/97KafiyTpbzR6TdzFwGTgJ8AhmbmgXnR+RMxuVXHdbuxYmD69etTIV77SvcdetQoWL96wHrJFi6rrzroSUV3Q3xG4tt8e9txz3T1kY8bAAO916TVaEQjXdp3g2r5HjNj0Y0uSutTov9gnZ2aXb4jPzGlNrKf9DjoIvvQlePTRqieqGTLhkUfgjjtg7ly4806YP/+lQe3JJ6sg15X+/V8auHbeef3DlaNGVdtJzdDMQChJaopGQ9yuEfGnzFwMEBGjgXdm5imtK61NZs6sQtyVV8L73rdh265cCfffXwW1js8dd1Sh7dlnV683atTqC/l3223dYazjDstmXdAvSZJ6hcgGXvoeEbdk5h5rzJuTmVNbVlkDpk2blrNnN3k0NxMmTIDXvAYuvLDrdZYtg7vvXh3SOgLb3XdXyzpssw3suuvqz5Qp1fcWWxjKJEnqYyLi5maOYDbaE9cvIiLrxBcR/YHNmlVEjxJR9cZdcEH1zLJ7731pUJs7t7rgu2PoMwImTqzC2VvesjqoTZ5c9bhJkiS1QKMh7pfABRHxn0ACHwaubFlV7XbQQXDaadWF/h0GDoRJk2D33eHII1f3ru2yCwwZ0r5aJUlSn9RoiPs88I/AR4AArgJOa1VRbXfggfD5z1fXonX0rO24YxXkJEmSeoCGronrqVpyTZwkSVILtOWauIiYBHwNmAIM7pifmTs2qxBJkiQ1rtEXhJ5B9f7UFcAM4L+oHvwrSZKkNmg0xA3JzGuohl8fzMwTgDe0rixJkiStS6M3NiyNiH7APRHxMeARYIvWlSVJkqR1abQn7lhgKPAJYC/gKKoX30uSJKkN1tsTVz/Y9x2Z+VngOeD9La9KkiRJ67TenrjMXAnsFeF7oiRJknqKRq+JmwP8PCIuBJ7vmJmZF7ekKkmSJK1ToyFuDLCIl96RmoAhTpIkqQ0aCnGZ6XVwkiRJPUijb2w4g6rn7SUy8381vSJJkiStV4wGD70AAA/aSURBVKPDqZd1+j0YeCswv/nlSJIkqRGNDqf+rPN0RJwL/KolFUmSJGm9Gn3Y75omAds1sxBJkiQ1rtFr4p7lpdfEPQp8viUVSZIkab0aHU4d0epCJEmS1LiGhlMj4q0RMbLT9KiIOLx1ZUmSJGldGr0m7suZ+XTHRGYuBr7cmpIkSZK0Po2GuK7Wa/TxJJIkSWqyRkPc7Ij4TkS8PCJ2jIjvAje3sjBJkiStXaMh7uPAi8D5wAXAC8A/taooSZIkrVujd6c+D3yhxbVIkiSpQY3enXp1RIzqND06In7ZurIkSZK0Lo0Op46r70gFIDOfArZoTUmSJElan0ZD3KqI+OtrtiJiB176BgdJkiR1o0YfE/Il4PcR8dt6+vXAMa0pSZIkSevT6I0NV0bENKrgdgvwc6o7VCVJktQGDYW4iPgg8ElgAlWImw7cALyhdaVJkiRpbRq9Ju6TwN7Ag5k5A5gKLGxZVZIkSVqnRkPc0sxcChARgzLzTmCX1pUlSZKkdWn0xoZ59XPiLgGujoingPmtK0uSJEnr0uiNDW+tf54QEdcCI4ErW1aVJEmS1qnRnri/yszfrn8tSZIktVKj18RJkiSpBzHESZIkFcgQJ0mSVCBDnCRJUoEMcZIkSQUyxEmSJBXIECdJklQgQ5wkSVKBDHGSJEkFMsRJkiQVyBAnSZJUIEOcJElSgQxxkiRJBTLESZIkFcgQJ0mSVKC2hbiI6B8RcyLisnp6TERcHRH31N+j21WbJElST9fOnrhPAnM7TX8BuCYzJwHX1NOSJEnqQltCXERMAA4CTus0+zDgrPr3WcDh3V2XJElSKdrVE3cS8DlgVad5W2bmAoD6e4t2FCZJklSCbg9xEXEw8Hhm3ryR2x8TEbMjYvbChQubXJ0kSVIZ2tET91rg0Ih4ADgPeENE/BR4LCK2Bqi/H+9q48w8NTOnZea08ePHd1fNkiRJPUq3h7jM/GJmTsjMHYAjgV9n5lHApcDR9WpHAz/v7tokSZJK0ZOeE/d14E0RcQ/wpnpakiRJXRjQzoNn5m+A39S/FwH7t7MeSZKkUvSknjhJkiQ1yBAnSZJUIEOcJElSgQxxkiRJBTLESZIkFcgQJ0mSVCBDnCRJUoEMcZIkSQUyxEmSJBXIECdJklQgQ5wkSVKBDHGSJEkFMsRJkiQVyBAnSZJUIEOcJElSgQxxkiRJBTLESZIkFcgQJ0mSVCBDnCRJUoEMcZIkSQUyxEmSJBXIECdJklQgQ5wkSVKBDHGSJEkFMsRJkiQVyBAnSZJUIEOcJElSgQxxkiRJBTLESZIkFcgQJ0mSVCBDnCRJUoEMcZIkSQUyxEmSJBXIECdJklQgQ5wkSVKBDHGSJEkFMsRJkiQVyBAnSZJUIEOcJElSgQxxkiRJBTLESZIkFcgQJ0mSVCBDnCRJUoEMcZIkSQUyxEmSJBXIECdJklQgQ5wkSVKBDHGSJEkFMsRJkiQVyBAnSZJUIEOcJElSgQxxkiRJBTLESZIkFcgQJ0mSVCBDnCRJUoEMcZIkSQUyxEmSJBXIECdJklQgQ5wkSVKBDHGSJEkFMsRJkiQVyBAnSZJUIEOcJElSgQxxkiRJBTLESZIkFcgQJ0mSVCBDnCRJUoEMcZIkSQUyxEmSJBXIECdJklQgQ5wkSVKBDHGSJEkFMsRJkiQVyBAnSZJUIEOcJElSgQxxkiRJBTLESZIkFcgQJ0mSVKBuD3ERsW1EXBsRcyPi9oj4ZD1/TERcHRH31N+ju7s2SZKkUrSjJ24F8OnM3BWYDvxTREwBvgBck5mTgGvqaUmSJHWh20NcZi7IzD/Vv58F5gIvAw4DzqpXOws4vLtrkyRJKkVbr4mLiB2AqcAfgC0zcwFUQQ/Yon2VSZIk9WxtC3ERMRz4GXBsZj6zAdsdExGzI2L2woULW1egJElSD9aWEBcRA6kC3NmZeXE9+7GI2LpevjXweFfbZuapmTktM6eNHz++ewqWJEnqYdpxd2oApwNzM/M7nRZdChxd/z4a+Hl31yZJklSKAW045muB9wD/ExG31POOA74OXBARHwAeAma1oTZJkqQidHuIy8zfA7GWxft3Zy2SJEml8o0NkiRJBTLESZIkFcgQJ0mSVCBDnCRJUoEMcZIkSQUyxEmSJBXIECdJklQgQ5wkSVKBDHGSJEkFMsRJkiQVyBAnSZJUIEOcJElSgQxxkiRJBTLESZIkFcgQJ0mSVCBDnCRJUoEMcZIkSQUyxEmSJBXIECdJklQgQ5wkSVKBDHGSJEkFMsRJkiQVyBAnSZJUIEOcJElSgQxxkiRJBTLESZIkFcgQJ0mSVCBDnCRJUoEMcZIkSQUyxEmSJBXIECdJklQgQ5wkSVKBDHGSJEkFMsRJkiQVyBAnSZJUIEOcJElSgQxxkiRJBTLESZIkFcgQJ0mSVCBDnCRJUoEMcZIkSQUyxEmSJBXIECdJklQgQ5wkSVKBDHGSJEkFMsRJkiQVyBAnSZJUIEOcJElSgQxxkiRJBTLESZIkFcgQJ0mSVCBDnCRJUoEMcZIkSQUyxEmSJBXIECdJklQgQ5wkSVKBDHGSJEkFMsRJkiQVyBAnSZJUIEOcJElSgQxxkiRJBTLESZIkFcgQJ0mSVCBDnCRJUoEMcZIkSQUyxEmSJBXIECdJklQgQ5wkSVKBDHGSJEkFMsRJkiQVyBAnSZJUIEOcJElSgQxxkiRJBTLESZIkFcgQJ0mSVCBDnCRJUoEMcZIkSQUyxEmSJBWox4W4iDggIu6KiHsj4gvtrkeSJKkn6lEhLiL6A/8HOBCYArwzIqa0typJkqSep0eFOOBVwL2ZeX9mvgicBxzW5pokSZJ6nJ4W4l4GPNxpel49T5IkSZ0MaHcBa4gu5uVLVog4BjimnlwWEbe1vCp1Ng54ot1F9DG2efezzbufbd79bPPut0szd9bTQtw8YNtO0xOA+Z1XyMxTgVMBImJ2Zk7rvvJkm3c/27z72ebdzzbvfrZ594uI2c3cX08bTr0JmBQREyNiM+BI4NI21yRJktTj9KieuMxcEREfA34J9Ad+nJm3t7ksSZKkHqdHhTiAzLwCuKLB1U9tZS3qkm3e/Wzz7mebdz/bvPvZ5t2vqW0embn+tSRJktSj9LRr4iRJktSAYkOcr+dqjYjYNiKujYi5EXF7RHyynj8mIq6OiHvq79Gdtvli/Xe4KyLe0r7qyxUR/SNiTkRcVk/b3i0WEaMi4qKIuLP+3/urbffWiYhP1f9NuS0izo2IwbZ380XEjyPi8c6P39qYdo6IvSLif+pl34+Irh4BJtba5t+q/9vy54j4vxExqtOyprV5kSHO13O11Arg05m5KzAd+Ke6bb8AXJOZk4Br6mnqZUcCrwAOAE6p/z7aMJ8E5naatr1b73vAlZk5Gdidqv1t9xaIiJcBnwCmZeZuVDeuHYnt3QpnUrVZZxvTzj+geibrpPqz5j612pn8bftcDeyWmX8H3A18EZrf5kWGOHw9V8tk5oLM/FP9+1mqf9heRtW+Z9WrnQUcXv8+DDgvM5dl5l+Ae6n+PmpQREwADgJO6zTb9m6hiNgceD1wOkBmvpiZi7HdW2kAMCQiBgBDqZ4Bans3WWZeBzy5xuwNaueI2BrYPDNvyOrC+f/qtI3W0FWbZ+ZVmbminryR6rm30OQ2LzXE+XqubhAROwBTgT8AW2bmAqiCHrBFvZp/i013EvA5YFWnebZ3a+0ILATOqIexT4uIYdjuLZGZjwDfBh4CFgBPZ+ZV2N7dZUPb+WX17zXna+P8L+AX9e+mtnmpIW69r+fSpomI4cDPgGMz85l1rdrFPP8WDYqIg4HHM/PmRjfpYp7tveEGAHsCP8jMqcDz1ENMa2G7b4L6GqzDgInANsCwiDhqXZt0Mc/2br61tbPt3yQR8SWqy5TO7pjVxWob3ealhrj1vp5LGy8iBlIFuLMz8+J69mN1dy/19+P1fP8Wm+a1wKER8QDVZQFviIifYnu32jxgXmb+oZ6+iCrU2e6t8UbgL5m5MDOXAxcDr8H27i4b2s7zWD3813m+NkBEHA0cDLw7Vz/PraltXmqI8/VcLVLfDXM6MDczv9Np0aXA0fXvo4Gfd5p/ZEQMioiJVBdj/rG76i1dZn4xMydk5g5U/zv+dWYehe3dUpn5KPBwRHS8jHp/4A5s91Z5CJgeEUPr/8bsT3W9re3dPTaonesh12cjYnr993pvp23UgIg4APg8cGhmLum0qLltnplFfoCZVHd83Ad8qd319JYPsA9VF+6fgVvqz0xgLNVdTffU32M6bfOl+u9wF3Bgu8+h1A+wH3BZ/dv2bn177wHMrv+3fgkw2nZvaXt/BbgTuA34CTDI9m5JO59Ldd3hcqrenQ9sTDsD0+q/1X3AydQvB/DTcJvfS3XtW8e/o//Zijb3jQ2SJEkFKnU4VZIkqU8zxEmSJBXIECdJklQgQ5wkSVKBDHGSJEkFMsRJKkJEHBsRQ5u0r0MjYl1vaFjf9idExGeaUYskbSxDnKRSHEv14vRNlpmXZubXm7EvSWoXQ5ykHiUihkXE5RFxa0TcFhFHRMQnqN65eW1EXFuv9+aIuCEi/hQRF9bv+yUiHoiIb0TEH+vPTl0c430RcXL9+8yI+H5E/HdE3B8Rb19LXV+KiLsi4lfALp3mvzwiroyImyPidxExeV37jYitI+K6iLilPr/Xret8JGltDHGSepoDgPmZuXtm7gZcmZnfp3qP4IzMnBER44B/Bt6YmXtSvXnhf3faxzOZ+Sqqp56f1MAxt6Z6W8nBwN/00EXEXlSvRZsKvA3Yu9PiU4GPZ+ZewGeAU9az33cBv8zMPYDdgVsaOB9J+hsD2l2AJK3hf4BvR8Q3qF5D9rsu1pkOTAGur14zyGbADZ2Wn9vp+7sNHPOSzFwF3BERW3ax/HXA/836HYgRcWn9PZzqRe4X1nVA9Tqpde33JuDHETGwXn5LROy7nvORpL9hiJPUo2Tm3XXP10zgaxFxVWZ+dY3VArg6M9+5tt2s5ffaLFtj3+vbZ4d+wOK6V62h/WbmdRHxeuAg4CcR8S3gKdZ9PpL0NxxOldSjRMQ2wJLM/CnwbWDPetGzwIj6943Aazuud4uIoRGxc6fdHNHpuxk9WtcBb42IIRExAjgEIDOfAf4SEbPqOiIidl/XjiJie+DxzPwRcDrV+a3vfCTpb9gTJ6mneSXwrYhYBSwHPlLPPxX4RUQsqK+Lex9wbkR0DF/+M3B3/XtQRPyB6v+obnLvVmb+KSLOB24BHgQ6D/G+G/hBRPwzMBA4D7h1HbvbD/hsRCwHngPem5kL13M+kvQ3IrORkQZJKkNEPABMy8wn2l2LJLWSw6mSJEkFsidOkiSpQPbESZIkFcgQJ0mSVCBDnCRJUoEMcZIkSQUyxEmSJBXIECdJklSg/w9TI4jcnGGMcwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots(figsize=(10,8))\n",
    "ax.set(title='Accuracy vs steps', xlim=(0,1200), ylim=(0,100))\n",
    "df2=df\n",
    "y_rbf=df2.accuracy_rbf\n",
    "y_lin=df2.accuracy_linear\n",
    "print(\"max for rbf\\n\",df2[df2.accuracy_rbf==df2.accuracy_rbf.max()])\n",
    "print(\"max for linear\\n\",df2[df2.accuracy_linear==df2.accuracy_linear.max()])\n",
    "# print(df2[df2.recall==df2.recall.max()])\n",
    "\n",
    "ax.plot(df2.step, y_lin, 'r', label='linear')\n",
    "ax.plot(df2.step, y_rbf, 'g', label='rbf')\n",
    "plt.xlabel('step in dense')\n",
    "plt.ylabel('accuracy')\n",
    "ax.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>step</th>\n",
       "      <th>accuracy_linear</th>\n",
       "      <th>accuracy_rbf</th>\n",
       "      <th>normalized</th>\n",
       "      <th>scaler</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5</td>\n",
       "      <td>65.548654</td>\n",
       "      <td>78.937198</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20</td>\n",
       "      <td>63.446055</td>\n",
       "      <td>72.614063</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>50</td>\n",
       "      <td>52.413159</td>\n",
       "      <td>62.947627</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>100</td>\n",
       "      <td>53.275056</td>\n",
       "      <td>59.352810</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>300</td>\n",
       "      <td>54.897631</td>\n",
       "      <td>57.619048</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>500</td>\n",
       "      <td>53.771950</td>\n",
       "      <td>57.119086</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   step  accuracy_linear  accuracy_rbf  normalized  scaler\n",
       "0     5        65.548654     78.937198           1       1\n",
       "1    20        63.446055     72.614063           1       1\n",
       "2    50        52.413159     62.947627           1       1\n",
       "3   100        53.275056     59.352810           1       1\n",
       "4   300        54.897631     57.619048           1       1\n",
       "5   500        53.771950     57.119086           1       1"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('Different_densities_analysis.pkl','wb') as f:\n",
    "    cPickle.dump(df,f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## My Analysis on Pyramid level"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_pyramid_levels(pyramid):\n",
    "    step=5\n",
    "\n",
    "    today = datetime.now()\n",
    "    dt_string = today.strftime(\"%H:%M:%S\")\n",
    "    print(f\"{dt_string} started doing step={pyramid}\")\n",
    "\n",
    "    Train_descriptors = []\n",
    "    Train_label_per_descriptor = []\n",
    "    pyramid_levels = pyramid # level zero has 1 tile, level 1 has 4 tiles => 5 histograms that will be concatenated\n",
    "\n",
    "    for filename,labels in zip(train_images_filenames,train_labels):\n",
    "        ima=cv2.imread(filename)\n",
    "        gray=cv2.cvtColor(ima,cv2.COLOR_BGR2GRAY)\n",
    "        # break the image into pieces\n",
    "        image_cells = []\n",
    "        for i in range(pyramid_levels+1):\n",
    "\n",
    "            level_cells = get_pyramid_image_cells(gray,level=i)\n",
    "            image_cells = image_cells + level_cells\n",
    "            # compute descriptors for each tile\n",
    "        Train_descriptors_cell = []\n",
    "        for cell in image_cells:\n",
    "            des=compute_dense_sift(cell,SIFTdetector, step)\n",
    "            Train_descriptors_cell.append(des)\n",
    "\n",
    "        Train_descriptors.append(Train_descriptors_cell)\n",
    "        Train_label_per_descriptor.append(labels)\n",
    "\n",
    "    D=np.vstack([des for descriptors_cells in Train_descriptors for des in descriptors_cells])\n",
    "\n",
    "    #codebook\n",
    "\n",
    "    k=128\n",
    "    codebook = MiniBatchKMeans(n_clusters=k, verbose=False, batch_size=k * 20,compute_labels=False,reassignment_ratio=10**-4,random_state=42)\n",
    "    codebook.fit(D)\n",
    "\n",
    "    #creating visual words for Training\n",
    "\n",
    "    visual_words=np.zeros((len(Train_descriptors),k*21),dtype=np.float32)\n",
    "    for idx,cells in enumerate(Train_descriptors):\n",
    "\n",
    "        image_histograms = []\n",
    "        for id2,image_cell in enumerate(cells):\n",
    "            cell_words = codebook.predict(image_cell)\n",
    "            image_histograms.append(normalize(np.bincount(cell_words,minlength=k).reshape(1,-1)))\n",
    "            # concatenated histogram has k*(1+2**(2*pyramid_levels)) bins\n",
    "            # normalize(np.bincount(words,minlength=k).reshape(1,-1))\n",
    "\n",
    "        concatenated_histogram=image_histograms[0].copy()\n",
    "        for cell_idx in range(1,len(image_histograms)):\n",
    "            # concatenate histograms\n",
    "            concatenated_histogram = np.concatenate((concatenated_histogram,image_histograms[cell_idx]))\n",
    "        visual_words[idx,:]=concatenated_histogram.flatten()\n",
    "        \n",
    "    #PCA\n",
    "    pca = PCA(n_components=127)\n",
    "    VWpca = pca.fit_transform(visual_words)\n",
    "\n",
    "    #SVN kernels\n",
    "\n",
    "    scaler = StandardScaler()\n",
    "    scaler.fit_transform(VWpca)\n",
    "\n",
    "    lin_clf = svm.LinearSVC(max_iter=2000, C=10)\n",
    "    lin_clf.fit(VWpca, train_labels)\n",
    "\n",
    "    rbf_svc = svm.SVC(kernel='rbf', C=1000, gamma=0.001)\n",
    "    rbf_svc.fit(VWpca, train_labels)\n",
    "\n",
    "    today = datetime.now()\n",
    "    dt_string = today.strftime(\"%H:%M:%S\")\n",
    "    print(f\"{dt_string} started doing test step={pyramid}\")\n",
    "\n",
    "    #creating SIFT descriptor\n",
    "\n",
    "    Test_descriptors = []\n",
    "    Test_label_per_descriptor = []\n",
    "    # pyramid_levels = 1 # level zero has 1 tile, level 1 has 4 tiles => 5 histograms that will be concatenated\n",
    "    for filename,labels in zip(test_images_filenames,test_labels):\n",
    "        ima=cv2.imread(filename)\n",
    "        gray=cv2.cvtColor(ima,cv2.COLOR_BGR2GRAY)\n",
    "        # break the image into pieces\n",
    "        image_cells = []\n",
    "        for i in range(pyramid_levels+1):\n",
    "\n",
    "            level_cells = get_pyramid_image_cells(gray,level=i)\n",
    "            image_cells = image_cells + level_cells\n",
    "            # compute descriptors for each tile\n",
    "        Test_descriptors_cell = []\n",
    "        for cell in image_cells:\n",
    "            des=compute_dense_sift(cell,SIFTdetector, step)\n",
    "            Test_descriptors_cell.append(des)\n",
    "\n",
    "        Test_descriptors.append(Test_descriptors_cell)\n",
    "        Test_label_per_descriptor.append(labels)\n",
    "\n",
    "    #creating visual tests\n",
    "\n",
    "    visual_words_test=np.zeros((len(Test_descriptors),k*21),dtype=np.float32)\n",
    "    for idx,cells in enumerate(Test_descriptors):\n",
    "\n",
    "        image_histograms = []\n",
    "        for id2,image_cell in enumerate(cells):\n",
    "            cell_words = codebook.predict(image_cell)\n",
    "            image_histograms.append(normalize(np.bincount(cell_words,minlength=k).reshape(1,-1)))\n",
    "            # concatenated histogram has k*(1+2**(2*pyramid_levels)) bins\n",
    "            # normalize(np.bincount(words,minlength=k).reshape(1,-1))\n",
    "\n",
    "        concatenated_histogram=image_histograms[0].copy()\n",
    "        for cell_idx in range(1,len(image_histograms)):\n",
    "            # concatenate histograms\n",
    "            concatenated_histogram = np.concatenate((concatenated_histogram,image_histograms[cell_idx]))\n",
    "        visual_words_test[idx,:]=concatenated_histogram.flatten()\n",
    "\n",
    "\n",
    "    #scaler test\n",
    "    visual_words_test=scaler.transform(visual_words_test)\n",
    "    \n",
    "    #PCA\n",
    "    vwtestpca = pca.transform(visual_words_test)\n",
    "\n",
    "    #accuracy for test linear\n",
    "#     scores = cross_val_score(lin_clf, visual_words_test, test_labels, cv=5)\n",
    "    scores = cross_val_score(lin_clf, vwtestpca, test_labels, cv=5)\n",
    "    accuracy_linear=scores.mean()*100\n",
    "\n",
    "    print(\"linear=\",accuracy_linear)\n",
    "\n",
    "\n",
    "    #accuracy for test rbf\n",
    "    scores = cross_val_score(rbf_svc, visual_words_test, test_labels, cv=5)\n",
    "    accuracy_rbf=scores.mean()*100\n",
    "\n",
    "    print(\"rbf=\",accuracy_rbf)\n",
    "\n",
    "    return accuracy_linear,accuracy_rbf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "01:42:32 started doing step=2\n"
     ]
    }
   ],
   "source": [
    "piramid=[2]\n",
    "data_pyramid_levels=[]\n",
    "for level in piramid:\n",
    "    accuracy_linear,accuracy_rbf=create_pyramid_levels(level)\n",
    "    data_pyramid_levels.append([level,5,accuracy_linear,accuracy_rbf,1,1])\n",
    "\n",
    "df_level=pd.DataFrame(data_pyramid_levels, columns=['level','step','accuracy_linear','accuracy_rbf','normalized','scaler'])    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>level</th>\n",
       "      <th>step</th>\n",
       "      <th>accuracy_linear</th>\n",
       "      <th>accuracy_rbf</th>\n",
       "      <th>normalized</th>\n",
       "      <th>scaler</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>65.921325</td>\n",
       "      <td>77.201135</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   level  step  accuracy_linear  accuracy_rbf  normalized  scaler\n",
       "0      2     5        65.921325     77.201135           1       1"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_level"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('Different_pyramid_level_2.pkl','wb') as f:\n",
    "    cPickle.dump(df_level,f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}