Salary_Modeling_Master.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "toc": true
   },
   "source": [
    "<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
    "<div class=\"toc\"><ul class=\"toc-item\"><li><span><a href=\"#Modeling\" data-toc-modified-id=\"Modeling-1\"><span class=\"toc-item-num\">1&nbsp;&nbsp;</span>Modeling</a></span><ul class=\"toc-item\"><li><span><a href=\"#Import-Libraries-and-the-final-salary-data-set\" data-toc-modified-id=\"Import-Libraries-and-the-final-salary-data-set-1.1\"><span class=\"toc-item-num\">1.1&nbsp;&nbsp;</span>Import Libraries and the final salary data set</a></span><ul class=\"toc-item\"><li><span><a href=\"#Set-up-data-and-target\" data-toc-modified-id=\"Set-up-data-and-target-1.1.1\"><span class=\"toc-item-num\">1.1.1&nbsp;&nbsp;</span>Set up data and target</a></span><ul class=\"toc-item\"><li><span><a href=\"#Train-Test-Split\" data-toc-modified-id=\"Train-Test-Split-1.1.1.1\"><span class=\"toc-item-num\">1.1.1.1&nbsp;&nbsp;</span>Train-Test Split</a></span></li></ul></li></ul></li><li><span><a href=\"#Model-1:--Use-Word2Vec-Embedding-in-Classification-Models\" data-toc-modified-id=\"Model-1:--Use-Word2Vec-Embedding-in-Classification-Models-1.2\"><span class=\"toc-item-num\">1.2&nbsp;&nbsp;</span>Model 1:  Use Word2Vec Embedding in Classification Models</a></span><ul class=\"toc-item\"><li><span><a href=\"#Word2Vec-Model-from-scratch\" data-toc-modified-id=\"Word2Vec-Model-from-scratch-1.2.1\"><span class=\"toc-item-num\">1.2.1&nbsp;&nbsp;</span>Word2Vec Model from scratch</a></span></li><li><span><a href=\"#Use-Word2Vec-in-the-pipeline\" data-toc-modified-id=\"Use-Word2Vec-in-the-pipeline-1.2.2\"><span class=\"toc-item-num\">1.2.2&nbsp;&nbsp;</span>Use Word2Vec in the pipeline</a></span></li><li><span><a href=\"#Prediction-Probability-of-each-bin\" data-toc-modified-id=\"Prediction-Probability-of-each-bin-1.2.3\"><span class=\"toc-item-num\">1.2.3&nbsp;&nbsp;</span>Prediction Probability of each bin</a></span></li><li><span><a href=\"#Random-Forest-with-Grid-Search\" data-toc-modified-id=\"Random-Forest-with-Grid-Search-1.2.4\"><span class=\"toc-item-num\">1.2.4&nbsp;&nbsp;</span>Random Forest with Grid Search</a></span></li><li><span><a href=\"#SGD-with-Grid-Search\" data-toc-modified-id=\"SGD-with-Grid-Search-1.2.5\"><span class=\"toc-item-num\">1.2.5&nbsp;&nbsp;</span>SGD with Grid Search</a></span></li></ul></li><li><span><a href=\"#Model-2:-Count-vectorizer-+-TF-IDF-with-GridSearch\" data-toc-modified-id=\"Model-2:-Count-vectorizer-+-TF-IDF-with-GridSearch-1.3\"><span class=\"toc-item-num\">1.3&nbsp;&nbsp;</span>Model 2: Count vectorizer + TF-IDF with GridSearch</a></span><ul class=\"toc-item\"><li><span><a href=\"#Train-Test-Split\" data-toc-modified-id=\"Train-Test-Split-1.3.1\"><span class=\"toc-item-num\">1.3.1&nbsp;&nbsp;</span>Train-Test Split</a></span></li><li><span><a href=\"#SGD-Classifier-with-Grid-Search\" data-toc-modified-id=\"SGD-Classifier-with-Grid-Search-1.3.2\"><span class=\"toc-item-num\">1.3.2&nbsp;&nbsp;</span>SGD Classifier with Grid Search</a></span><ul class=\"toc-item\"><li><span><a href=\"#Confusion-Matrix\" data-toc-modified-id=\"Confusion-Matrix-1.3.2.1\"><span class=\"toc-item-num\">1.3.2.1&nbsp;&nbsp;</span>Confusion Matrix</a></span></li></ul></li><li><span><a href=\"#Random-Forest-Classifier-with-Grid-Search\" data-toc-modified-id=\"Random-Forest-Classifier-with-Grid-Search-1.3.3\"><span class=\"toc-item-num\">1.3.3&nbsp;&nbsp;</span>Random Forest Classifier with Grid Search</a></span></li></ul></li><li><span><a href=\"#Model-3:-Use-Pretrained-GloVe-Model\" data-toc-modified-id=\"Model-3:-Use-Pretrained-GloVe-Model-1.4\"><span class=\"toc-item-num\">1.4&nbsp;&nbsp;</span>Model 3: Use Pretrained GloVe Model</a></span><ul class=\"toc-item\"><li><ul class=\"toc-item\"><li><span><a href=\"#Getting-the-Total-Vocabulary\" data-toc-modified-id=\"Getting-the-Total-Vocabulary-1.4.0.1\"><span class=\"toc-item-num\">1.4.0.1&nbsp;&nbsp;</span>Getting the Total Vocabulary</a></span></li></ul></li><li><span><a href=\"#Random-Forest-with-Grid-Search\" data-toc-modified-id=\"Random-Forest-with-Grid-Search-1.4.1\"><span class=\"toc-item-num\">1.4.1&nbsp;&nbsp;</span>Random Forest with Grid Search</a></span></li><li><span><a href=\"#SGD-Classifier-in-the-pipeline\" data-toc-modified-id=\"SGD-Classifier-in-the-pipeline-1.4.2\"><span class=\"toc-item-num\">1.4.2&nbsp;&nbsp;</span>SGD Classifier in the pipeline</a></span></li></ul></li><li><span><a href=\"#Model-4:-Deep-Learning-Models-with-Word-Embeddings\" data-toc-modified-id=\"Model-4:-Deep-Learning-Models-with-Word-Embeddings-1.5\"><span class=\"toc-item-num\">1.5&nbsp;&nbsp;</span>Model 4: Deep Learning Models with Word Embeddings</a></span><ul class=\"toc-item\"><li><span><a href=\"#Word-Embedding\" data-toc-modified-id=\"Word-Embedding-1.5.1\"><span class=\"toc-item-num\">1.5.1&nbsp;&nbsp;</span>Word Embedding</a></span><ul class=\"toc-item\"><li><span><a href=\"#Train-Test-Split\" data-toc-modified-id=\"Train-Test-Split-1.5.1.1\"><span class=\"toc-item-num\">1.5.1.1&nbsp;&nbsp;</span>Train-Test Split</a></span></li></ul></li><li><span><a href=\"#Create-a-base-model\" data-toc-modified-id=\"Create-a-base-model-1.5.2\"><span class=\"toc-item-num\">1.5.2&nbsp;&nbsp;</span>Create a base model</a></span></li><li><span><a href=\"#Hyperparameter-tuning-on-a-regular-embedding-layer\" data-toc-modified-id=\"Hyperparameter-tuning-on-a-regular-embedding-layer-1.5.3\"><span class=\"toc-item-num\">1.5.3&nbsp;&nbsp;</span>Hyperparameter tuning on a regular embedding layer</a></span></li></ul></li><li><span><a href=\"#Model-5:-Use-pre-trained-Word2Vec-as-the-embedding-layer\" data-toc-modified-id=\"Model-5:-Use-pre-trained-Word2Vec-as-the-embedding-layer-1.6\"><span class=\"toc-item-num\">1.6&nbsp;&nbsp;</span>Model 5: Use pre-trained Word2Vec as the embedding layer</a></span><ul class=\"toc-item\"><li><ul class=\"toc-item\"><li><span><a href=\"#Tuning-the-model\" data-toc-modified-id=\"Tuning-the-model-1.6.0.1\"><span class=\"toc-item-num\">1.6.0.1&nbsp;&nbsp;</span>Tuning the model</a></span></li><li><span><a href=\"#Grid-Search\" data-toc-modified-id=\"Grid-Search-1.6.0.2\"><span class=\"toc-item-num\">1.6.0.2&nbsp;&nbsp;</span>Grid Search</a></span></li></ul></li><li><span><a href=\"#Try-Count-Vectorizer-and-TFIDF-in-Deep-Learning-Models-with-Grid-Search\" data-toc-modified-id=\"Try-Count-Vectorizer-and-TFIDF-in-Deep-Learning-Models-with-Grid-Search-1.6.1\"><span class=\"toc-item-num\">1.6.1&nbsp;&nbsp;</span>Try Count Vectorizer and TFIDF in Deep Learning Models with Grid Search</a></span><ul class=\"toc-item\"><li><span><a href=\"#Grid-Search\" data-toc-modified-id=\"Grid-Search-1.6.1.1\"><span class=\"toc-item-num\">1.6.1.1&nbsp;&nbsp;</span>Grid Search</a></span></li></ul></li></ul></li><li><span><a href=\"#Model-6:-Mixed-Input-Model-with-City,-Rate-Type-and-Cost-of-Living-Indexes\" data-toc-modified-id=\"Model-6:-Mixed-Input-Model-with-City,-Rate-Type-and-Cost-of-Living-Indexes-1.7\"><span class=\"toc-item-num\">1.7&nbsp;&nbsp;</span>Model 6: Mixed Input Model with City, Rate Type and Cost of Living Indexes</a></span><ul class=\"toc-item\"><li><ul class=\"toc-item\"><li><span><a href=\"#Explore-continuous-variables-and-categorical-variables\" data-toc-modified-id=\"Explore-continuous-variables-and-categorical-variables-1.7.0.1\"><span class=\"toc-item-num\">1.7.0.1&nbsp;&nbsp;</span>Explore continuous variables and categorical variables</a></span></li></ul></li><li><span><a href=\"#Put-them-together\" data-toc-modified-id=\"Put-them-together-1.7.1\"><span class=\"toc-item-num\">1.7.1&nbsp;&nbsp;</span>Put them together</a></span></li><li><span><a href=\"#Try-Word2Vec-Embedding-in-Mixed-Input-Model\" data-toc-modified-id=\"Try-Word2Vec-Embedding-in-Mixed-Input-Model-1.7.2\"><span class=\"toc-item-num\">1.7.2&nbsp;&nbsp;</span>Try Word2Vec Embedding in Mixed Input Model</a></span></li></ul></li><li><span><a href=\"#Model-7:-Use-Job-Title-along-with-City,-Cost-of-Living-Index-in-Classification-Models\" data-toc-modified-id=\"Model-7:-Use-Job-Title-along-with-City,-Cost-of-Living-Index-in-Classification-Models-1.8\"><span class=\"toc-item-num\">1.8&nbsp;&nbsp;</span>Model 7: Use Job Title along with City, Cost of Living Index in Classification Models</a></span><ul class=\"toc-item\"><li><span><a href=\"#Explore-Title\" data-toc-modified-id=\"Explore-Title-1.8.1\"><span class=\"toc-item-num\">1.8.1&nbsp;&nbsp;</span>Explore Title</a></span></li><li><span><a href=\"#Random-Forest-as-base-model\" data-toc-modified-id=\"Random-Forest-as-base-model-1.8.2\"><span class=\"toc-item-num\">1.8.2&nbsp;&nbsp;</span>Random Forest as base model</a></span></li><li><span><a href=\"#XGBoost\" data-toc-modified-id=\"XGBoost-1.8.3\"><span class=\"toc-item-num\">1.8.3&nbsp;&nbsp;</span>XGBoost</a></span></li></ul></li></ul></li><li><span><a href=\"#Conclusion\" data-toc-modified-id=\"Conclusion-2\"><span class=\"toc-item-num\">2&nbsp;&nbsp;</span>Conclusion</a></span></li></ul></div>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "LPrg-wLJde1r"
   },
   "source": [
    "# Modeling"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "dkTX4ZxFde1s"
   },
   "source": [
    "## Import Libraries and the final salary data set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "KgjYcXBGde1t"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n",
      "/Applications/anaconda3/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:526: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/Applications/anaconda3/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:527: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/Applications/anaconda3/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:528: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/Applications/anaconda3/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:529: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/Applications/anaconda3/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:530: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/Applications/anaconda3/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:535: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "\n",
    "#NLP\n",
    "from nltk import word_tokenize, FreqDist\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.collocations import *\n",
    "import string, re\n",
    "from gensim.models import Word2Vec\n",
    "from sklearn.manifold import TSNE\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "#Model Data Prep\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler, OneHotEncoder\n",
    "from sklearn.impute import SimpleImputer\n",
    "\n",
    "#Machine Learning\n",
    "from sklearn.metrics import accuracy_score, f1_score, mean_squared_error, r2_score\n",
    "from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.svm import SVR\n",
    "from sklearn.linear_model import LinearRegression, SGDClassifier\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "#Deep learning\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.layers import Input, Dense, LSTM, Embedding\n",
    "from keras.layers import Dropout, Activation, Bidirectional, GlobalMaxPool1D\n",
    "from keras.layers.convolutional import Conv1D, Conv2D\n",
    "from keras.layers.convolutional import MaxPooling1D, MaxPooling2D\n",
    "from keras.models import Sequential\n",
    "from keras import initializers, regularizers, constraints, optimizers, layers\n",
    "from keras.preprocessing import text, sequence\n",
    "from keras.layers import Flatten\n",
    "from keras.utils import to_categorical\n",
    "from keras.callbacks import EarlyStopping\n",
    "from keras import regularizers\n",
    "from keras.wrappers.scikit_learn import KerasClassifier #to use Keras in sklearn\n",
    "\n",
    "#Deep Learning - Mixed inputs\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "from keras.layers.core import Activation\n",
    "from keras.layers.core import Dropout\n",
    "from keras.layers.core import Dense\n",
    "from keras.layers import Flatten\n",
    "from keras.layers import Input\n",
    "from keras.models import Model\n",
    "from keras.layers import concatenate\n",
    "from keras.optimizers import Adam, RMSprop\n",
    "\n",
    "#Save Model\n",
    "import pickle\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from pprint import pprint\n",
    "from time import time\n",
    "import logging\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "np.random.seed(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 74,
     "resources": {
      "http://localhost:8080/nbextensions/google.colab/files.js": {
       "data": "Ly8gQ29weXJpZ2h0IDIwMTcgR29vZ2xlIExMQwovLwovLyBMaWNlbnNlZCB1bmRlciB0aGUgQXBhY2hlIExpY2Vuc2UsIFZlcnNpb24gMi4wICh0aGUgIkxpY2Vuc2UiKTsKLy8geW91IG1heSBub3QgdXNlIHRoaXMgZmlsZSBleGNlcHQgaW4gY29tcGxpYW5jZSB3aXRoIHRoZSBMaWNlbnNlLgovLyBZb3UgbWF5IG9idGFpbiBhIGNvcHkgb2YgdGhlIExpY2Vuc2UgYXQKLy8KLy8gICAgICBodHRwOi8vd3d3LmFwYWNoZS5vcmcvbGljZW5zZXMvTElDRU5TRS0yLjAKLy8KLy8gVW5sZXNzIHJlcXVpcmVkIGJ5IGFwcGxpY2FibGUgbGF3IG9yIGFncmVlZCB0byBpbiB3cml0aW5nLCBzb2Z0d2FyZQovLyBkaXN0cmlidXRlZCB1bmRlciB0aGUgTGljZW5zZSBpcyBkaXN0cmlidXRlZCBvbiBhbiAiQVMgSVMiIEJBU0lTLAovLyBXSVRIT1VUIFdBUlJBTlRJRVMgT1IgQ09ORElUSU9OUyBPRiBBTlkgS0lORCwgZWl0aGVyIGV4cHJlc3Mgb3IgaW1wbGllZC4KLy8gU2VlIHRoZSBMaWNlbnNlIGZvciB0aGUgc3BlY2lmaWMgbGFuZ3VhZ2UgZ292ZXJuaW5nIHBlcm1pc3Npb25zIGFuZAovLyBsaW1pdGF0aW9ucyB1bmRlciB0aGUgTGljZW5zZS4KCi8qKgogKiBAZmlsZW92ZXJ2aWV3IEhlbHBlcnMgZm9yIGdvb2dsZS5jb2xhYiBQeXRob24gbW9kdWxlLgogKi8KKGZ1bmN0aW9uKHNjb3BlKSB7CmZ1bmN0aW9uIHNwYW4odGV4dCwgc3R5bGVBdHRyaWJ1dGVzID0ge30pIHsKICBjb25zdCBlbGVtZW50ID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnc3BhbicpOwogIGVsZW1lbnQudGV4dENvbnRlbnQgPSB0ZXh0OwogIGZvciAoY29uc3Qga2V5IG9mIE9iamVjdC5rZXlzKHN0eWxlQXR0cmlidXRlcykpIHsKICAgIGVsZW1lbnQuc3R5bGVba2V5XSA9IHN0eWxlQXR0cmlidXRlc1trZXldOwogIH0KICByZXR1cm4gZWxlbWVudDsKfQoKLy8gTWF4IG51bWJlciBvZiBieXRlcyB3aGljaCB3aWxsIGJlIHVwbG9hZGVkIGF0IGEgdGltZS4KY29uc3QgTUFYX1BBWUxPQURfU0laRSA9IDEwMCAqIDEwMjQ7Ci8vIE1heCBhbW91bnQgb2YgdGltZSB0byBibG9jayB3YWl0aW5nIGZvciB0aGUgdXNlci4KY29uc3QgRklMRV9DSEFOR0VfVElNRU9VVF9NUyA9IDMwICogMTAwMDsKCmZ1bmN0aW9uIF91cGxvYWRGaWxlcyhpbnB1dElkLCBvdXRwdXRJZCkgewogIGNvbnN0IHN0ZXBzID0gdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKTsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIC8vIENhY2hlIHN0ZXBzIG9uIHRoZSBvdXRwdXRFbGVtZW50IHRvIG1ha2UgaXQgYXZhaWxhYmxlIGZvciB0aGUgbmV4dCBjYWxsCiAgLy8gdG8gdXBsb2FkRmlsZXNDb250aW51ZSBmcm9tIFB5dGhvbi4KICBvdXRwdXRFbGVtZW50LnN0ZXBzID0gc3RlcHM7CgogIHJldHVybiBfdXBsb2FkRmlsZXNDb250aW51ZShvdXRwdXRJZCk7Cn0KCi8vIFRoaXMgaXMgcm91Z2hseSBhbiBhc3luYyBnZW5lcmF0b3IgKG5vdCBzdXBwb3J0ZWQgaW4gdGhlIGJyb3dzZXIgeWV0KSwKLy8gd2hlcmUgdGhlcmUgYXJlIG11bHRpcGxlIGFzeW5jaHJvbm91cyBzdGVwcyBhbmQgdGhlIFB5dGhvbiBzaWRlIGlzIGdvaW5nCi8vIHRvIHBvbGwgZm9yIGNvbXBsZXRpb24gb2YgZWFjaCBzdGVwLgovLyBUaGlzIHVzZXMgYSBQcm9taXNlIHRvIGJsb2NrIHRoZSBweXRob24gc2lkZSBvbiBjb21wbGV0aW9uIG9mIGVhY2ggc3RlcCwKLy8gdGhlbiBwYXNzZXMgdGhlIHJlc3VsdCBvZiB0aGUgcHJldmlvdXMgc3RlcCBhcyB0aGUgaW5wdXQgdG8gdGhlIG5leHQgc3RlcC4KZnVuY3Rpb24gX3VwbG9hZEZpbGVzQ29udGludWUob3V0cHV0SWQpIHsKICBjb25zdCBvdXRwdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQob3V0cHV0SWQpOwogIGNvbnN0IHN0ZXBzID0gb3V0cHV0RWxlbWVudC5zdGVwczsKCiAgY29uc3QgbmV4dCA9IHN0ZXBzLm5leHQob3V0cHV0RWxlbWVudC5sYXN0UHJvbWlzZVZhbHVlKTsKICByZXR1cm4gUHJvbWlzZS5yZXNvbHZlKG5leHQudmFsdWUucHJvbWlzZSkudGhlbigodmFsdWUpID0+IHsKICAgIC8vIENhY2hlIHRoZSBsYXN0IHByb21pc2UgdmFsdWUgdG8gbWFrZSBpdCBhdmFpbGFibGUgdG8gdGhlIG5leHQKICAgIC8vIHN0ZXAgb2YgdGhlIGdlbmVyYXRvci4KICAgIG91dHB1dEVsZW1lbnQubGFzdFByb21pc2VWYWx1ZSA9IHZhbHVlOwogICAgcmV0dXJuIG5leHQudmFsdWUucmVzcG9uc2U7CiAgfSk7Cn0KCi8qKgogKiBHZW5lcmF0b3IgZnVuY3Rpb24gd2hpY2ggaXMgY2FsbGVkIGJldHdlZW4gZWFjaCBhc3luYyBzdGVwIG9mIHRoZSB1cGxvYWQKICogcHJvY2Vzcy4KICogQHBhcmFtIHtzdHJpbmd9IGlucHV0SWQgRWxlbWVudCBJRCBvZiB0aGUgaW5wdXQgZmlsZSBwaWNrZXIgZWxlbWVudC4KICogQHBhcmFtIHtzdHJpbmd9IG91dHB1dElkIEVsZW1lbnQgSUQgb2YgdGhlIG91dHB1dCBkaXNwbGF5LgogKiBAcmV0dXJuIHshSXRlcmFibGU8IU9iamVjdD59IEl0ZXJhYmxlIG9mIG5leHQgc3RlcHMuCiAqLwpmdW5jdGlvbiogdXBsb2FkRmlsZXNTdGVwKGlucHV0SWQsIG91dHB1dElkKSB7CiAgY29uc3QgaW5wdXRFbGVtZW50ID0gZG9jdW1lbnQuZ2V0RWxlbWVudEJ5SWQoaW5wdXRJZCk7CiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gZmFsc2U7CgogIGNvbnN0IG91dHB1dEVsZW1lbnQgPSBkb2N1bWVudC5nZXRFbGVtZW50QnlJZChvdXRwdXRJZCk7CiAgb3V0cHV0RWxlbWVudC5pbm5lckhUTUwgPSAnJzsKCiAgY29uc3QgcGlja2VkUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBpbnB1dEVsZW1lbnQuYWRkRXZlbnRMaXN0ZW5lcignY2hhbmdlJywgKGUpID0+IHsKICAgICAgcmVzb2x2ZShlLnRhcmdldC5maWxlcyk7CiAgICB9KTsKICB9KTsKCiAgY29uc3QgY2FuY2VsID0gZG9jdW1lbnQuY3JlYXRlRWxlbWVudCgnYnV0dG9uJyk7CiAgaW5wdXRFbGVtZW50LnBhcmVudEVsZW1lbnQuYXBwZW5kQ2hpbGQoY2FuY2VsKTsKICBjYW5jZWwudGV4dENvbnRlbnQgPSAnQ2FuY2VsIHVwbG9hZCc7CiAgY29uc3QgY2FuY2VsUHJvbWlzZSA9IG5ldyBQcm9taXNlKChyZXNvbHZlKSA9PiB7CiAgICBjYW5jZWwub25jbGljayA9ICgpID0+IHsKICAgICAgcmVzb2x2ZShudWxsKTsKICAgIH07CiAgfSk7CgogIC8vIENhbmNlbCB1cGxvYWQgaWYgdXNlciBoYXNuJ3QgcGlja2VkIGFueXRoaW5nIGluIHRpbWVvdXQuCiAgY29uc3QgdGltZW91dFByb21pc2UgPSBuZXcgUHJvbWlzZSgocmVzb2x2ZSkgPT4gewogICAgc2V0VGltZW91dCgoKSA9PiB7CiAgICAgIHJlc29sdmUobnVsbCk7CiAgICB9LCBGSUxFX0NIQU5HRV9USU1FT1VUX01TKTsKICB9KTsKCiAgLy8gV2FpdCBmb3IgdGhlIHVzZXIgdG8gcGljayB0aGUgZmlsZXMuCiAgY29uc3QgZmlsZXMgPSB5aWVsZCB7CiAgICBwcm9taXNlOiBQcm9taXNlLnJhY2UoW3BpY2tlZFByb21pc2UsIHRpbWVvdXRQcm9taXNlLCBjYW5jZWxQcm9taXNlXSksCiAgICByZXNwb25zZTogewogICAgICBhY3Rpb246ICdzdGFydGluZycsCiAgICB9CiAgfTsKCiAgaWYgKCFmaWxlcykgewogICAgcmV0dXJuIHsKICAgICAgcmVzcG9uc2U6IHsKICAgICAgICBhY3Rpb246ICdjb21wbGV0ZScsCiAgICAgIH0KICAgIH07CiAgfQoKICBjYW5jZWwucmVtb3ZlKCk7CgogIC8vIERpc2FibGUgdGhlIGlucHV0IGVsZW1lbnQgc2luY2UgZnVydGhlciBwaWNrcyBhcmUgbm90IGFsbG93ZWQuCiAgaW5wdXRFbGVtZW50LmRpc2FibGVkID0gdHJ1ZTsKCiAgZm9yIChjb25zdCBmaWxlIG9mIGZpbGVzKSB7CiAgICBjb25zdCBsaSA9IGRvY3VtZW50LmNyZWF0ZUVsZW1lbnQoJ2xpJyk7CiAgICBsaS5hcHBlbmQoc3BhbihmaWxlLm5hbWUsIHtmb250V2VpZ2h0OiAnYm9sZCd9KSk7CiAgICBsaS5hcHBlbmQoc3BhbigKICAgICAgICBgKCR7ZmlsZS50eXBlIHx8ICduL2EnfSkgLSAke2ZpbGUuc2l6ZX0gYnl0ZXMsIGAgKwogICAgICAgIGBsYXN0IG1vZGlmaWVkOiAkewogICAgICAgICAgICBmaWxlLmxhc3RNb2RpZmllZERhdGUgPyBmaWxlLmxhc3RNb2RpZmllZERhdGUudG9Mb2NhbGVEYXRlU3RyaW5nKCkgOgogICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAgICAnbi9hJ30gLSBgKSk7CiAgICBjb25zdCBwZXJjZW50ID0gc3BhbignMCUgZG9uZScpOwogICAgbGkuYXBwZW5kQ2hpbGQocGVyY2VudCk7CgogICAgb3V0cHV0RWxlbWVudC5hcHBlbmRDaGlsZChsaSk7CgogICAgY29uc3QgZmlsZURhdGFQcm9taXNlID0gbmV3IFByb21pc2UoKHJlc29sdmUpID0+IHsKICAgICAgY29uc3QgcmVhZGVyID0gbmV3IEZpbGVSZWFkZXIoKTsKICAgICAgcmVhZGVyLm9ubG9hZCA9IChlKSA9PiB7CiAgICAgICAgcmVzb2x2ZShlLnRhcmdldC5yZXN1bHQpOwogICAgICB9OwogICAgICByZWFkZXIucmVhZEFzQXJyYXlCdWZmZXIoZmlsZSk7CiAgICB9KTsKICAgIC8vIFdhaXQgZm9yIHRoZSBkYXRhIHRvIGJlIHJlYWR5LgogICAgbGV0IGZpbGVEYXRhID0geWllbGQgewogICAgICBwcm9taXNlOiBmaWxlRGF0YVByb21pc2UsCiAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgYWN0aW9uOiAnY29udGludWUnLAogICAgICB9CiAgICB9OwoKICAgIC8vIFVzZSBhIGNodW5rZWQgc2VuZGluZyB0byBhdm9pZCBtZXNzYWdlIHNpemUgbGltaXRzLiBTZWUgYi82MjExNTY2MC4KICAgIGxldCBwb3NpdGlvbiA9IDA7CiAgICB3aGlsZSAocG9zaXRpb24gPCBmaWxlRGF0YS5ieXRlTGVuZ3RoKSB7CiAgICAgIGNvbnN0IGxlbmd0aCA9IE1hdGgubWluKGZpbGVEYXRhLmJ5dGVMZW5ndGggLSBwb3NpdGlvbiwgTUFYX1BBWUxPQURfU0laRSk7CiAgICAgIGNvbnN0IGNodW5rID0gbmV3IFVpbnQ4QXJyYXkoZmlsZURhdGEsIHBvc2l0aW9uLCBsZW5ndGgpOwogICAgICBwb3NpdGlvbiArPSBsZW5ndGg7CgogICAgICBjb25zdCBiYXNlNjQgPSBidG9hKFN0cmluZy5mcm9tQ2hhckNvZGUuYXBwbHkobnVsbCwgY2h1bmspKTsKICAgICAgeWllbGQgewogICAgICAgIHJlc3BvbnNlOiB7CiAgICAgICAgICBhY3Rpb246ICdhcHBlbmQnLAogICAgICAgICAgZmlsZTogZmlsZS5uYW1lLAogICAgICAgICAgZGF0YTogYmFzZTY0LAogICAgICAgIH0sCiAgICAgIH07CiAgICAgIHBlcmNlbnQudGV4dENvbnRlbnQgPQogICAgICAgICAgYCR7TWF0aC5yb3VuZCgocG9zaXRpb24gLyBmaWxlRGF0YS5ieXRlTGVuZ3RoKSAqIDEwMCl9JSBkb25lYDsKICAgIH0KICB9CgogIC8vIEFsbCBkb25lLgogIHlpZWxkIHsKICAgIHJlc3BvbnNlOiB7CiAgICAgIGFjdGlvbjogJ2NvbXBsZXRlJywKICAgIH0KICB9Owp9CgpzY29wZS5nb29nbGUgPSBzY29wZS5nb29nbGUgfHwge307CnNjb3BlLmdvb2dsZS5jb2xhYiA9IHNjb3BlLmdvb2dsZS5jb2xhYiB8fCB7fTsKc2NvcGUuZ29vZ2xlLmNvbGFiLl9maWxlcyA9IHsKICBfdXBsb2FkRmlsZXMsCiAgX3VwbG9hZEZpbGVzQ29udGludWUsCn07Cn0pKHNlbGYpOwo=",
       "headers": [
        [
         "content-type",
         "application/javascript"
        ]
       ],
       "ok": true,
       "status": 200,
       "status_text": ""
      }
     }
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 59741,
     "status": "ok",
     "timestamp": 1590874287738,
     "user": {
      "displayName": "Bonnie Ma",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gj0duL5aTKrqBP362PgHUpwQ_3yTA5e2b6MsV-F=s64",
      "userId": "16062141668243618164"
     },
     "user_tz": 240
    },
    "id": "-RDL6eIhde1z",
    "outputId": "2a0b36ca-82b0-49c9-8be4-f873db17024f"
   },
   "outputs": [],
   "source": [
    "salary_df = pd.read_csv(\"salary_final.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1008 entries, 0 to 1007\n",
      "Data columns (total 25 columns):\n",
      "Unnamed: 0                        1008 non-null int64\n",
      "Unnamed: 0.1                      1008 non-null int64\n",
      "Title                             1008 non-null object\n",
      "Company                           1008 non-null object\n",
      "Link                              1008 non-null object\n",
      "Review                            1008 non-null object\n",
      "Salary                            1008 non-null object\n",
      "Description                       1008 non-null object\n",
      "Location2                         1003 non-null object\n",
      "Rate_by                           1008 non-null object\n",
      "low                               1008 non-null object\n",
      "high                              798 non-null object\n",
      "min                               1008 non-null float64\n",
      "max                               1008 non-null float64\n",
      "adjusted_salary                   1008 non-null float64\n",
      "adjusted_yearly_salary            1008 non-null float64\n",
      "Rank                              938 non-null float64\n",
      "City                              938 non-null object\n",
      "Cost of Living Index              938 non-null float64\n",
      "Rent Index                        938 non-null float64\n",
      "Cost of Living Plus Rent Index    938 non-null float64\n",
      "Groceries Index                   938 non-null float64\n",
      "Restaurant Price Index            938 non-null float64\n",
      "Local Purchasing Power Index      938 non-null float64\n",
      "combined_text                     1008 non-null object\n",
      "dtypes: float64(11), int64(2), object(12)\n",
      "memory usage: 197.0+ KB\n"
     ]
    }
   ],
   "source": [
    "salary_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 164
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 516,
     "status": "error",
     "timestamp": 1590873846701,
     "user": {
      "displayName": "Bonnie Ma",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AOh14Gj0duL5aTKrqBP362PgHUpwQ_3yTA5e2b6MsV-F=s64",
      "userId": "16062141668243618164"
     },
     "user_tz": 240
    },
    "id": "f1wMqzFmde12",
    "outputId": "b7f2da49-6002-4e28-df06-e7968d5057ad"
   },
   "outputs": [],
   "source": [
    "salary_df = salary_df.drop_duplicates(subset=\"Link\", keep='first')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "EDxA_PQ7de17",
    "outputId": "73bd1b11-06c0-4c18-a2f1-a7548002b7b3"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[<matplotlib.axes._subplots.AxesSubplot object at 0x110ce7be0>]],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEICAYAAACktLTqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAaeUlEQVR4nO3dfZBc1X3m8e8DAiEzRC8WTGSheHCQE2OUyGgCeL3rzAQbI0hWsAYWjEECHDkbqOBaZYOIt2wcm5ScWGbjihcsAot4WQ+KMYHiJQQLZgleY0BERgKZRYCC9WIpGCQYTCiL/PaPeyZptbqnX6Z7puf4+VR19b3n3r7n17d7nr59+naPIgIzM8vLAeNdgJmZtZ7D3cwsQw53M7MMOdzNzDLkcDczy5DD3cwsQw53a5qkHkkhaVKav0/S4g6oKyQdPd51DJN0o6QvjUO/fZK2jnW/1hkc7tYyEbEwIlaPZhuSlkh6pFU1mf28cribtZGkA8e7hmYMvxuzicvhbvuRtFzS85Jel/SMpDNS+4GSviLpZUkvAKeV3W5Q0qfS9JWSbilZVj6Es0TSC6mPFyWdJ+l9wLXAByUNSdqd1p2c+n1J0k5J10qaUrLt/yZph6Ttki6q4/79RtrOpJK2j0tan6YPKNkHP5G0RtKMknX/WtKPJe2R9LCk95csu1HSNZLulfQG0F/W90ZJv1Myf1Dan/NHqPcQSbekWnZLelxSd1p2oaRNaT++IOnTI2yn4uOali2R9F1JV0t6BfiipFckzStZ5whJb0o6vNY+tvHncLdKngf+AzAV+AJwi6RZwO8Cvw18AOgFzmxm45IOBb4GLIyIw4B/B6yPiE3A7wHfi4iuiJiWbvJl4L3AfOBoYDbwubStU4A/BD4KzAU+Uqv/iHgc+Em6zbBPAjen6T8ATgd+E3gX8Crw9ZJ170t9HQE8Cdxa1sUngKuAw4DyIaabUl/DTgV2RMT6EUpeTPFYzAHeSbGP3kzLdlE8Jr8AXAhcLem4Ktup9rgOOwF4Id2vPwEGymo9F/hORPzTCLVap4gIX3wZ8QKsBxYBDwK/V9J+MhDApDQ/CHwqTV8J3FKybs/wusChwG7g48CUsr6WAI+UzAt4A/jlkrYPAi+m6RuAFSXL3pv6ObrGfbocuDVNzwB+CsxK85uAk0rWnQX8bPh+lm1nWupvapq/EbipbJ0bgS+l6XcBrwO/kOa/BfxRjVovAv4v8Gt1PFZ/A1yWpvuArbUe15L9/lLZ8hOAHwEHpPkngLPH+/noS30XH7nbfiRdIGl9GgLYDRwLzKQIph+VrPqPzWw/It4A/jPFEegOSfdI+tUqqx8OvANYV1LP36Z2RlHTLcDvSOoCzgb+PiJ2pGXvBu4o6W8T8DbQnYamVqThjdeALek2M0u2XVrPPiJiO/Bd4OOSpgEL2f/Iv9zNwP3AQBp6+jNJBwFIWijp0TSEspvincDMShsZ4XGtWHdEfJ/ihfU30+NzNHBXjVqtQzjcbR+S3g1cB1wKvDOKoZGNFEfQOyiGBob90gibeoMilIf9YunCiLg/Ij5KcVT8w9QnFEfBpV6mGIJ4f0RMS5epEdGVljdSU2n/24DvAWcA5/NvQzJQhNzCkv6mRcQh6TafoHgX8xGK4Y2edBuVbr5G96sphjvOohiC2laj1p9FxBci4hiKIazfBi6QNBm4HfgK0J0eq3vLaimKG/lxHanu4VrPB74VEf9c475Zh3C4W7lDKf7I/wmKD+wojvAA1gB/IOlISdOB5SNsZz3wYUm/JGkqcMXwAkndkv5jGnt/CxiiODIG2AkcKelggIj4F4pQulrSEen2syV9rKSmJZKOkfQO4PMN3NebgD8C5gF3lLRfC1yVAhFJh0talJYdlmr+CcWL15820N+wvwGOAy5LNYxIUr+keSrOvHmNYojobeBgYDLFY7VX0kKKobJKRnpcR3IzxQvgJ+up1TqHw932ERHPACspjmp3UgTfd9Pi6yiGB35A8UHit0fYzgPAbcBTwDrg7pLFBwDLgO3AKxQfXP5+WvYg8DTwY0kvp7bLgc3Ao2ko5DvAr6R+7gP+R7rd5nRdrztIQzBpqGjYX1AMP/ydpNeBRynGn6EIuH8EtgHPpGUNiYg3KY64j2KEfVjiFynG5l+jGCL6PxSfZ7xO8eHvGooPfT9BlWGTGo/rSLVupXisA/j7Omq1DqEI/7MOaw1JDwN/FRET5ghP0vPApyPiO2Pc7+eA90bEJ2uuPM4k3QBsj4j/Pt61WP38RQVriTQk8h7gxfGupV6SPk5xRNrI0X4r+p0BXEwxjt3RJPUA/4ni9FebQDwsY6OWxsJ/TDFc0DE/HSDpaRVfhiq/nCdpELgGuCSN649VTb9L8YHtfRHxcEn7eVVqfXqsaqtQ6xcpPnT984iYMC/aVvCwjJlZhnzkbmaWoY4Yc585c2b09PRUXPbGG29w6KGHjm1BTXKt7eFa22ci1eta97du3bqXI6Lyb/2M91dkI4IFCxZENQ899FDVZZ3GtbaHa22fiVSva90f8ET45wfMzH5+ONzNzDLkcDczy5DD3cwsQw53M7MMOdzNzDLkcDczy5DD3cwsQw53M7MMdcTPD0xUPcvv2Wd+2by9LClra4ctK05rex9mNrH5yN3MLEMOdzOzDNUMd0mHSHpM0g/SPz/4Qmo/StL3JT0n6bbhf2gsaXKa35yW97T3LpiZWbl6jtzfAn4rIn4dmA+cIulE4MvA1RExl+Kf816c1r8YeDUijgauTuuZmdkYqhnu6Zclh9LsQekSwG9R/Ed2gNXA6Wl6UZonLT9JklpWsZmZ1VTXv9mTdCCwDjga+Drw58Cj6egcSXMo/ifksZI2AqdExNa07HnghIh4uWybS4GlAN3d3QsGBgYq9j00NERXV1eTd6+9Nmzbs8989xTY+Wb7+503e+qot9HJ+7Wca22fiVSva91ff3//uojorbSsrlMhI+JtYL6kacAdwPsqrZauKx2l7/cKEhGrgFUAvb290dfXV7HvwcFBqi0bb+WnPS6bt5eVG9p/dumW8/pGvY1O3q/lXGv7TKR6XWtjGjpbJiJ2A4PAicA0ScNJdiSwPU1vBeYApOVTgVdaUayZmdWnnrNlDk9H7EiaAnwE2AQ8BJyZVlsM3Jmm70rzpOUPRj1jP2Zm1jL1jCHMAlancfcDgDURcbekZ4ABSV8C/gG4Pq1/PXCzpM0UR+zntKFuMzMbQc1wj4ingA9UaH8BOL5C+z8DZ7WkOjMza4q/oWpmliGHu5lZhhzuZmYZcribmWXI4W5mliGHu5lZhhzuZmYZcribmWXI4W5mliGHu5lZhhzuZmYZcribmWXI4W5mliGHu5lZhhzuZmYZcribmWXI4W5mliGHu5lZhhzuZmYZcribmWXI4W5mliGHu5lZhhzuZmYZcribmWXI4W5mlqGa4S5pjqSHJG2S9LSky1L7lZK2SVqfLqeW3OYKSZslPSvpY+28A2Zmtr9JdayzF1gWEU9KOgxYJ+mBtOzqiPhK6cqSjgHOAd4PvAv4jqT3RsTbrSzczMyqq3nkHhE7IuLJNP06sAmYPcJNFgEDEfFWRLwIbAaOb0WxZmZWH0VE/StLPcDDwLHAfwWWAK8BT1Ac3b8q6S+BRyPilnSb64H7IuJbZdtaCiwF6O7uXjAwMFCxz6GhIbq6uhq6U2Nlw7Y9+8x3T4Gdb7a/33mzp456G528X8u51vaZSPW61v319/evi4jeSsvqGZYBQFIXcDvwmYh4TdI1wBeBSNcrgYsAVbj5fq8gEbEKWAXQ29sbfX19FfsdHByk2rLxtmT5PfvML5u3l5Ub6t6lTdtyXt+ot9HJ+7Wca22fiVSva21MXUkk6SCKYL81Ir4NEBE7S5ZfB9ydZrcCc0pufiSwvSXVGgA9ZS8qzVg2b+9+L0712LLitFH3bWbtV8/ZMgKuBzZFxFdL2meVrHYGsDFN3wWcI2mypKOAucBjrSvZzMxqqefI/UPA+cAGSetT2x8D50qaTzHksgX4NEBEPC1pDfAMxZk2l/hMGTOzsVUz3CPiESqPo987wm2uAq4aRV1mZjYK/oaqmVmGHO5mZhlyuJuZZcjhbmaWIYe7mVmGHO5mZhlyuJuZZcjhbmaWIYe7mVmGHO5mZhlyuJuZZcjhbmaWIYe7mVmGHO5mZhlyuJuZZcjhbmaWIYe7mVmGHO5mZhlyuJuZZcjhbmaWIYe7mVmGHO5mZhlyuJuZZcjhbmaWoZrhLmmOpIckbZL0tKTLUvsMSQ9Iei5dT0/tkvQ1SZslPSXpuHbfCTMz21c9R+57gWUR8T7gROASSccAy4G1ETEXWJvmARYCc9NlKXBNy6s2M7MR1Qz3iNgREU+m6deBTcBsYBGwOq22Gjg9TS8CborCo8A0SbNaXrmZmVWliKh/ZakHeBg4FngpIqaVLHs1IqZLuhtYERGPpPa1wOUR8UTZtpZSHNnT3d29YGBgoGKfQ0NDdHV1NXKfxsyGbXv2me+eAjvfHKdiGtRsrfNmT219MTV08nOg3ESqFSZWva51f/39/esiorfSskn1bkRSF3A78JmIeE1S1VUrtO33ChIRq4BVAL29vdHX11dxY4ODg1RbNt6WLL9nn/ll8/ayckPdu3RcNVvrlvP6Wl9MDZ38HCg3kWqFiVWva21MXWfLSDqIIthvjYhvp+adw8Mt6XpXat8KzCm5+ZHA9taUa2Zm9ajnbBkB1wObIuKrJYvuAhan6cXAnSXtF6SzZk4E9kTEjhbWbGZmNdTzvvxDwPnABknrU9sfAyuANZIuBl4CzkrL7gVOBTYDPwUubGnFZmZWU81wTx+MVhtgP6nC+gFcMsq6zMxsFPwNVTOzDDnczcwy5HA3M8uQw93MLEMOdzOzDDnczcwy5HA3M8uQw93MLEMOdzOzDDnczcwy5HA3M8uQw93MLEMOdzOzDDnczcwy5HA3M8uQw93MLEMOdzOzDDnczcwy5HA3M8tQPf8g2+xf9Sy/Z8z7XDZvL31j3qvZxOYjdzOzDDnczcwy5HA3M8uQw93MLEM1w13SDZJ2SdpY0nalpG2S1qfLqSXLrpC0WdKzkj7WrsLNzKy6eo7cbwROqdB+dUTMT5d7ASQdA5wDvD/d5n9KOrBVxZqZWX1qhntEPAy8Uuf2FgEDEfFWRLwIbAaOH0V9ZmbWhNGMuV8q6ak0bDM9tc0GflSyztbUZmZmY0gRUXslqQe4OyKOTfPdwMtAAF8EZkXERZK+DnwvIm5J610P3BsRt1fY5lJgKUB3d/eCgYGBin0PDQ3R1dXV+D0bAxu27dlnvnsK7HxznIpp0ESr9YgZU8e7jLp08vO1kolUr2vdX39//7qI6K20rKlvqEbEzuFpSdcBd6fZrcCcklWPBLZX2cYqYBVAb29v9PX1VexrcHCQasvG25Kyb2sum7eXlRsmxpd+J1qtZ3foc6BcJz9fK5lI9brWxjQ1LCNpVsnsGcDwmTR3AedImizpKGAu8NjoSjQzs0bVPHST9E2gD5gpaSvweaBP0nyKYZktwKcBIuJpSWuAZ4C9wCUR8XZ7Sjczs2pqhntEnFuh+foR1r8KuGo0RZmZ2ej4G6pmZhlyuJuZZcjhbmaWIYe7mVmGHO5mZhlyuJuZZcjhbmaWIYe7mVmGHO5mZhlyuJuZZcjhbmaWIYe7mVmGHO5mZhlyuJuZZcjhbmaWIYe7mVmGHO5mZhlyuJuZZcjhbmaWIYe7mVmGHO5mZhlyuJuZZcjhbmaWoUnjXcBo9Sy/Z7xLMDPrOD5yNzPLUM1wl3SDpF2SNpa0zZD0gKTn0vX01C5JX5O0WdJTko5rZ/FmZlZZPUfuNwKnlLUtB9ZGxFxgbZoHWAjMTZelwDWtKdPMzBpRM9wj4mHglbLmRcDqNL0aOL2k/aYoPApMkzSrVcWamVl9FBG1V5J6gLsj4tg0vzsippUsfzUipku6G1gREY+k9rXA5RHxRIVtLqU4uqe7u3vBwMBAxb6Hhobo6uqqWtuGbXtq1j9WuqfAzjfHu4r6TLRaj5gxdbzLqEut52unmUj1utb99ff3r4uI3krLWn22jCq0VXz1iIhVwCqA3t7e6Ovrq7jBwcFBqi0DWNJBZ8ssm7eXlRsmxglIE63Ws0d4DnSSWs/XTjOR6nWtjWn2bJmdw8Mt6XpXat8KzClZ70hge/PlmZlZM5oN97uAxWl6MXBnSfsF6ayZE4E9EbFjlDWamVmDar4vl/RNoA+YKWkr8HlgBbBG0sXAS8BZafV7gVOBzcBPgQvbULOZmdVQM9wj4twqi06qsG4Al4y2KDMzGx1/Q9XMLEMOdzOzDDnczcwy5HA3M8uQw93MLEMOdzOzDDnczcwy5HA3M8vQxPjlKPu5N57/TnHLitPGrW+zZvnI3cwsQw53M7MMOdzNzDLkcDczy5DD3cwsQw53M7MMOdzNzDLkcDczy5DD3cwsQw53M7MMOdzNzDLkcDczy5DD3cwsQw53M7MMOdzNzDI0qt9zl7QFeB14G9gbEb2SZgC3AT3AFuDsiHh1dGWamVkjWnHk3h8R8yOiN80vB9ZGxFxgbZo3M7Mx1I5hmUXA6jS9Gji9DX2YmdkIFBHN31h6EXgVCOAbEbFK0u6ImFayzqsRMb3CbZcCSwG6u7sXDAwMVOxjaGiIrq6uqjVs2Lan6fpbrXsK7HxzvKuoj2ut37zZU+tet9bztdNMpHpd6/76+/vXlYya7GO0/0P1QxGxXdIRwAOSfljvDSNiFbAKoLe3N/r6+iquNzg4SLVlAEvG8X9rlls2by8rN0yMf0vrWuu35by+utet9XztNBOpXtfamFENy0TE9nS9C7gDOB7YKWkWQLreNdoizcysMU0fDkk6FDggIl5P0ycDfwLcBSwGVqTrO1tRqNl46Wng3eGyeXtb9m5yy4rTWrId+/k0mve63cAdkoa3878j4m8lPQ6skXQx8BJw1ujLNDOzRjQd7hHxAvDrFdp/Apw0mqLMzGx0/A1VM7MMOdzNzDLkcDczy5DD3cwsQw53M7MMOdzNzDLkcDczy5DD3cwsQw53M7MMOdzNzDLkcDczy5DD3cwsQw53M7MMOdzNzDLkcDczy5DD3cwsQw53M7MMOdzNzDLkcDczy5DD3cwsQw53M7MMOdzNzDLkcDczy5DD3cwsQw53M7MMTWrXhiWdAvwFcCDwVxGxol19mVkeepbfU3XZsnl7WTLC8tHYsuK0tmx3PLUl3CUdCHwd+CiwFXhc0l0R8Uw7+jPL0UhB1yqVAjPHoKul1fu6kReidu3vdh25Hw9sjogXACQNAIsAh7tZhxuLFxVrP0VE6zcqnQmcEhGfSvPnAydExKUl6ywFlqbZXwGerbK5mcDLLS+yPVxre7jW9plI9brW/b07Ig6vtKBdR+6q0LbPq0hErAJW1dyQ9ERE9LaqsHZyre3hWttnItXrWhvTrrNltgJzSuaPBLa3qS8zMyvTrnB/HJgr6ShJBwPnAHe1qS8zMyvTlmGZiNgr6VLgfopTIW+IiKeb3FzNoZsO4lrbw7W2z0Sq17U2oC0fqJqZ2fjyN1TNzDLkcDczy1FEdOQFOIXi3PfNwPIx7nsLsAFYDzyR2mYADwDPpevpqV3A11KdTwHHlWxncVr/OWBxSfuCtP3N6bZqoLYbgF3AxpK2ttdWrY8m670S2Jb273rg1JJlV6S+nwU+Vuv5ABwFfD/VdRtwcGqfnOY3p+U9NeqcAzwEbAKeBi7r5H07Qr2duG8PAR4DfpBq/UKz22/VfWii1huBF0v26/xOeB6MeF9Gc+N2XSg+hH0eeA9wcNrRx4xh/1uAmWVtfzb8pAGWA19O06cC96UH+UTg+yUP1AvpenqaHg6Gx4APptvcByxsoLYPA8exb1i2vbZqfTRZ75XAH1ZY95j0WE9Of5TPp+dC1ecDsAY4J01fC/yXNP37wLVp+hzgthp1zhr+wwQOA/5fqqcj9+0I9XbivhXQlaYPogjbExvdfivvQxO13gicWWH9cf8bq3pfRnPjdl3SHb+/ZP4K4Iox7H8L+4f7s8Cskj+sZ9P0N4Bzy9cDzgW+UdL+jdQ2C/hhSfs+69VZXw/7hmXba6vWR5P1XknlANrncaY42+qD1Z4P6Y/jZWBS+fNm+LZpelJar5F3SHdS/DZSR+/bCvV29L4F3gE8CZzQ6PZbeR+aqPVGKod7Rz0PSi+dOuY+G/hRyfzW1DZWAvg7SevSzyQAdEfEDoB0fURqr1brSO1bK7SPxljUVq2PZl0q6SlJN0ia3mS97wR2R8TeCvX+623S8j1p/Zok9QAfoDhq6/h9W1YvdOC+lXSgpPUUQ3QPUBxpN7r9Vt6HumuNiOH9elXar1dLmlxea501jdnfWKeGe82fL2izD0XEccBC4BJJHx5h3Wq1NtreDp1a2zXALwPzgR3AytTeynqbui+SuoDbgc9ExGsjrdpgTW3ZtxXq7ch9GxFvR8R8im+rHw+8r4ntj8k+L69V0rEU7wR+FfgNiqGWy1tca8t1ariP688XRMT2dL0LuIPiybhT0iyAdL2rRq0jtR9ZoX00xqK2an00LCJ2pj+gfwGuo9i/zdT7MjBN0qSy9n22lZZPBV4ZqS5JB1EE5a0R8e3U3LH7tlK9nbpvh0XEbmCQYny60e238j40UuspEbEjCm8B/4vm9+uY/I1B54b7uP18gaRDJR02PA2cDGxM/S9Oqy2mGOMktV+gwonAnvSW6n7gZEnT01vjkynG+3YAr0s6UZKAC0q21ayxqK1aHw0bfgInZ1Ds3+E+zpE0WdJRwFyKD58qPh+iGJx8CDizyn0frvdM4MG0frWaBFwPbIqIr5Ys6sh9W63eDt23h0ualqanAB+hOMun0e238j40UusPS0JXwOll+7Xj/saAzvxANT1PTqU4A+B54LNj2O97KD5tHz4V6rOp/Z3AWorTlNYCM1K7KP4xyfMUpzf1lmzrIorTnTYDF5a096Ynx/PAX9LYB33fpHi7/TOKo4CLx6K2an00We/NqZ6nKJ7Qs0rW/2zq+1lKziKq9nxIj9dj6X78NTA5tR+S5jen5e+pUee/p3h7/BQlpxF26r4dod5O3Le/BvxDqmkj8Llmt9+q+9BErQ+m/boRuIV/O6Nm3P/Gql388wNmZhnq1GEZMzMbBYe7mVmGHO5mZhlyuJuZZcjhbmaWIYe7mVmGHO5mZhn6//rW5pta7W2BAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "salary_df.hist('adjusted_yearly_salary')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "m5sKPx_Ide1_",
    "outputId": "611794c0-0337-4cd6-bdde-c977c4909b18"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "352500.0"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "salary_df['adjusted_yearly_salary'].max() - salary_df['adjusted_yearly_salary'].min()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "JZbWB7hbde2C"
   },
   "source": [
    "Salary bins: [< 50,000, 50,000-75,000, 75,000-100,000, 100,000-125,000, 125,000-150,000, 150,000-175,000, 175,000-200,000, 200,000-225,000, 225,000-250,000, >250,000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Hd-3IVznde2D",
    "outputId": "1b484a63-d4fb-43d0-baa5-c06f4456e0ed"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.000000     10000.000000\n",
       "0.166667     54026.666667\n",
       "0.333333     70000.000000\n",
       "0.500000     90000.000000\n",
       "0.666667    114400.000000\n",
       "0.833333    145600.000000\n",
       "1.000000    362500.000000\n",
       "Name: adjusted_yearly_salary, dtype: float64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "salary_df['adjusted_yearly_salary'].quantile(list(np.linspace(0,1,7)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "aVZgrcgvde2G"
   },
   "outputs": [],
   "source": [
    "#Create Bins\n",
    "BINS = [0,50000,70000,90000,120000,150000,400000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "G5DgQQK_de2J"
   },
   "outputs": [],
   "source": [
    "salary_df['Salary_Bins'] = pd.cut(x=salary_df['adjusted_yearly_salary'], bins=BINS, labels=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "R3nP4LoVde2M",
    "outputId": "51fbccda-f6c1-4745-9215-7612f2d783ac"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0       5\n",
       "2       1\n",
       "4       0\n",
       "6       2\n",
       "8       3\n",
       "       ..\n",
       "1003    3\n",
       "1004    3\n",
       "1005    3\n",
       "1006    2\n",
       "1007    2\n",
       "Name: Salary_Bins, Length: 963, dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "salary_df['Salary_Bins']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "salary_df.to_csv(\"salary_final_2.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "rXB4mFKPde2P",
    "outputId": "171cbcc6-7848-486f-ff92-65ee4c2d162a"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x13f28a320>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAEECAYAAADTdnSRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAS00lEQVR4nO3df7DldV3H8edL8EdmBcaVtoXtqm2WmK56I4tMjH4gOKFNFNQgEbk6A6UzjRNho9Zkw2Tm9EuaVQhoDDGRpKSU2UJGC2IXV1gEFXSVjXX3giUqDrbLuz/O946ny9m9Z+855569H56PmTvnez7fz/f7fX9hefHZz/3+SFUhSWrL46ZdgCRp/Ax3SWqQ4S5JDTLcJalBhrskNchwl6QGHT7tAgCOOuqomp2dnXYZkrSqbN269f6qmhm07pAI99nZWbZs2TLtMiRpVUnyhf2tc1pGkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1KBD4iYmqUWzF3xoRY+346JTV/R4OrQ5cpekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1aMlwT3Jskn9LcmeSO5K8rmt/apLrk3y2+zyya0+SP09yd5Lbkrxg0ichSfr/hhm57wV+u6p+CHgRcF6SZwMXAJuraj2wufsO8DJgffezEbh47FVLkg5oyXCvql1VdWu3/FXgTmAtcBpwedftcuAV3fJpwBXVcxNwRJI1Y69ckrRfBzXnnmQWeD5wM3B0Ve2C3v8AgKd13dYC9/ZttrNrkyStkKHDPclTgKuB11fVgwfqOqCtBuxvY5ItSbbMz88PW4YkaQhDhXuSx9ML9vdU1Qe65t0L0y3d556ufSdwbN/mxwD3Ld5nVW2qqrmqmpuZmVlu/ZKkAYa5WibAJcCdVfWnfauuBc7uls8GPtjX/qruqpkXAV9ZmL6RJK2MYZ7nfgJwFnB7km1d24XARcD7kpwLfBE4vVt3HXAKcDfwEHDOWCuWJC1pyXCvqo8xeB4d4KQB/Qs4b8S6JEkj8A5VSWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1KBhrnPXlMxe8KEVPd6Oi05d0eNJmhxH7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGDfMmpkuT7Emyva/tqiTbup8dCy/xSDKb5Bt96/56ksVLkgYb5g7Vy4C/BK5YaKiqX15YTvJ24Ct9/e+pqg3jKlCSdPCGeRPTjUlmB63r3q/6S8BPjbcsSdIoRp1zfzGwu6o+29f29CSfSPLRJC8ecf+SpGUY9cFhZwJX9n3fBayrqgeSvBD4hyTHVdWDizdMshHYCLBu3boRy5Ak9Vv2yD3J4cAvAFcttFXVw1X1QLe8FbgH+IFB21fVpqqaq6q5mZmZ5ZYhSRpglGmZnwbuqqqdCw1JZpIc1i0/A1gPfG60EiVJB2uYSyGvBP4DeFaSnUnO7Vadwf+fkgH4SeC2JJ8E3g+8tqq+PM6CJUlLG+ZqmTP30/5rA9quBq4evSxJ0ii8Q1WSGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1KAlX9aR5FLg5cCeqnpO1/YW4NXAfNftwqq6rlv3u8C5wD7gt6rqwxOoWw2YveBDK3q8HReduqLHk6ZpmJH7ZcDJA9rfUVUbup+FYH82vdfvHddt886Fd6pKklbOkuFeVTcCw74H9TTgvVX1cFV9HrgbOH6E+iRJyzDKnPv5SW5LcmmSI7u2tcC9fX12dm2SpBW03HC/GHgmsAHYBby9a8+AvjVoB0k2JtmSZMv8/PygLpKkZVpWuFfV7qraV1WPAO/iW1MvO4Fj+7oeA9y3n31sqqq5qpqbmZlZThmSpP1YVrgnWdP39ZXA9m75WuCMJE9M8nRgPfCfo5UoSTpYw1wKeSVwInBUkp3Am4ETk2ygN+WyA3gNQFXdkeR9wKeAvcB5VbVvMqVLkvZnyXCvqjMHNF9ygP5vBd46SlGSpNF4h6okNchwl6QGLTktcyjz9nVJGsyRuyQ1yHCXpAYZ7pLUIMNdkhq0qn+hKkmT0MLFGo7cJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lq0JLh3r0Ae0+S7X1tb0tyV/eC7GuSHNG1zyb5RpJt3c9fT7J4SdJgw4zcLwNOXtR2PfCcqnou8Bngd/vW3VNVG7qf146nTEnSwVgy3KvqRuDLi9o+UlV7u6830XsRtiTpEDGOOfdfB/657/vTk3wiyUeTvHgM+5ckHaSRni2T5I30XoT9nq5pF7Cuqh5I8kLgH5IcV1UPDth2I7ARYN26daOUIUlaZNkj9yRnAy8HfrWqCqCqHq6qB7rlrcA9wA8M2r6qNlXVXFXNzczMLLcMSdIAyxq5JzkZ+B3gJVX1UF/7DPDlqtqX5BnAeuBzY6lU0iGjhacmtm7JcE9yJXAicFSSncCb6V0d80Tg+iQAN3VXxvwk8AdJ9gL7gNdW1ZcH7liSNDFLhntVnTmg+ZL99L0auHrUoiRJo/EOVUlqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkho0VLgnuTTJniTb+9qemuT6JJ/tPo/s2pPkz5PcneS2JC+YVPGSpMGGHblfBpy8qO0CYHNVrQc2d98BXkbv9Xrr6b0A++LRy5QkHYyhwr2qbgQWvy7vNODybvly4BV97VdUz03AEUnWjKNYSdJwRplzP7qqdgF0n0/r2tcC9/b129m1SZJWyCR+oZoBbfWoTsnGJFuSbJmfn59AGZL02DVKuO9emG7pPvd07TuBY/v6HQPct3jjqtpUVXNVNTczMzNCGZKkxUYJ92uBs7vls4EP9rW/qrtq5kXAVxambyRJK+PwYToluRI4ETgqyU7gzcBFwPuSnAt8ETi9634dcApwN/AQcM6Ya5YkLWGocK+qM/ez6qQBfQs4b5SiJEmj8Q5VSWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDhnpZxyBJngVc1df0DOBNwBHAq4GFt15fWFXXLbtCSdJBW3a4V9WngQ0ASQ4D/gu4ht5r9d5RVX8ylgolSQdtXNMyJwH3VNUXxrQ/SdIIxhXuZwBX9n0/P8ltSS5NcuSYjiFJGtLI4Z7kCcDPA3/fNV0MPJPelM0u4O372W5jki1JtszPzw/qIklapnGM3F8G3FpVuwGqandV7auqR4B3AccP2qiqNlXVXFXNzczMjKEMSdKCcYT7mfRNySRZ07fulcD2MRxDknQQln21DECSJwM/A7ymr/mPk2wACtixaJ0kaQWMFO5V9RDw3YvazhqpIknSyLxDVZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAaN9Dx3gCQ7gK8C+4C9VTWX5KnAVcAsvRd2/FJV/feox5IkDWdcI/eXVtWGqprrvl8AbK6q9cDm7rskaYVMalrmNODybvly4BUTOo4kaYBxhHsBH0myNcnGru3oqtoF0H0+bQzHkSQNaeQ5d+CEqrovydOA65PcNcxG3f8INgKsW7duDGVIkhaMPHKvqvu6zz3ANcDxwO4kawC6zz0DtttUVXNVNTczMzNqGZKkPiOFe5JvT/IdC8vAzwLbgWuBs7tuZwMfHOU4kqSDM+q0zNHANUkW9vV3VfUvSW4B3pfkXOCLwOkjHkeSdBBGCveq+hzwvAHtDwAnjbJvSdLyeYeqJDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDVp2uCc5Nsm/JbkzyR1JXte1vyXJfyXZ1v2cMr5yJUnDGOVlHXuB366qW7tX7W1Ncn237h1V9SejlydJWo5lh3tV7QJ2dctfTXInsHZchUmSlm8sc+5JZoHnAzd3TecnuS3JpUmOHMcxJEnDGznckzwFuBp4fVU9CFwMPBPYQG9k//b9bLcxyZYkW+bn50ctQ5LUZ6RwT/J4esH+nqr6AEBV7a6qfVX1CPAu4PhB21bVpqqaq6q5mZmZUcqQJC0yytUyAS4B7qyqP+1rX9PX7ZXA9uWXJ0lajlGuljkBOAu4Pcm2ru1C4MwkG4ACdgCvGalCSdJBG+VqmY8BGbDquuWXI0kaB+9QlaQGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1aGLhnuTkJJ9OcneSCyZ1HEnSo00k3JMcBvwV8DLg2fRevffsSRxLkvRokxq5Hw/cXVWfq6pvAu8FTpvQsSRJi0wq3NcC9/Z939m1SZJWQKpq/DtNTgd+rqp+o/t+FnB8Vf1mX5+NwMbu67OAT4+9kP07Crh/BY+30jy/1a3l82v53GDlz+/7qmpm0IrDJ3TAncCxfd+PAe7r71BVm4BNEzr+ASXZUlVz0zj2SvD8VreWz6/lc4ND6/wmNS1zC7A+ydOTPAE4A7h2QseSJC0ykZF7Ve1Ncj7wYeAw4NKqumMSx5IkPdqkpmWoquuA6ya1/xFNZTpoBXl+q1vL59fyucEhdH4T+YWqJGm6fPyAJDXIcJekBk1szv1QkuQH6d0huxYoepdlXltVd061MA2l+/e3Fri5qr7W135yVf3L9CobXZLjgaqqW7pHdJwM3NX9zqo5Sa6oqldNu45JSPIT9O7O315VH5l6Pa3PuSf5HeBMeo9A2Nk1H0Pv8sz3VtVF06pt0pKcU1V/M+06RpHkt4DzgDuBDcDrquqD3bpbq+oF06xvFEneTO/5S4cD1wM/CtwA/DTw4ap66/SqG12SxZc/B3gp8K8AVfXzK17UGCX5z6o6vlt+Nb0/p9cAPwv847Sz5bEQ7p8Bjquq/13U/gTgjqpaP53KJi/JF6tq3bTrGEWS24Efq6qvJZkF3g/8bVX9WZJPVNXzp1rgCLpz2wA8EfgScExVPZjk2+j9LeW5Uy1wREluBT4FvJve35gDXElvYEVVfXR61Y2u/89fkluAU6pqPsm3AzdV1Q9Ps77HwrTMI8D3Al9Y1L6mW7eqJbltf6uAo1eylgk5bGEqpqp2JDkReH+S76N3jqvZ3qraBzyU5J6qehCgqr6RZNX/2QTmgNcBbwTeUFXbknxjtYd6n8clOZLe7y5TVfMAVfX1JHunW9pjI9xfD2xO8lm+9TCzdcD3A+dPrarxORr4OeC/F7UH+PeVL2fsvpRkQ1VtA+hG8C8HLgWmOjIag28meXJVPQS8cKExyXfRwMCjqh4B3pHk77vP3bSVOd8FbKX331ol+Z6q+lKSp3AIDDyan5YBSPI4er/oWEvvH/pO4JZu1LSqJbkE+Juq+tiAdX9XVb8yhbLGJskx9Ea4Xxqw7oSq+vgUyhqLJE+sqocHtB8FrKmq26dQ1sQkORU4oaounHYtk5TkycDRVfX5qdbxWAh3SXqs8Tp3SWqQ4S5JDTLcJalBhrtWnSRvTHJHktuSbEvyowfoe1mSX5xgLZcl+XxXx13djUkL697ti+E1LS1dlqTHgCQ/BrwceEFVPdxdWfKEMe7/8Ko62GuU31BV70/yJOBT3S32n194zaQ0DY7ctdqsAe5fuISwqu6vqvuSvCnJLUm2J9mU5FHXGe+vT5IbkvxRko8Cb+xG4o/v1n1nkh0L35fwpO7z6337neuWv5bkrUk+meSmJEd37ad39XwyyY2j/sORFhjuWm0+Ahyb5DNJ3pnkJV37X1bVj1TVc4Bvoze6X+xAfY6oqpdU1e/Te77LqV37GcDVix9fscjbkmyjd//Ee6tqz4A+C7ekPw+4EXh11/4mei+Tfx6wqp+1okOL4a5VpXsUwQuBjcA8cFWSXwNemuTm7nktPwUcN2DzA/W5qm/53cA53fI5wFIPX3tDVW0Avgc4KcmPD+jzTeCfuuWtwGy3/HHgsu7BU4ctcRxpaM65a9Xp7iy+AbihC+rXAM8F5qrq3iRv4VtTJAB08+HvPECfr/ft/+NJZru/FRxWVduHrOtrSW4AfoJHP/rhf+tbdwzuo/tvr6pe2/1C+FRgW/eohQeGOZ50II7ctaokeVaS/id5bgA+3S3f3z3XY9DVMU8aok+/K+g9wXDoRyYnOZzeY3vvOYhtnllVN1fVm4D7gWOH3VY6EEfuWm2eAvxFkiOAvcDd9KZo/ge4HdgB3LJ4o6r6nyTvOlCfRd4D/CG9gF/K25L8Hr2rdjYDHxjmRPq2XU/vmUebgU8exLbSfvlsGWmA7tr406rqrGnXIi2HI3dpkSR/Qe8NSadMuxZpuRy5S0NI8lfACYua/2y1v8ZQ7TLcJalBXi0jSQ0y3CWpQYa7JDXIcJekBhnuktSg/wOkEnSZ9/ZGCQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "salary_df.groupby('Salary_Bins')['Location2'].count().plot.bar()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "8mQWqLklde2R"
   },
   "outputs": [],
   "source": [
    "#Categorical target\n",
    "target_class = salary_df['Salary_Bins'].astype('category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_word(job_descriptions):\n",
    "    jd_data=[]\n",
    "    pattern = \"([a-zA-Z]+(?:'[a-z]+)?)\"\n",
    "    stopwords_list = stopwords.words('english')\n",
    "    stopwords_list += list(string.punctuation)\n",
    "    stopwords_list += ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9']\n",
    "    stopwords_list += ['www','http','com','llc','covid','non','pm','am','eg','e','g','ie','i']\n",
    "    \n",
    "    for jd in job_descriptions:\n",
    "        jd_tokens_raw = nltk.regexp_tokenize(jd, pattern)\n",
    "        jd_tokens=[word.lower() for word in jd_tokens_raw]\n",
    "        jd_words_stopped = [word for word in jd_tokens if word not in stopwords_list]\n",
    "        jd_data.append(jd_words_stopped)\n",
    "\n",
    "    return pd.Series(jd_data) # Turn list of lists to series of lists to use in Word2Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = clean_word(salary_df['combined_text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['data',\n",
       " 'scientist',\n",
       " 'job',\n",
       " 'summary',\n",
       " 'nsa',\n",
       " 'search',\n",
       " 'computer',\n",
       " 'science',\n",
       " 'professionals',\n",
       " 'solve',\n",
       " 'complex',\n",
       " 'problems',\n",
       " 'test',\n",
       " 'innovative',\n",
       " 'approaches',\n",
       " 'research',\n",
       " 'new',\n",
       " 'solutions',\n",
       " 'storing',\n",
       " 'manipulating',\n",
       " 'presenting',\n",
       " 'information',\n",
       " 'looking',\n",
       " 'apply',\n",
       " 'computer',\n",
       " 'science',\n",
       " 'expertise',\n",
       " 'projects',\n",
       " 'seek',\n",
       " 'create',\n",
       " 'new',\n",
       " 'standards',\n",
       " 'transformation',\n",
       " 'information',\n",
       " 'want',\n",
       " 'develop',\n",
       " 'technologies',\n",
       " 'tools',\n",
       " 'part',\n",
       " 'cutting',\n",
       " 'edge',\n",
       " 'innovations',\n",
       " 'join',\n",
       " 'team',\n",
       " 'experts',\n",
       " 'help',\n",
       " 'protect',\n",
       " 'national',\n",
       " 'security',\n",
       " 'interests',\n",
       " 'part',\n",
       " \"world's\",\n",
       " 'advanced',\n",
       " 'team',\n",
       " 'computer',\n",
       " 'science',\n",
       " 'professionals',\n",
       " 'qualifications',\n",
       " 'candidates',\n",
       " \"nsa's\",\n",
       " 'data',\n",
       " 'scientist',\n",
       " 'roles',\n",
       " 'asked',\n",
       " 'complete',\n",
       " 'data',\n",
       " 'science',\n",
       " 'examination',\n",
       " 'evaluating',\n",
       " 'knowledge',\n",
       " 'statistics',\n",
       " 'mathematics',\n",
       " 'computer',\n",
       " 'science',\n",
       " 'topics',\n",
       " 'pertain',\n",
       " 'data',\n",
       " 'science',\n",
       " 'work',\n",
       " 'passing',\n",
       " 'examination',\n",
       " 'requirement',\n",
       " 'order',\n",
       " 'considered',\n",
       " 'selection',\n",
       " 'data',\n",
       " 'scientist',\n",
       " 'position',\n",
       " 'salary',\n",
       " 'range',\n",
       " 'entry',\n",
       " 'developmental',\n",
       " 'qualifications',\n",
       " 'listed',\n",
       " 'minimum',\n",
       " 'acceptable',\n",
       " 'considered',\n",
       " 'position',\n",
       " 'salary',\n",
       " 'offers',\n",
       " 'based',\n",
       " 'candidates',\n",
       " 'education',\n",
       " 'level',\n",
       " 'years',\n",
       " 'experience',\n",
       " 'relevant',\n",
       " 'position',\n",
       " 'also',\n",
       " 'take',\n",
       " 'account',\n",
       " 'information',\n",
       " 'provided',\n",
       " 'hiring',\n",
       " 'manager',\n",
       " 'organization',\n",
       " 'regarding',\n",
       " 'work',\n",
       " 'level',\n",
       " 'position',\n",
       " 'entry',\n",
       " \"bachelor's\",\n",
       " 'degree',\n",
       " 'experience',\n",
       " \"associate's\",\n",
       " 'degree',\n",
       " 'plus',\n",
       " 'years',\n",
       " 'relevant',\n",
       " 'experience',\n",
       " 'may',\n",
       " 'considered',\n",
       " 'individuals',\n",
       " 'depth',\n",
       " 'experience',\n",
       " 'clearly',\n",
       " 'related',\n",
       " 'position',\n",
       " 'degree',\n",
       " 'must',\n",
       " 'mathematics',\n",
       " 'applied',\n",
       " 'mathematics',\n",
       " 'statistics',\n",
       " 'applied',\n",
       " 'statistics',\n",
       " 'machine',\n",
       " 'learning',\n",
       " 'data',\n",
       " 'science',\n",
       " 'operations',\n",
       " 'research',\n",
       " 'computer',\n",
       " 'science',\n",
       " 'degree',\n",
       " 'related',\n",
       " 'field',\n",
       " 'computer',\n",
       " 'information',\n",
       " 'systems',\n",
       " 'engineering',\n",
       " 'degree',\n",
       " 'physical',\n",
       " 'hard',\n",
       " 'sciences',\n",
       " 'physics',\n",
       " 'chemistry',\n",
       " 'biology',\n",
       " 'astronomy',\n",
       " 'science',\n",
       " 'disciplines',\n",
       " 'behavioral',\n",
       " 'social',\n",
       " 'life',\n",
       " 'may',\n",
       " 'considered',\n",
       " 'includes',\n",
       " 'concentration',\n",
       " 'coursework',\n",
       " 'typically',\n",
       " 'courses',\n",
       " 'advanced',\n",
       " 'mathematics',\n",
       " 'typically',\n",
       " 'level',\n",
       " 'higher',\n",
       " 'calculus',\n",
       " 'differential',\n",
       " 'equations',\n",
       " 'discrete',\n",
       " 'mathematics',\n",
       " 'computer',\n",
       " 'science',\n",
       " 'algorithms',\n",
       " 'programming',\n",
       " 'data',\n",
       " 'structures',\n",
       " 'data',\n",
       " 'mining',\n",
       " 'artificial',\n",
       " 'intelligence',\n",
       " 'college',\n",
       " 'level',\n",
       " 'algebra',\n",
       " 'math',\n",
       " 'courses',\n",
       " 'intended',\n",
       " 'meet',\n",
       " 'basic',\n",
       " 'college',\n",
       " 'level',\n",
       " 'requirement',\n",
       " 'upper',\n",
       " 'level',\n",
       " 'math',\n",
       " 'courses',\n",
       " 'designated',\n",
       " 'elementary',\n",
       " 'basic',\n",
       " 'count',\n",
       " 'note',\n",
       " 'broader',\n",
       " 'range',\n",
       " 'degrees',\n",
       " 'considered',\n",
       " 'accompanied',\n",
       " 'certificate',\n",
       " 'data',\n",
       " 'science',\n",
       " 'accredited',\n",
       " 'college',\n",
       " 'university',\n",
       " 'relevant',\n",
       " 'experience',\n",
       " 'must',\n",
       " 'designing',\n",
       " 'implementing',\n",
       " 'machine',\n",
       " 'learning',\n",
       " 'data',\n",
       " 'mining',\n",
       " 'advanced',\n",
       " 'analytical',\n",
       " 'algorithms',\n",
       " 'programming',\n",
       " 'data',\n",
       " 'science',\n",
       " 'advanced',\n",
       " 'statistical',\n",
       " 'analysis',\n",
       " 'artificial',\n",
       " 'intelligence',\n",
       " 'software',\n",
       " 'engineering',\n",
       " 'experience',\n",
       " 'one',\n",
       " 'area',\n",
       " 'strongly',\n",
       " 'preferred',\n",
       " 'salary',\n",
       " 'range',\n",
       " 'full',\n",
       " 'performance',\n",
       " 'qualifications',\n",
       " 'listed',\n",
       " 'minimum',\n",
       " 'acceptable',\n",
       " 'considered',\n",
       " 'position',\n",
       " 'salary',\n",
       " 'offers',\n",
       " 'based',\n",
       " 'candidates',\n",
       " 'education',\n",
       " 'level',\n",
       " 'years',\n",
       " 'experience',\n",
       " 'relevant',\n",
       " 'position',\n",
       " 'also',\n",
       " 'take',\n",
       " 'account',\n",
       " 'information',\n",
       " 'provided',\n",
       " 'hiring',\n",
       " 'manager',\n",
       " 'organization',\n",
       " 'regarding',\n",
       " 'work',\n",
       " 'level',\n",
       " 'position',\n",
       " 'entry',\n",
       " \"bachelor's\",\n",
       " 'degree',\n",
       " 'plus',\n",
       " 'years',\n",
       " 'relevant',\n",
       " 'experience',\n",
       " \"master's\",\n",
       " 'degree',\n",
       " 'plus',\n",
       " 'year',\n",
       " 'relevant',\n",
       " 'experience',\n",
       " 'doctoral',\n",
       " 'degree',\n",
       " 'experience',\n",
       " \"associate's\",\n",
       " 'degree',\n",
       " 'plus',\n",
       " 'years',\n",
       " 'relevant',\n",
       " 'experience',\n",
       " 'may',\n",
       " 'considered',\n",
       " 'individuals',\n",
       " 'depth',\n",
       " 'experience',\n",
       " 'clearly',\n",
       " 'related',\n",
       " 'position',\n",
       " 'degree',\n",
       " 'must',\n",
       " 'mathematics',\n",
       " 'applied',\n",
       " 'mathematics',\n",
       " 'statistics',\n",
       " 'applied',\n",
       " 'statistics',\n",
       " 'machine',\n",
       " 'learning',\n",
       " 'data',\n",
       " 'science',\n",
       " 'operations',\n",
       " 'research',\n",
       " 'computer',\n",
       " 'science',\n",
       " 'degree',\n",
       " 'related',\n",
       " 'field',\n",
       " 'computer',\n",
       " 'information',\n",
       " 'systems',\n",
       " 'engineering',\n",
       " 'degree',\n",
       " 'physical',\n",
       " 'hard',\n",
       " 'sciences',\n",
       " 'physics',\n",
       " 'chemistry',\n",
       " 'biology',\n",
       " 'astronomy',\n",
       " 'science',\n",
       " 'disciplines',\n",
       " 'behavioral',\n",
       " 'social',\n",
       " 'life',\n",
       " 'may',\n",
       " 'considered',\n",
       " 'includes',\n",
       " 'concentration',\n",
       " 'coursework',\n",
       " 'typically',\n",
       " 'courses',\n",
       " 'advanced',\n",
       " 'mathematics',\n",
       " 'typically',\n",
       " 'level',\n",
       " 'higher',\n",
       " 'calculus',\n",
       " 'differential',\n",
       " 'equations',\n",
       " 'discrete',\n",
       " 'mathematics',\n",
       " 'computer',\n",
       " 'science',\n",
       " 'algorithms',\n",
       " 'programming',\n",
       " 'data',\n",
       " 'structures',\n",
       " 'data',\n",
       " 'mining',\n",
       " 'artificial',\n",
       " 'intelligence',\n",
       " 'college',\n",
       " 'level',\n",
       " 'algebra',\n",
       " 'math',\n",
       " 'courses',\n",
       " 'intended',\n",
       " 'meet',\n",
       " 'basic',\n",
       " 'college',\n",
       " 'level',\n",
       " 'requirement',\n",
       " 'upper',\n",
       " 'level',\n",
       " 'math',\n",
       " 'courses',\n",
       " 'designated',\n",
       " 'elementary',\n",
       " 'basic',\n",
       " 'count',\n",
       " 'note',\n",
       " 'broader',\n",
       " 'range',\n",
       " 'degrees',\n",
       " 'considered',\n",
       " 'accompanied',\n",
       " 'certificate',\n",
       " 'data',\n",
       " 'science',\n",
       " 'accredited',\n",
       " 'college',\n",
       " 'university',\n",
       " 'relevant',\n",
       " 'experience',\n",
       " 'must',\n",
       " 'two',\n",
       " 'following',\n",
       " 'designing',\n",
       " 'implementing',\n",
       " 'machine',\n",
       " 'learning',\n",
       " 'data',\n",
       " 'mining',\n",
       " 'advanced',\n",
       " 'analytical',\n",
       " 'algorithms',\n",
       " 'programming',\n",
       " 'data',\n",
       " 'science',\n",
       " 'advanced',\n",
       " 'statistical',\n",
       " 'analysis',\n",
       " 'artificial',\n",
       " 'intelligence',\n",
       " 'software',\n",
       " 'engineering',\n",
       " 'salary',\n",
       " 'range',\n",
       " 'senior',\n",
       " 'qualifications',\n",
       " 'listed',\n",
       " 'minimum',\n",
       " 'acceptable',\n",
       " 'considered',\n",
       " 'position',\n",
       " 'salary',\n",
       " 'offers',\n",
       " 'based',\n",
       " 'candidates',\n",
       " 'education',\n",
       " 'level',\n",
       " 'years',\n",
       " 'experience',\n",
       " 'relevant',\n",
       " 'position',\n",
       " 'also',\n",
       " 'take',\n",
       " 'account',\n",
       " 'information',\n",
       " 'provided',\n",
       " 'hiring',\n",
       " 'manager',\n",
       " 'organization',\n",
       " 'regarding',\n",
       " 'work',\n",
       " 'level',\n",
       " 'position',\n",
       " 'entry',\n",
       " \"bachelor's\",\n",
       " 'degree',\n",
       " 'plus',\n",
       " 'years',\n",
       " 'relevant',\n",
       " 'experience',\n",
       " \"master's\",\n",
       " 'degree',\n",
       " 'plus',\n",
       " 'years',\n",
       " 'relevant',\n",
       " 'experience',\n",
       " 'doctoral',\n",
       " 'degree',\n",
       " 'plus',\n",
       " 'years',\n",
       " 'relevant',\n",
       " 'experience',\n",
       " \"associate's\",\n",
       " 'degree',\n",
       " 'plus',\n",
       " 'years',\n",
       " 'relevant',\n",
       " 'experience',\n",
       " 'may',\n",
       " 'considered',\n",
       " 'individuals',\n",
       " 'depth',\n",
       " 'experience',\n",
       " 'clearly',\n",
       " 'related',\n",
       " 'position',\n",
       " 'degree',\n",
       " 'must',\n",
       " 'mathematics',\n",
       " 'applied',\n",
       " 'mathematics',\n",
       " 'statistics',\n",
       " 'applied',\n",
       " 'statistics',\n",
       " 'machine',\n",
       " 'learning',\n",
       " 'data',\n",
       " 'science',\n",
       " 'operations',\n",
       " 'research',\n",
       " 'computer',\n",
       " 'science',\n",
       " 'degree',\n",
       " 'related',\n",
       " 'field',\n",
       " 'computer',\n",
       " 'information',\n",
       " 'systems',\n",
       " 'engineering',\n",
       " 'degree',\n",
       " 'physical',\n",
       " 'hard',\n",
       " 'sciences',\n",
       " 'physics',\n",
       " 'chemistry',\n",
       " 'biology',\n",
       " 'astronomy',\n",
       " 'science',\n",
       " 'disciplines',\n",
       " 'behavioral',\n",
       " 'social',\n",
       " 'life',\n",
       " 'may',\n",
       " 'considered',\n",
       " 'includes',\n",
       " 'concentration',\n",
       " 'coursework',\n",
       " 'typically',\n",
       " 'courses',\n",
       " 'advanced',\n",
       " 'mathematics',\n",
       " 'typically',\n",
       " 'level',\n",
       " 'higher',\n",
       " 'calculus',\n",
       " 'differential',\n",
       " 'equations',\n",
       " 'discrete',\n",
       " 'mathematics',\n",
       " 'computer',\n",
       " 'science',\n",
       " 'algorithms',\n",
       " 'programming',\n",
       " 'data',\n",
       " 'structures',\n",
       " 'data',\n",
       " 'mining',\n",
       " 'artificial',\n",
       " 'intelligence',\n",
       " 'college',\n",
       " 'level',\n",
       " 'algebra',\n",
       " 'math',\n",
       " 'courses',\n",
       " 'intended',\n",
       " 'meet',\n",
       " 'basic',\n",
       " 'college',\n",
       " 'level',\n",
       " 'requirement',\n",
       " 'upper',\n",
       " 'level',\n",
       " 'math',\n",
       " 'courses',\n",
       " 'designated',\n",
       " 'elementary',\n",
       " 'basic',\n",
       " 'count',\n",
       " 'note',\n",
       " 'broader',\n",
       " 'range',\n",
       " 'degrees',\n",
       " 'considered',\n",
       " 'accompanied',\n",
       " 'certificate',\n",
       " 'data',\n",
       " 'science',\n",
       " 'accredited',\n",
       " 'college',\n",
       " 'university',\n",
       " 'relevant',\n",
       " 'experience',\n",
       " 'must',\n",
       " 'two',\n",
       " 'following',\n",
       " 'designing',\n",
       " 'implementing',\n",
       " 'machine',\n",
       " 'learning',\n",
       " 'data',\n",
       " 'mining',\n",
       " 'advanced',\n",
       " 'analytical',\n",
       " 'algorithms',\n",
       " 'programming',\n",
       " 'data',\n",
       " 'science',\n",
       " 'advanced',\n",
       " 'statistical',\n",
       " 'analysis',\n",
       " 'artificial',\n",
       " 'intelligence',\n",
       " 'software',\n",
       " 'engineering',\n",
       " 'salary',\n",
       " 'range',\n",
       " 'expert',\n",
       " 'qualifications',\n",
       " 'listed',\n",
       " 'minimum',\n",
       " 'acceptable',\n",
       " 'considered',\n",
       " 'position',\n",
       " 'salary',\n",
       " 'offers',\n",
       " 'based',\n",
       " 'candidates',\n",
       " 'education',\n",
       " 'level',\n",
       " 'years',\n",
       " 'experience',\n",
       " 'relevant',\n",
       " 'position',\n",
       " 'also',\n",
       " 'take',\n",
       " 'account',\n",
       " 'information',\n",
       " 'provided',\n",
       " 'hiring',\n",
       " 'manager',\n",
       " 'organization',\n",
       " 'regarding',\n",
       " 'work',\n",
       " 'level',\n",
       " 'position',\n",
       " 'entry',\n",
       " \"bachelor's\",\n",
       " 'degree',\n",
       " 'plus',\n",
       " 'years',\n",
       " 'relevant',\n",
       " 'experience',\n",
       " \"master's\",\n",
       " 'degree',\n",
       " 'plus',\n",
       " 'years',\n",
       " 'relevant',\n",
       " 'experience',\n",
       " 'doctoral',\n",
       " 'degree',\n",
       " 'plus',\n",
       " 'years',\n",
       " 'relevant',\n",
       " 'experience',\n",
       " \"associate's\",\n",
       " 'degree',\n",
       " 'plus',\n",
       " 'years',\n",
       " 'relevant',\n",
       " 'experience',\n",
       " 'may',\n",
       " 'considered',\n",
       " 'individuals',\n",
       " 'depth',\n",
       " 'experience',\n",
       " 'clearly',\n",
       " 'related',\n",
       " 'position',\n",
       " 'degree',\n",
       " 'must',\n",
       " 'mathematics',\n",
       " 'applied',\n",
       " 'mathematics',\n",
       " 'statistics',\n",
       " 'applied',\n",
       " 'statistics',\n",
       " 'machine',\n",
       " 'learning',\n",
       " 'data',\n",
       " 'science',\n",
       " 'operations',\n",
       " 'research',\n",
       " 'computer',\n",
       " 'science',\n",
       " 'degree',\n",
       " 'related',\n",
       " 'field',\n",
       " 'computer',\n",
       " 'information',\n",
       " 'systems',\n",
       " 'engineering',\n",
       " 'degree',\n",
       " 'physical',\n",
       " 'hard',\n",
       " 'sciences',\n",
       " 'physics',\n",
       " 'chemistry',\n",
       " 'biology',\n",
       " 'astronomy',\n",
       " 'science',\n",
       " 'disciplines',\n",
       " 'behavioral',\n",
       " 'social',\n",
       " 'life',\n",
       " 'may',\n",
       " 'considered',\n",
       " 'includes',\n",
       " 'concentration',\n",
       " 'coursework',\n",
       " 'typically',\n",
       " 'courses',\n",
       " 'advanced',\n",
       " 'mathematics',\n",
       " 'typically',\n",
       " 'level',\n",
       " 'higher',\n",
       " 'calculus',\n",
       " 'differential',\n",
       " 'equations',\n",
       " 'discrete',\n",
       " 'mathematics',\n",
       " 'computer',\n",
       " 'science',\n",
       " 'algorithms',\n",
       " 'programming',\n",
       " 'data',\n",
       " 'structures',\n",
       " 'data',\n",
       " 'mining',\n",
       " 'artificial',\n",
       " 'intelligence',\n",
       " 'college',\n",
       " 'level',\n",
       " 'algebra',\n",
       " 'math',\n",
       " 'courses',\n",
       " 'intended',\n",
       " 'meet',\n",
       " 'basic',\n",
       " 'college',\n",
       " 'level',\n",
       " 'requirement',\n",
       " 'upper',\n",
       " 'level',\n",
       " 'math',\n",
       " 'courses',\n",
       " 'designated',\n",
       " 'elementary',\n",
       " 'basic',\n",
       " 'count',\n",
       " 'note',\n",
       " 'broader',\n",
       " 'range',\n",
       " 'degrees',\n",
       " 'considered',\n",
       " 'accompanied',\n",
       " 'certificate',\n",
       " 'data',\n",
       " 'science',\n",
       " 'accredited',\n",
       " 'college',\n",
       " 'university',\n",
       " 'relevant',\n",
       " 'experience',\n",
       " 'must',\n",
       " 'two',\n",
       " 'following',\n",
       " 'designing',\n",
       " 'implementing',\n",
       " 'machine',\n",
       " 'learning',\n",
       " 'data',\n",
       " 'mining',\n",
       " 'advanced',\n",
       " 'analytical',\n",
       " 'algorithms',\n",
       " 'programming',\n",
       " 'data',\n",
       " 'science',\n",
       " 'advanced',\n",
       " 'statistical',\n",
       " 'analysis',\n",
       " 'artificial',\n",
       " 'intelligence',\n",
       " 'software',\n",
       " 'engineering',\n",
       " 'competencies',\n",
       " 'ideal',\n",
       " 'candidate',\n",
       " 'someone',\n",
       " 'desire',\n",
       " 'continual',\n",
       " 'learning',\n",
       " 'strong',\n",
       " 'problem',\n",
       " 'solving',\n",
       " 'analytic',\n",
       " 'interpersonal',\n",
       " 'skills',\n",
       " 'might',\n",
       " 'great',\n",
       " 'fit',\n",
       " 'team',\n",
       " 'following',\n",
       " 'describe',\n",
       " 'completed',\n",
       " 'degree',\n",
       " 'program',\n",
       " 'fields',\n",
       " 'mathematics',\n",
       " 'statistics',\n",
       " 'computer',\n",
       " 'science',\n",
       " 'computational',\n",
       " 'sciences',\n",
       " 'passion',\n",
       " 'rigorous',\n",
       " 'analysis',\n",
       " 'data',\n",
       " 'tenacity',\n",
       " 'integrity',\n",
       " 'persistence',\n",
       " 'willingness',\n",
       " 'learn',\n",
       " 'ability',\n",
       " 'solve',\n",
       " 'complex',\n",
       " 'problems',\n",
       " 'use',\n",
       " 'critical',\n",
       " 'thinking',\n",
       " 'reasoning',\n",
       " 'make',\n",
       " 'analytic',\n",
       " 'determinations',\n",
       " 'works',\n",
       " 'effectively',\n",
       " 'collaborative',\n",
       " 'environment',\n",
       " 'strong',\n",
       " 'communications',\n",
       " 'skills',\n",
       " 'technical',\n",
       " 'technical',\n",
       " 'audiences',\n",
       " 'desire',\n",
       " 'serve',\n",
       " 'million',\n",
       " 'fellow',\n",
       " 'americans',\n",
       " 'make',\n",
       " 'difference',\n",
       " 'world',\n",
       " 'events',\n",
       " 'pay',\n",
       " 'benefits',\n",
       " 'work',\n",
       " 'schedule',\n",
       " 'job',\n",
       " 'training',\n",
       " 'internal',\n",
       " 'nsa',\n",
       " 'courses',\n",
       " 'external',\n",
       " 'training',\n",
       " 'made',\n",
       " 'available',\n",
       " 'based',\n",
       " 'need',\n",
       " 'experience',\n",
       " 'selectee',\n",
       " 'monday',\n",
       " 'friday',\n",
       " 'basic',\n",
       " 'hr',\n",
       " 'day',\n",
       " 'requirements',\n",
       " 'flexible',\n",
       " 'apply',\n",
       " 'apply',\n",
       " 'position',\n",
       " 'please',\n",
       " 'click',\n",
       " 'apply',\n",
       " 'button',\n",
       " 'located',\n",
       " 'top',\n",
       " 'right',\n",
       " 'posting',\n",
       " 'completing',\n",
       " 'application',\n",
       " 'first',\n",
       " 'time',\n",
       " 'reviewing',\n",
       " 'previously',\n",
       " 'entered',\n",
       " 'information',\n",
       " 'clicking',\n",
       " 'submit',\n",
       " 'button',\n",
       " 'receive',\n",
       " 'confirmation',\n",
       " 'email',\n",
       " 'please',\n",
       " 'ensure',\n",
       " 'spam',\n",
       " 'filters',\n",
       " 'configured',\n",
       " 'accept',\n",
       " 'emails',\n",
       " 'noreply',\n",
       " 'intelligencecareers',\n",
       " 'gov',\n",
       " 'please',\n",
       " 'note',\n",
       " 'u',\n",
       " 'citizenship',\n",
       " 'required',\n",
       " 'applicants',\n",
       " 'reasonable',\n",
       " 'accommodations',\n",
       " 'provided',\n",
       " 'applicants',\n",
       " 'disabilities',\n",
       " 'application',\n",
       " 'hiring',\n",
       " 'process',\n",
       " 'appropriate',\n",
       " 'nsa',\n",
       " 'equal',\n",
       " 'opportunity',\n",
       " 'employer',\n",
       " 'abides',\n",
       " 'applicable',\n",
       " 'employment',\n",
       " 'laws',\n",
       " 'regulations',\n",
       " 'applicants',\n",
       " 'employees',\n",
       " 'subject',\n",
       " 'random',\n",
       " 'drug',\n",
       " 'testing',\n",
       " 'accordance',\n",
       " 'executive',\n",
       " 'order',\n",
       " 'employment',\n",
       " 'contingent',\n",
       " 'upon',\n",
       " 'successful',\n",
       " 'completion',\n",
       " 'security',\n",
       " 'background',\n",
       " 'investigation',\n",
       " 'polygraph',\n",
       " 'position',\n",
       " 'defense',\n",
       " 'civilian',\n",
       " 'intelligence',\n",
       " 'personnel',\n",
       " 'system',\n",
       " 'dcips',\n",
       " 'position',\n",
       " 'excepted',\n",
       " 'service',\n",
       " 'u',\n",
       " 'c',\n",
       " 'dod',\n",
       " 'components',\n",
       " 'dcips',\n",
       " 'positions',\n",
       " 'apply',\n",
       " 'veterans',\n",
       " 'preference',\n",
       " 'eligible',\n",
       " 'candidates',\n",
       " 'defined',\n",
       " 'section',\n",
       " 'title',\n",
       " 'usc',\n",
       " 'accordance',\n",
       " 'procedures',\n",
       " 'provided',\n",
       " ...]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set up data and target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "d3CTDfDQde2W"
   },
   "outputs": [],
   "source": [
    "data = clean_word(salary_df['combined_text'])\n",
    "#target = salary_df['adjusted_yearly_salary'] #continuous target\n",
    "target_class = salary_df['Salary_Bins'].astype('category') #categorical target"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Train-Test Split\n",
    "\n",
    "data is tokenized job description, target_class is salary classes. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(data, target_class, test_size=0.1, random_state=4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ulhtwoNAde2Z"
   },
   "source": [
    "## Model 1:  Use Word2Vec Embedding in Classification Models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "FxXpOkfxde2Z"
   },
   "source": [
    "We are going to use Word2Vec model in this section. Word2vec is a group of related models that are used to produce word embeddings. These models are shallow, two-layer neural networks that are trained to reconstruct linguistic contexts of words. Word2vec takes as its input a large corpus of text and produces a vector space, typically of several hundred dimensions, with each unique word in the corpus being assigned a corresponding vector in the space. Word vectors are positioned in the vector space such that words that share common contexts in the corpus are located close to one another in the space.\n",
    "\n",
    "We need to specify following parameter to the model:\n",
    "* the `size` of the word vectors we want to learn \n",
    "* the `window` size to use when training the model\n",
    "* `min_count`, which corresponds to the minimum number of times a word must be used in the corpus in order to be included in the training (for instance, `min_count=5` would only learn word embeddings for words that appear 5 or more times throughout the entire training set)\n",
    "* `workers`, the number of threads to use for training, which can speed up processing (`4` is typically used, since most processors nowadays have at least 4 cores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "lNyuoEtbde2a",
    "outputId": "9345c09a-c451-4c32-a380-fc5ed50dafed"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['data',\n",
       " 'scientist',\n",
       " 'position',\n",
       " 'requires',\n",
       " 'top',\n",
       " 'secret',\n",
       " 'ts',\n",
       " 'sci',\n",
       " 'clearance',\n",
       " 'polygraph',\n",
       " 'estimated',\n",
       " 'salary',\n",
       " 'position',\n",
       " 'level',\n",
       " 'level',\n",
       " 'subject',\n",
       " 'matter',\n",
       " 'expert',\n",
       " 'salary',\n",
       " 'range',\n",
       " 'per',\n",
       " 'year',\n",
       " 'b',\n",
       " 'corp',\n",
       " 'estimated',\n",
       " 'salary',\n",
       " 'ranges',\n",
       " 'position',\n",
       " 'level',\n",
       " 'min',\n",
       " 'salary',\n",
       " 'max',\n",
       " 'salary',\n",
       " 'level',\n",
       " 'subject',\n",
       " 'matter',\n",
       " 'expert',\n",
       " 'level',\n",
       " 'expert',\n",
       " 'level',\n",
       " 'senior',\n",
       " 'level',\n",
       " 'full',\n",
       " 'performance',\n",
       " 'mandatory',\n",
       " 'requirements',\n",
       " 'java',\n",
       " 'development',\n",
       " 'experience',\n",
       " 'production',\n",
       " 'environment',\n",
       " 'required',\n",
       " 'software',\n",
       " 'developer',\n",
       " 'list',\n",
       " 'years',\n",
       " 'experience',\n",
       " 'demonstrated',\n",
       " 'job',\n",
       " 'experience',\n",
       " 'troubleshooting',\n",
       " 'user',\n",
       " 'issues',\n",
       " 'list',\n",
       " 'years',\n",
       " 'experience',\n",
       " 'demonstrated',\n",
       " 'job',\n",
       " 'experience',\n",
       " 'writing',\n",
       " 'system',\n",
       " 'automation',\n",
       " 'scripts',\n",
       " 'linux',\n",
       " 'environment',\n",
       " 'using',\n",
       " 'tools',\n",
       " 'bash',\n",
       " 'sh',\n",
       " 'python',\n",
       " 'ruby',\n",
       " 'perl',\n",
       " 'list',\n",
       " 'years',\n",
       " 'experience',\n",
       " 'demonstrated',\n",
       " 'job',\n",
       " 'experience',\n",
       " 'installing',\n",
       " 'configuring',\n",
       " 'patching',\n",
       " 'troubleshooting',\n",
       " 'rhel',\n",
       " 'based',\n",
       " 'sl',\n",
       " 'redhat',\n",
       " 'centos',\n",
       " 'linux',\n",
       " 'os',\n",
       " 'list',\n",
       " 'years',\n",
       " 'experience',\n",
       " 'demonstrated',\n",
       " 'job',\n",
       " 'experience',\n",
       " 'installing',\n",
       " 'configuring',\n",
       " 'patching',\n",
       " 'troubleshooting',\n",
       " 'middleware',\n",
       " 'components',\n",
       " 'linux',\n",
       " 'environment',\n",
       " 'mysql',\n",
       " 'tomcat',\n",
       " 'apache',\n",
       " 'etc',\n",
       " 'list',\n",
       " 'years',\n",
       " 'experience',\n",
       " 'strong',\n",
       " 'writing',\n",
       " 'speaking',\n",
       " 'editing',\n",
       " 'skills',\n",
       " 'along',\n",
       " 'ability',\n",
       " 'utilize',\n",
       " 'internal',\n",
       " 'social',\n",
       " 'media',\n",
       " 'tools',\n",
       " 'help',\n",
       " 'messaging',\n",
       " 'promotion',\n",
       " 'marketing',\n",
       " 'list',\n",
       " 'years',\n",
       " 'experience',\n",
       " 'ability',\n",
       " 'organize',\n",
       " 'prioritize',\n",
       " 'tight',\n",
       " 'deadlines',\n",
       " 'list',\n",
       " 'years',\n",
       " 'experience',\n",
       " 'system',\n",
       " 'engineering',\n",
       " 'experience',\n",
       " 'production',\n",
       " 'environment',\n",
       " 'required',\n",
       " 'system',\n",
       " 'engineer',\n",
       " 'list',\n",
       " 'years',\n",
       " 'experience',\n",
       " 'system',\n",
       " 'administration',\n",
       " 'experience',\n",
       " 'production',\n",
       " 'environment',\n",
       " 'required',\n",
       " 'system',\n",
       " 'engineer',\n",
       " 'list',\n",
       " 'years',\n",
       " 'experience',\n",
       " 'development',\n",
       " 'skills',\n",
       " 'experience',\n",
       " 'production',\n",
       " 'environment',\n",
       " 'required',\n",
       " 'system',\n",
       " 'engineer',\n",
       " 'list',\n",
       " 'years',\n",
       " 'experience',\n",
       " 'amazon',\n",
       " 'cloud',\n",
       " 'services',\n",
       " 'list',\n",
       " 'years',\n",
       " 'experience',\n",
       " 'devops',\n",
       " 'practices',\n",
       " 'list',\n",
       " 'years',\n",
       " 'experience',\n",
       " 'server',\n",
       " 'automation',\n",
       " 'tools',\n",
       " 'docker',\n",
       " 'jenkins',\n",
       " 'puppet',\n",
       " 'list',\n",
       " 'years',\n",
       " 'experience',\n",
       " 'docker',\n",
       " 'list',\n",
       " 'years',\n",
       " 'experience',\n",
       " 'kubernettes',\n",
       " 'list',\n",
       " 'years',\n",
       " 'experience',\n",
       " 'optional',\n",
       " 'requirements',\n",
       " 'b',\n",
       " 'corp',\n",
       " 'company',\n",
       " 'information',\n",
       " 'b',\n",
       " 'corp',\n",
       " 'small',\n",
       " 'defense',\n",
       " 'contracting',\n",
       " 'company',\n",
       " 'focuses',\n",
       " 'providing',\n",
       " 'optimum',\n",
       " 'environment',\n",
       " 'mission',\n",
       " 'focused',\n",
       " 'highly',\n",
       " 'skilled',\n",
       " 'consultants',\n",
       " 'support',\n",
       " 'united',\n",
       " 'states',\n",
       " 'america',\n",
       " 'intelligence',\n",
       " 'community',\n",
       " 'defense',\n",
       " 'organizations',\n",
       " 'b',\n",
       " 'corp',\n",
       " 'provides',\n",
       " 'low',\n",
       " 'overhead',\n",
       " 'highly',\n",
       " 'efficient',\n",
       " 'high',\n",
       " 'salary',\n",
       " 'environment',\n",
       " 'allows',\n",
       " 'employees',\n",
       " 'excel',\n",
       " 'meeting',\n",
       " 'client',\n",
       " 'needs',\n",
       " 'b',\n",
       " 'corp',\n",
       " 'looking',\n",
       " 'information',\n",
       " 'technology',\n",
       " 'professionals',\n",
       " 'high',\n",
       " 'sense',\n",
       " 'personal',\n",
       " 'responsibility',\n",
       " 'self',\n",
       " 'motivation',\n",
       " 'mission',\n",
       " 'drive',\n",
       " 'b',\n",
       " 'corp',\n",
       " 'dedication',\n",
       " 'care',\n",
       " 'employees',\n",
       " 'reflected',\n",
       " 'outstanding',\n",
       " 'compensation',\n",
       " 'benefits',\n",
       " 'b',\n",
       " 'corp',\n",
       " 'provides',\n",
       " 'salaries',\n",
       " 'second',\n",
       " 'none',\n",
       " 'b',\n",
       " 'corp',\n",
       " 'benefits',\n",
       " 'reflect',\n",
       " 'company',\n",
       " 'policy',\n",
       " 'putting',\n",
       " 'employees',\n",
       " 'first',\n",
       " 'health',\n",
       " 'insurance',\n",
       " 'demonstrates',\n",
       " 'employer',\n",
       " 'coverage',\n",
       " 'providing',\n",
       " 'employees',\n",
       " 'plan',\n",
       " 'copay',\n",
       " 'coinsurance',\n",
       " 'hsa',\n",
       " 'allow',\n",
       " 'employees',\n",
       " 'accrue',\n",
       " 'health',\n",
       " 'savings',\n",
       " 'future',\n",
       " 'b',\n",
       " 'corp',\n",
       " 'maximum',\n",
       " 'flexibility',\n",
       " 'comp',\n",
       " 'makeup',\n",
       " 'time',\n",
       " 'policy',\n",
       " 'along',\n",
       " 'company',\n",
       " 'cafeteria',\n",
       " 'style',\n",
       " 'benefit',\n",
       " 'plan',\n",
       " 'allows',\n",
       " 'employees',\n",
       " 'maximize',\n",
       " 'benefit',\n",
       " 'dollars',\n",
       " 'reflects',\n",
       " 'b',\n",
       " 'corp',\n",
       " 'commitment',\n",
       " 'employees',\n",
       " 'compensation',\n",
       " 'outstanding',\n",
       " 'salaries',\n",
       " 'retirement',\n",
       " 'full',\n",
       " 'vanguard',\n",
       " 'k',\n",
       " 'plan',\n",
       " 'featuring',\n",
       " 'full',\n",
       " 'scope',\n",
       " 'investment',\n",
       " 'options',\n",
       " 'employer',\n",
       " 'matched',\n",
       " 'contribution',\n",
       " 'employee',\n",
       " 'salary',\n",
       " 'ability',\n",
       " 'max',\n",
       " 'k',\n",
       " 'savings',\n",
       " 'k',\n",
       " 'k',\n",
       " 'employees',\n",
       " 'receive',\n",
       " 'b',\n",
       " 'corp',\n",
       " 'phantom',\n",
       " 'stock',\n",
       " 'year',\n",
       " 'year',\n",
       " 'vesting',\n",
       " 'period',\n",
       " 'insurance',\n",
       " 'employer',\n",
       " 'paid',\n",
       " 'premiums',\n",
       " 'united',\n",
       " 'health',\n",
       " 'care',\n",
       " 'choice',\n",
       " 'plus',\n",
       " 'hsa',\n",
       " 'pos',\n",
       " 'gold',\n",
       " 'w',\n",
       " 'hsa',\n",
       " 'employer',\n",
       " 'funded',\n",
       " 'hsa',\n",
       " 'cover',\n",
       " 'health',\n",
       " 'care',\n",
       " 'deductible',\n",
       " 'health',\n",
       " 'insurance',\n",
       " 'copay',\n",
       " 'co',\n",
       " 'insurance',\n",
       " 'full',\n",
       " 'scope',\n",
       " 'protection',\n",
       " 'family',\n",
       " 'employer',\n",
       " 'premium',\n",
       " 'coverage',\n",
       " 'single',\n",
       " 'family',\n",
       " 'health',\n",
       " 'equity',\n",
       " 'hsa',\n",
       " 'b',\n",
       " 'corp',\n",
       " 'contributes',\n",
       " 'single',\n",
       " 'family',\n",
       " 'health',\n",
       " 'equity',\n",
       " 'hsa',\n",
       " 'cover',\n",
       " 'health',\n",
       " 'care',\n",
       " 'deductible',\n",
       " 'mutual',\n",
       " 'omaha',\n",
       " 'dental',\n",
       " 'vsp',\n",
       " 'vision',\n",
       " 'insurance',\n",
       " 'mutual',\n",
       " 'omaha',\n",
       " 'short',\n",
       " 'term',\n",
       " 'disability',\n",
       " 'salary',\n",
       " 'week',\n",
       " 'mutual',\n",
       " 'omaha',\n",
       " 'long',\n",
       " 'term',\n",
       " 'disability',\n",
       " 'salary',\n",
       " 'month',\n",
       " 'mutual',\n",
       " 'omaha',\n",
       " 'life',\n",
       " 'insurance',\n",
       " 'employee',\n",
       " 'referral',\n",
       " 'bonus',\n",
       " 'refer',\n",
       " 'friend',\n",
       " 'coworker',\n",
       " 'receive',\n",
       " 'per',\n",
       " 'year',\n",
       " 'every',\n",
       " 'year',\n",
       " 'person',\n",
       " 'works',\n",
       " 'b',\n",
       " 'corp',\n",
       " 'paid',\n",
       " 'time',\n",
       " 'pto',\n",
       " 'seven',\n",
       " 'weeks',\n",
       " 'leave',\n",
       " 'per',\n",
       " 'year',\n",
       " 'including',\n",
       " 'ten',\n",
       " 'federal',\n",
       " 'holidays',\n",
       " 'ability',\n",
       " 'purchase',\n",
       " 'additional',\n",
       " 'weeks',\n",
       " 'vacation',\n",
       " 'flexible',\n",
       " 'work',\n",
       " 'schedule',\n",
       " 'comp',\n",
       " 'time',\n",
       " 'customer',\n",
       " 'approval',\n",
       " 'tuition',\n",
       " 'training',\n",
       " 'free',\n",
       " 'cbtnuggets',\n",
       " 'online',\n",
       " 'training',\n",
       " 'account',\n",
       " 'online',\n",
       " 'courses',\n",
       " 'large',\n",
       " 'variety',\n",
       " 'topics',\n",
       " 'including',\n",
       " 'networking',\n",
       " 'security',\n",
       " 'virtualization',\n",
       " 'cloud',\n",
       " 'trusted',\n",
       " 'vendors',\n",
       " 'cisco',\n",
       " 'microsoft',\n",
       " 'google',\n",
       " 'train',\n",
       " 'anytime',\n",
       " 'anywhere',\n",
       " 'variety',\n",
       " 'devices',\n",
       " 'even',\n",
       " 'offline',\n",
       " 'transcender',\n",
       " 'practice',\n",
       " 'exams',\n",
       " 'virtual',\n",
       " 'labs',\n",
       " 'free',\n",
       " 'l',\n",
       " 'inux',\n",
       " 'academy',\n",
       " 'online',\n",
       " 'training',\n",
       " 'account',\n",
       " 'internal',\n",
       " 'tracking',\n",
       " 'j']"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "3IHAm2Bode2d"
   },
   "source": [
    "http://nadbordrozd.github.io/blog/2016/05/20/text-classification-with-word2vec/\n",
    "TF-IDF embedding, focus on important words.\n",
    "<br>https://scikit-learn.org/stable/modules/classes.html<br>\n",
    "\n",
    "#module-sklearn.feature_extraction.text\n",
    "<br>https://scikit-learn.org/stable/auto_examples/model_selection/grid_search_text_feature_extraction.html<br>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "cqMsrOjGde2d"
   },
   "source": [
    "### Word2Vec Model from scratch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create a Word2Vec model with 100 dimensions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "nw7rYGqdde2e"
   },
   "outputs": [],
   "source": [
    "w2v_model1 = Word2Vec(data,\n",
    "                size=64, #size is the number of dimensions of the N-dimensional space and Word2Vec maps the words into\n",
    "                window=10,\n",
    "                min_count=10, #min frequency appeared in the corpus\n",
    "                workers=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "UZ0XpKJkde2h",
    "outputId": "06eb2c99-3434-4668-d890-e61ea841fbb3"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2994179, 3460040)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w2v_model1.train(data,total_examples=w2v_model1.corpus_count, epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "60ZVS0yRde2o"
   },
   "outputs": [],
   "source": [
    "wv = w2v_model1.wv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "5qIEslLmde2r",
    "outputId": "7f6a50ea-bf60-4759-ab32-053e6572bfed"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Vocabulary size:3717\n"
     ]
    }
   ],
   "source": [
    "#After filtering for min frequency, there are 5514 words in the corpus\n",
    "words = list(wv.vocab)\n",
    "print('Vocabulary size:%d' % len(words))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "71xJIPOqde2t",
    "outputId": "71cd73e4-5d2f-410c-bcca-3a4ff093e87c",
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('structured', 0.6605768203735352),\n",
       " ('unstructured', 0.6462306380271912),\n",
       " ('extract', 0.6209700107574463),\n",
       " ('raw', 0.6112159490585327),\n",
       " ('manipulate', 0.6002713441848755),\n",
       " ('sets', 0.591453492641449),\n",
       " ('etl', 0.5882083177566528),\n",
       " ('extraction', 0.5832967162132263),\n",
       " ('visualization', 0.5795661211013794),\n",
       " ('datasets', 0.5696102380752563)]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wv.most_similar('data')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "aTdsdQNxde2v"
   },
   "source": [
    "To be able to use in a pipeline, need fit and transform"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "QBpVqOQnde2v",
    "outputId": "a0718a80-fc40-4838-b0fe-b953f590cd63"
   },
   "outputs": [],
   "source": [
    "#wrap into a dictionary to use in the pipeline\n",
    "w2v = dict(zip(w2v_model1.wv.index2word, w2v_model1.wv.vectors))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "BGgMRpmxde2y",
    "outputId": "393b5dc6-2d7a-445a-a966-61ff23fd6301",
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3717"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(w2v.values())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "SbWOxcOade20"
   },
   "source": [
    "Averaging word vectors for all words in a text."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "DULSNPEFde21"
   },
   "outputs": [],
   "source": [
    "#Averaging word vectors for all words in a text.\n",
    "\n",
    "class MeanEmbeddingVectorizer(object):\n",
    "    def __init__(self, word2vec):\n",
    "        self.word2vec = word2vec\n",
    "        # if a text is empty we should return a vector of zeros\n",
    "        # with the same dimensionality as all the other vectors\n",
    "        self.dim = len(word2vec.values())\n",
    "\n",
    "    def fit(self, X, y):\n",
    "        return self\n",
    "\n",
    "    def transform(self, X):\n",
    "        return np.array([\n",
    "            np.mean([self.word2vec[w] for w in words if w in self.word2vec]\n",
    "                    or [np.zeros(self.dim)], axis=0)\n",
    "            for words in X\n",
    "        ])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Use Word2Vec in the pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Import libraries\n",
    "from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "from pprint import pprint\n",
    "from time import time\n",
    "import logging\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "a8e477ezde23"
   },
   "outputs": [],
   "source": [
    "#Use pipeline \n",
    "\n",
    "rf =  Pipeline([('Word2Vec Vectorizer', MeanEmbeddingVectorizer(w2v)),\n",
    "              ('Random Forest', RandomForestClassifier(n_estimators=50, criterion='entropy', verbose=True, n_jobs=3))])\n",
    "\n",
    "svc = Pipeline([('Word2Vec Vectorizer', MeanEmbeddingVectorizer(w2v)),\n",
    "              ('Support Vector Machine', SVC(kernel='rbf', C=0.5))])\n",
    "\n",
    "sgd = Pipeline([('Word2Vec Vectorizer', MeanEmbeddingVectorizer(w2v)),\n",
    "               ('Stochastic Gradient Descent', SGDClassifier(alpha=0.001, n_jobs=3))])\n",
    "\n",
    "#svr = Pipeline([('Word2Vec Vectorizer', MeanEmbeddingVectorizer(w2v)),\n",
    "              #('SVR-lin', SVR(kernel='rbf', C=0.5))])\n",
    "                \n",
    "#lr = Pipeline([('Word2Vec Vectorizer', MeanEmbeddingVectorizer(w2v)),\n",
    "              #('Linear Regression', LinearRegression(n_jobs=3))])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "1oM8V_MRde3A"
   },
   "outputs": [],
   "source": [
    "models = [('Random Forest', rf),\n",
    "          ('Support Vector Machine', svc),\n",
    "          ('Stochastic Gradient Descent', sgd)]\n",
    "          #('Linear Regression', lr)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "bqilR_tLde3C",
    "outputId": "28c64516-689b-4e27-8444-e208a04b41ff"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=3)]: Using backend LokyBackend with 3 concurrent workers.\n",
      "[Parallel(n_jobs=3)]: Done  50 out of  50 | elapsed:    0.1s finished\n",
      "[Parallel(n_jobs=3)]: Using backend ThreadingBackend with 3 concurrent workers.\n",
      "[Parallel(n_jobs=3)]: Done  44 tasks      | elapsed:    0.0s\n",
      "[Parallel(n_jobs=3)]: Done  50 out of  50 | elapsed:    0.0s finished\n",
      "[Parallel(n_jobs=3)]: Using backend LokyBackend with 3 concurrent workers.\n",
      "[Parallel(n_jobs=3)]: Done  50 out of  50 | elapsed:    0.1s finished\n",
      "[Parallel(n_jobs=3)]: Using backend ThreadingBackend with 3 concurrent workers.\n",
      "[Parallel(n_jobs=3)]: Done  44 tasks      | elapsed:    0.0s\n",
      "[Parallel(n_jobs=3)]: Done  50 out of  50 | elapsed:    0.0s finished\n"
     ]
    }
   ],
   "source": [
    "scores = [(name, cross_val_score(model, X_train, y_train, cv=2).mean()) for name, model, in models]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Y-FaU-O1de3E",
    "outputId": "3fec4d8f-517e-4460-f386-30f8592df0d2"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('Random Forest', 0.44110854503464203),\n",
       " ('Support Vector Machine', 0.384526558891455),\n",
       " ('Stochastic Gradient Descent', 0.3741339491916859)]"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Random Forest performed the best among three models. Explore the prediction probability."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "YbxuzGm8de25"
   },
   "outputs": [],
   "source": [
    "rf =  Pipeline([('Word2Vec Vectorizer', MeanEmbeddingVectorizer(w2v)),\n",
    "              ('Random Forest', RandomForestClassifier(n_estimators=100, criterion='entropy', min_samples_leaf=3, verbose=True, n_jobs=3))])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "I074RlvRde29",
    "outputId": "771787cc-a8e9-4dc6-ec85-dfa7051890a0"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=3)]: Using backend LokyBackend with 3 concurrent workers.\n",
      "[Parallel(n_jobs=3)]: Done 100 out of 100 | elapsed:    0.2s finished\n",
      "[Parallel(n_jobs=3)]: Using backend ThreadingBackend with 3 concurrent workers.\n",
      "[Parallel(n_jobs=3)]: Done  44 tasks      | elapsed:    0.0s\n",
      "[Parallel(n_jobs=3)]: Done 100 out of 100 | elapsed:    0.0s finished\n",
      "[Parallel(n_jobs=3)]: Using backend LokyBackend with 3 concurrent workers.\n",
      "[Parallel(n_jobs=3)]: Done 100 out of 100 | elapsed:    0.2s finished\n",
      "[Parallel(n_jobs=3)]: Using backend ThreadingBackend with 3 concurrent workers.\n",
      "[Parallel(n_jobs=3)]: Done  44 tasks      | elapsed:    0.0s\n",
      "[Parallel(n_jobs=3)]: Done 100 out of 100 | elapsed:    0.0s finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.4341801385681293"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_val_score(rf, X_train, y_train, cv=2).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=3)]: Using backend ThreadingBackend with 3 concurrent workers.\n",
      "[Parallel(n_jobs=3)]: Done  44 tasks      | elapsed:    0.2s\n",
      "[Parallel(n_jobs=3)]: Done 100 out of 100 | elapsed:    0.4s finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "         steps=[('Word2Vec Vectorizer',\n",
       "                 <__main__.MeanEmbeddingVectorizer object at 0x14088ffd0>),\n",
       "                ('Random Forest',\n",
       "                 RandomForestClassifier(bootstrap=True, ccp_alpha=0.0,\n",
       "                                        class_weight=None, criterion='entropy',\n",
       "                                        max_depth=None, max_features='auto',\n",
       "                                        max_leaf_nodes=None, max_samples=None,\n",
       "                                        min_impurity_decrease=0.0,\n",
       "                                        min_impurity_split=None,\n",
       "                                        min_samples_leaf=3, min_samples_split=2,\n",
       "                                        min_weight_fraction_leaf=0.0,\n",
       "                                        n_estimators=100, n_jobs=3,\n",
       "                                        oob_score=False, random_state=None,\n",
       "                                        verbose=True, warm_start=False))],\n",
       "         verbose=False)"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=3)]: Using backend ThreadingBackend with 3 concurrent workers.\n",
      "[Parallel(n_jobs=3)]: Done  44 tasks      | elapsed:    0.0s\n",
      "[Parallel(n_jobs=3)]: Done 100 out of 100 | elapsed:    0.0s finished\n"
     ]
    }
   ],
   "source": [
    "y_pred = rf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=3)]: Using backend ThreadingBackend with 3 concurrent workers.\n",
      "[Parallel(n_jobs=3)]: Done  44 tasks      | elapsed:    0.0s\n",
      "[Parallel(n_jobs=3)]: Done 100 out of 100 | elapsed:    0.0s finished\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.5463917525773195"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=3)]: Using backend ThreadingBackend with 3 concurrent workers.\n",
      "[Parallel(n_jobs=3)]: Done  44 tasks      | elapsed:    0.0s\n",
      "[Parallel(n_jobs=3)]: Done 100 out of 100 | elapsed:    0.0s finished\n"
     ]
    }
   ],
   "source": [
    "proba_matrix = rf.predict_proba(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(97, 6)"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "proba_matrix.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prediction Probability of each bin"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.make_scorer.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_proba = pd.DataFrame(data = proba_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.091841</td>\n",
       "      <td>0.079220</td>\n",
       "      <td>0.113393</td>\n",
       "      <td>0.325758</td>\n",
       "      <td>0.192347</td>\n",
       "      <td>0.197440</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.032742</td>\n",
       "      <td>0.171451</td>\n",
       "      <td>0.260940</td>\n",
       "      <td>0.313378</td>\n",
       "      <td>0.147190</td>\n",
       "      <td>0.074298</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.062048</td>\n",
       "      <td>0.207405</td>\n",
       "      <td>0.161230</td>\n",
       "      <td>0.305325</td>\n",
       "      <td>0.142853</td>\n",
       "      <td>0.121139</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.034095</td>\n",
       "      <td>0.088317</td>\n",
       "      <td>0.279976</td>\n",
       "      <td>0.301020</td>\n",
       "      <td>0.180111</td>\n",
       "      <td>0.116480</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.005000</td>\n",
       "      <td>0.026865</td>\n",
       "      <td>0.003333</td>\n",
       "      <td>0.078917</td>\n",
       "      <td>0.028357</td>\n",
       "      <td>0.857528</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>0.207183</td>\n",
       "      <td>0.211310</td>\n",
       "      <td>0.295929</td>\n",
       "      <td>0.179917</td>\n",
       "      <td>0.079107</td>\n",
       "      <td>0.026556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>0.224579</td>\n",
       "      <td>0.376417</td>\n",
       "      <td>0.131333</td>\n",
       "      <td>0.082972</td>\n",
       "      <td>0.077024</td>\n",
       "      <td>0.107675</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>0.096667</td>\n",
       "      <td>0.134798</td>\n",
       "      <td>0.194730</td>\n",
       "      <td>0.327996</td>\n",
       "      <td>0.186786</td>\n",
       "      <td>0.059024</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>0.111837</td>\n",
       "      <td>0.300563</td>\n",
       "      <td>0.230238</td>\n",
       "      <td>0.245897</td>\n",
       "      <td>0.111464</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>0.061742</td>\n",
       "      <td>0.078512</td>\n",
       "      <td>0.084611</td>\n",
       "      <td>0.424520</td>\n",
       "      <td>0.281734</td>\n",
       "      <td>0.068881</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>97 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           0         1         2         3         4         5\n",
       "0   0.091841  0.079220  0.113393  0.325758  0.192347  0.197440\n",
       "1   0.032742  0.171451  0.260940  0.313378  0.147190  0.074298\n",
       "2   0.062048  0.207405  0.161230  0.305325  0.142853  0.121139\n",
       "3   0.034095  0.088317  0.279976  0.301020  0.180111  0.116480\n",
       "4   0.005000  0.026865  0.003333  0.078917  0.028357  0.857528\n",
       "..       ...       ...       ...       ...       ...       ...\n",
       "92  0.207183  0.211310  0.295929  0.179917  0.079107  0.026556\n",
       "93  0.224579  0.376417  0.131333  0.082972  0.077024  0.107675\n",
       "94  0.096667  0.134798  0.194730  0.327996  0.186786  0.059024\n",
       "95  0.111837  0.300563  0.230238  0.245897  0.111464  0.000000\n",
       "96  0.061742  0.078512  0.084611  0.424520  0.281734  0.068881\n",
       "\n",
       "[97 rows x 6 columns]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_proba"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>97.000000</td>\n",
       "      <td>97.000000</td>\n",
       "      <td>97.000000</td>\n",
       "      <td>97.000000</td>\n",
       "      <td>97.000000</td>\n",
       "      <td>97.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.132672</td>\n",
       "      <td>0.182064</td>\n",
       "      <td>0.151991</td>\n",
       "      <td>0.224048</td>\n",
       "      <td>0.150317</td>\n",
       "      <td>0.158908</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.159003</td>\n",
       "      <td>0.142848</td>\n",
       "      <td>0.103842</td>\n",
       "      <td>0.150178</td>\n",
       "      <td>0.131460</td>\n",
       "      <td>0.223469</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.003333</td>\n",
       "      <td>0.001667</td>\n",
       "      <td>0.002500</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.034095</td>\n",
       "      <td>0.078583</td>\n",
       "      <td>0.062421</td>\n",
       "      <td>0.107444</td>\n",
       "      <td>0.067258</td>\n",
       "      <td>0.028040</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.079353</td>\n",
       "      <td>0.150063</td>\n",
       "      <td>0.134349</td>\n",
       "      <td>0.205524</td>\n",
       "      <td>0.130917</td>\n",
       "      <td>0.074298</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.167679</td>\n",
       "      <td>0.270440</td>\n",
       "      <td>0.227440</td>\n",
       "      <td>0.310440</td>\n",
       "      <td>0.200540</td>\n",
       "      <td>0.162897</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.838083</td>\n",
       "      <td>0.887571</td>\n",
       "      <td>0.549690</td>\n",
       "      <td>0.727660</td>\n",
       "      <td>0.939310</td>\n",
       "      <td>0.967262</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               0          1          2          3          4          5\n",
       "count  97.000000  97.000000  97.000000  97.000000  97.000000  97.000000\n",
       "mean    0.132672   0.182064   0.151991   0.224048   0.150317   0.158908\n",
       "std     0.159003   0.142848   0.103842   0.150178   0.131460   0.223469\n",
       "min     0.000000   0.000000   0.003333   0.001667   0.002500   0.000000\n",
       "25%     0.034095   0.078583   0.062421   0.107444   0.067258   0.028040\n",
       "50%     0.079353   0.150063   0.134349   0.205524   0.130917   0.074298\n",
       "75%     0.167679   0.270440   0.227440   0.310440   0.200540   0.162897\n",
       "max     0.838083   0.887571   0.549690   0.727660   0.939310   0.967262"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_proba.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_IoNvOQ6de3G"
   },
   "source": [
    "We got a test accuracy of 0.48 with a simple RandomForest model. Worthy to do some grid search."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random Forest with Grid Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "AQACO63Mde3I"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performing grid search...\n",
      "pipeline: ['Word2Vec Vectorizer', 'rf']\n",
      "parameters:\n",
      "{'rf__criterion': ('gini', 'entropy'),\n",
      " 'rf__max_features': ('auto', 'sqrt', 'log2'),\n",
      " 'rf__min_samples_split': (2, 6, 8),\n",
      " 'rf__n_estimators': (30, 40, 50)}\n",
      "Fitting 5 folds for each of 54 candidates, totalling 270 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:   22.3s\n",
      "[Parallel(n_jobs=-1)]: Done 192 tasks      | elapsed:  1.7min\n",
      "[Parallel(n_jobs=-1)]: Done 270 out of 270 | elapsed:  2.4min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done in 143.505s\n",
      "\n",
      "Best score: 0.415\n",
      "Best parameters set:\n",
      "\trf__criterion: 'gini'\n",
      "\trf__max_features: 'sqrt'\n",
      "\trf__min_samples_split: 8\n",
      "\trf__n_estimators: 30\n"
     ]
    }
   ],
   "source": [
    "#Grid Search: Word2Vec with Random Forest Classifier\n",
    "pipeline =  Pipeline([('Word2Vec Vectorizer', MeanEmbeddingVectorizer(w2v)),\n",
    "              ('rf', RandomForestClassifier(n_jobs=3))])\n",
    "parameters = {\n",
    "    'rf__n_estimators':(30,40,50),\n",
    "    'rf__criterion':(\"gini\",\"entropy\"),\n",
    "    'rf__min_samples_split':(2,6,8),\n",
    "    'rf__max_features':(\"auto\",\"sqrt\",\"log2\")\n",
    "    \n",
    "}\n",
    "                \n",
    "if __name__ == \"__main__\":\n",
    "    # multiprocessing requires the fork to happen in a __main__ protected\n",
    "    # block\n",
    "\n",
    "    # find the best parameters for both the feature extraction and the\n",
    "    # classifier\n",
    "    grid_search = GridSearchCV(pipeline, parameters, n_jobs=-1, verbose=1)\n",
    "\n",
    "    print(\"Performing grid search...\")\n",
    "    print(\"pipeline:\", [name for name, _ in pipeline.steps])\n",
    "    print(\"parameters:\")\n",
    "    pprint(parameters)\n",
    "    t0 = time()\n",
    "    grid_search.fit(data, target_class)\n",
    "    print(\"done in %0.3fs\" % (time() - t0))\n",
    "    print()\n",
    "\n",
    "    print(\"Best score: %0.3f\" % grid_search.best_score_)\n",
    "    print(\"Best parameters set:\")\n",
    "    best_parameters = grid_search.best_estimator_.get_params()\n",
    "    for param_name in sorted(parameters.keys()):\n",
    "        print(\"\\t%s: %r\" % (param_name, best_parameters[param_name]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_8woTy_5de3L"
   },
   "source": [
    "Model performance was improved a little bit. Try another grid search."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "yFyq9_2Cde3N"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performing grid search...\n",
      "pipeline: ['Word2Vec Vectorizer', 'rf']\n",
      "parameters:\n",
      "{'rf__class_weight': ('balanced', None),\n",
      " 'rf__max_samples': (None, 0.5, 0.8, 1),\n",
      " 'rf__min_samples_split': (4, 6)}\n",
      "Fitting 5 folds for each of 16 candidates, totalling 80 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:   20.6s\n",
      "[Parallel(n_jobs=-1)]: Done  80 out of  80 | elapsed:   35.7s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done in 36.116s\n",
      "\n",
      "Best score: 0.461\n",
      "Best parameters set:\n",
      "\trf__class_weight: 'balanced'\n",
      "\trf__max_samples: 0.8\n",
      "\trf__min_samples_split: 4\n"
     ]
    }
   ],
   "source": [
    "#Grid Search: Word2Vec with Random Forest Classifier\n",
    "pipeline =  Pipeline([('Word2Vec Vectorizer', MeanEmbeddingVectorizer(w2v)),\n",
    "              ('rf', RandomForestClassifier(n_estimators = 30, criterion='gini', n_jobs=3, max_features='sqrt'))])\n",
    "parameters = {\n",
    "    'rf__class_weight':(\"balanced\",None),\n",
    "    'rf__min_samples_split':(4,6),\n",
    "    'rf__max_samples':(None, 0.5,0.8,1)   \n",
    "}\n",
    "                \n",
    "if __name__ == \"__main__\":\n",
    "    # multiprocessing requires the fork to happen in a __main__ protected\n",
    "    # block\n",
    "\n",
    "    # find the best parameters for both the feature extraction and the\n",
    "    # classifier\n",
    "    grid_search = GridSearchCV(pipeline, parameters, n_jobs=-1, verbose=1)\n",
    "\n",
    "    print(\"Performing grid search...\")\n",
    "    print(\"pipeline:\", [name for name, _ in pipeline.steps])\n",
    "    print(\"parameters:\")\n",
    "    pprint(parameters)\n",
    "    t0 = time()\n",
    "    grid_search.fit(X_train, y_train)\n",
    "    print(\"done in %0.3fs\" % (time() - t0))\n",
    "    print()\n",
    "\n",
    "    print(\"Best score: %0.3f\" % grid_search.best_score_)\n",
    "    print(\"Best parameters set:\")\n",
    "    best_parameters = grid_search.best_estimator_.get_params()\n",
    "    for param_name in sorted(parameters.keys()):\n",
    "        print(\"\\t%s: %r\" % (param_name, best_parameters[param_name]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Results improved from 0.40 to 0.46. Best estimator set so far: \n",
    "Best parameters set:\n",
    "\n",
    "\t- rf__class_weight: None\n",
    "\t- rf__max_samples: 0.8\n",
    "\t- rf__min_samples_split: 4\n",
    "    - rf__criterion: 'gini'\n",
    "\t- rf__max_features: 'sqrt'\n",
    "\t- rf__n_estimators: 30\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We used Word2Vec model to vectorize job descriptions and then use word vectors as inputs (100 dimensions) in the Random Forest model. Next we will try sklearn build in Count Vectorizer and then transform a count matrix to a normalized TF-IDF representation."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  SGD with Grid Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performing grid search...\n",
      "pipeline: ['Word2Vec Vectorizer', 'clf']\n",
      "parameters:\n",
      "{'clf__alpha': (0.0001, 0.001, 0.01),\n",
      " 'clf__l1_ratio': (0.1, 0.3, 0.5),\n",
      " 'clf__max_iter': (30, 50, 80),\n",
      " 'clf__penalty': ('l2', 'elasticnet')}\n",
      "Fitting 5 folds for each of 54 candidates, totalling 270 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:   19.4s\n",
      "[Parallel(n_jobs=-1)]: Done 192 tasks      | elapsed:  1.6min\n",
      "[Parallel(n_jobs=-1)]: Done 270 out of 270 | elapsed:  2.1min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done in 128.568s\n",
      "\n",
      "Best score: 0.354\n",
      "Best parameters set:\n",
      "\tclf__alpha: 0.01\n",
      "\tclf__l1_ratio: 0.1\n",
      "\tclf__max_iter: 30\n",
      "\tclf__penalty: 'l2'\n"
     ]
    }
   ],
   "source": [
    "#Try SGDClassifier \n",
    "pipeline = Pipeline([\n",
    "    ('Word2Vec Vectorizer', MeanEmbeddingVectorizer(w2v)),\n",
    "    ('clf', SGDClassifier(n_jobs=3, early_stopping=True, validation_fraction=0.2))\n",
    "])\n",
    "\n",
    "# uncommenting more parameters will give better exploring power but will\n",
    "# increase processing time in a combinatorial way\n",
    "parameters = {\n",
    "    \n",
    "    'clf__alpha': (0.0001, 0.001, 0.01),\n",
    "    'clf__penalty': ('l2', 'elasticnet'),\n",
    "    'clf__l1_ratio':(0, 0.1, 0.3, 0.5),\n",
    "    'clf__max_iter': (100,200,300)\n",
    "}\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    # multiprocessing requires the fork to happen in a __main__ protected\n",
    "    # block\n",
    "\n",
    "    # find the best parameters for both the feature extraction and the\n",
    "    # classifier\n",
    "    grid_search = GridSearchCV(pipeline, parameters, n_jobs=-1, verbose=1)\n",
    "\n",
    "    print(\"Performing grid search...\")\n",
    "    print(\"pipeline:\", [name for name, _ in pipeline.steps])\n",
    "    print(\"parameters:\")\n",
    "    pprint(parameters)\n",
    "    t0 = time()\n",
    "    grid_search.fit(X_train, y_train)\n",
    "    print(\"done in %0.3fs\" % (time() - t0))\n",
    "    print()\n",
    "\n",
    "    print(\"Best score: %0.3f\" % grid_search.best_score_)\n",
    "    print(\"Best parameters set:\")\n",
    "    best_parameters = grid_search.best_estimator_.get_params()\n",
    "    for param_name in sorted(parameters.keys()):\n",
    "        print(\"\\t%s: %r\" % (param_name, best_parameters[param_name]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "SGD did not perform very well even after Grid Search"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "FEKb-LvAde3b"
   },
   "source": [
    "## Model 2: Count vectorizer + TF-IDF with GridSearch\n",
    "\n",
    "Tf means term-frequency while tf-idf means term-frequency times inverse document-frequency. This is a common term weighting scheme in information retrieval. The goal of using tf-idf instead of the raw frequencies of occurence of a token in a given document is to scale down the impact of tokens that occur very frequently in a given corpus and are hence empirically less informative than features that occur in a small fraction of the training corpus. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "BQIAq8Wgde3b",
    "outputId": "c8c2550e-3fc2-4727-f4f8-932fe0620fb2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: pprint in /Applications/anaconda3/lib/python3.7/site-packages (0.1)\r\n"
     ]
    }
   ],
   "source": [
    "!pip install pprint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Dj0YIEjode3d"
   },
   "outputs": [],
   "source": [
    "from pprint import pprint\n",
    "from time import time\n",
    "import logging\n",
    "\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.pipeline import Pipeline\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "We_u01BDde3f"
   },
   "outputs": [],
   "source": [
    "#The input needs to be raw text\n",
    "text = salary_df['combined_text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0       5\n",
       "2       1\n",
       "4       0\n",
       "6       2\n",
       "8       3\n",
       "       ..\n",
       "1003    3\n",
       "1004    3\n",
       "1005    3\n",
       "1006    2\n",
       "1007    2\n",
       "Name: Salary_Bins, Length: 963, dtype: category\n",
       "Categories (6, int64): [0, 1, 2, 3, 4, 5]"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target_class"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train-Test Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(text, target_class, test_size=0.1, random_state=4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SGD Classifier with Grid Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "rC8dlEVGde3j",
    "outputId": "c32d049f-70f8-43d1-de90-8066b76faf6a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performing grid search...\n",
      "pipeline: ['vect', 'tfidf', 'clf']\n",
      "parameters:\n",
      "{'clf__alpha': (0.0001, 0.001),\n",
      " 'clf__loss': ('hinge', 'log'),\n",
      " 'clf__max_iter': (100, 200, 500),\n",
      " 'vect__max_df': (0.2, 0.5),\n",
      " 'vect__min_df': (0.01, 0.05)}\n",
      "Fitting 5 folds for each of 48 candidates, totalling 240 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:   24.4s\n",
      "[Parallel(n_jobs=-1)]: Done 192 tasks      | elapsed:  1.8min\n",
      "[Parallel(n_jobs=-1)]: Done 240 out of 240 | elapsed:  2.1min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done in 126.840s\n",
      "\n",
      "Best score: 0.553\n",
      "Best parameters set:\n",
      "\tclf__alpha: 0.0001\n",
      "\tclf__loss: 'log'\n",
      "\tclf__max_iter: 500\n",
      "\tvect__max_df: 0.5\n",
      "\tvect__min_df: 0.01\n"
     ]
    }
   ],
   "source": [
    "#Try SGDClassifier \n",
    "pipeline = Pipeline([\n",
    "    ('vect', CountVectorizer(max_df=0.5, min_df = 5, ngram_range=(1,2), stop_words='english')),\n",
    "    ('tfidf', TfidfTransformer()),\n",
    "    ('clf', SGDClassifier(n_jobs=3))\n",
    "])\n",
    "\n",
    "# uncommenting more parameters will give better exploring power but will\n",
    "# increase processing time in a combinatorial way\n",
    "parameters = {\n",
    "    'vect__max_df':(0.2,0.5),\n",
    "    'vect__min_df':(0.01,0.05),\n",
    "    #'tfidf__use_idf':(True, False),\n",
    "    #'tfidf__smooth_idf':(True, False),\n",
    "    'clf__alpha': (0.0001,0.001),\n",
    "    'clf__loss': ('hinge', 'log'), \n",
    "    #'clf__penalty': ('l2', 'elasticnet'),\n",
    "    'clf__max_iter': (100,200,500)\n",
    "}\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    # multiprocessing requires the fork to happen in a __main__ protected\n",
    "    # block\n",
    "\n",
    "    # find the best parameters for both the feature extraction and the\n",
    "    # classifier\n",
    "    grid_search = GridSearchCV(pipeline, parameters, n_jobs=-1, verbose=1)\n",
    "\n",
    "    print(\"Performing grid search...\")\n",
    "    print(\"pipeline:\", [name for name, _ in pipeline.steps])\n",
    "    print(\"parameters:\")\n",
    "    pprint(parameters)\n",
    "    t0 = time()\n",
    "    grid_search.fit(X_train, y_train)\n",
    "    print(\"done in %0.3fs\" % (time() - t0))\n",
    "    print()\n",
    "\n",
    "    print(\"Best score: %0.3f\" % grid_search.best_score_)\n",
    "    print(\"Best parameters set:\")\n",
    "    best_parameters = grid_search.best_estimator_.get_params()\n",
    "    for param_name in sorted(parameters.keys()):\n",
    "        print(\"\\t%s: %r\" % (param_name, best_parameters[param_name]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "SGD = Pipeline([\n",
    "    ('vect', CountVectorizer(max_df=0.5, min_df=0.01, ngram_range=(1,3), stop_words='english')),\n",
    "    ('tfidf', TfidfTransformer()),\n",
    "    ('clf', SGDClassifier(alpha=0.001, max_iter=500, n_jobs=3))\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "classifier = SGD.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = SGD.predict(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Confusion Matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[11,  1,  0,  0,  0,  0],\n",
       "       [ 1, 13,  2,  1,  0,  0],\n",
       "       [ 1,  4,  7,  1,  0,  0],\n",
       "       [ 0,  2,  3, 16,  1,  0],\n",
       "       [ 0,  0,  0,  2,  9,  1],\n",
       "       [ 1,  2,  1,  1,  3, 13]])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Confusion matrix, without normalization\n",
      "[[11  1  0  0  0  0]\n",
      " [ 1 13  2  1  0  0]\n",
      " [ 1  4  7  1  0  0]\n",
      " [ 0  2  3 17  0  0]\n",
      " [ 0  0  0  2  9  1]\n",
      " [ 1  2  2  1  2 13]]\n",
      "Normalized confusion matrix\n",
      "[[0.92 0.08 0.   0.   0.   0.  ]\n",
      " [0.06 0.76 0.12 0.06 0.   0.  ]\n",
      " [0.08 0.31 0.54 0.08 0.   0.  ]\n",
      " [0.   0.09 0.14 0.77 0.   0.  ]\n",
      " [0.   0.   0.   0.17 0.75 0.08]\n",
      " [0.05 0.1  0.1  0.05 0.1  0.62]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEWCAYAAACKSkfIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO2deZwUxfn/3x8uuVQQFJVDAREVDwRFooLGg4jxjhpv8dYYlWg0aoyiJtF4GzU/vxgPvEE8YqJRASNEUSPgBYoCisohtyiHCMvz+6NqsBlmd2d3Z3aGnee9r35Nd3XVU58+9unqquoqmRmO4zhOaVCv0AIcx3Gc2sOdvuM4TgnhTt9xHKeEcKfvOI5TQrjTdxzHKSHc6TuO45QQ7vRLAElNJP1T0mJJT9XAzomSXsmltkIhqY+kT4olP0lbSzJJDWpL0/qCpOmSDojrV0r6ex7yuFfSH3JttxiR99MvHiSdAFwMbAd8B7wH/MnMXq+h3ZOBC4A9zWxVjYUWOZIM6GJmUwutpTwkTQfONLORcXtr4HOgYa6vkaSHgBlmdlUu7dYW6ecqB/YGRHt758Le+oaX9IsESRcDdwB/BtoAHYC/AYfnwPxWwKel4PCzwUvT+cPP7XqAmflS4AXYGFgCHFNBnA0ID4VZcbkD2CDu2xeYAVwCzAVmA6fFfdcCPwArYx5nAIOARxO2twYMaBC3BwCfEd42PgdOTIS/nki3J/AOsDj+7pnY9xpwPfBGtPMK0LqcY0vpvyyh/wjgYOBTYCFwZSJ+L+BN4JsY926gUdw3Jh7L0ni8v0zY/x3wNfBIKiym6Rzz6BG3twTmA/tmce2GAJfE9bYx71/F7W2iXaXl9wiwGlgeNV6WuAanAl/G/H+f5fVf67rEMIv5nx2v/Q8xr3+WcxwGnAtMARYB9/BjTUA94Crgi3h9HgY2Trt3zoi6xyTCTgO+ivbOBXYHPojX7e5E3p2BV4EF8bgfA1ok9k8HDojrg4j3brzuSxLLKmBQ3Hc5MI1w730EHBnDtwe+B8pimm9i+EPAHxN5ngVMjdfveWDLbM7V+rAUXIAvBnBQvGEbVBDnOuAtYDNgU2AscH3ct29Mfx3QkOAslwEt4/41/yjlbKf+SRsAzYBvga5x3xZAt7g+gOhcgE3iDX9yTHd83G4V978W/+m2BZrE7RvLObaU/quj/rOAecDjwIZAt/iP2inG7wn0jvluDXwMDEzYM2CbDPb/QnCeTUg44RjnrGinKfAycEuW1+50oiMFTojHPDSx7x8JDcn8phMdWdo1uC/q2wVYAWyfxfVfc10ynQPSHFo5x2HAv4AWhLfMecBBieOYCnQCmgPPAI+k6X6YcO80SYTdCzQG+sXr91zU35bw8Ngn2tgGODBem00JD447Mp0r0u7dRJzuUfOucfsYwsO7HuHBvxTYooLzteYcAfsRHj49oqa7gDHZnKv1YfHqneKgFTDfKq5+ORG4zszmmtk8Qgn+5MT+lXH/SjN7kVCK6VpNPauBHSU1MbPZZjYpQ5yfA1PM7BEzW2VmTwCTgUMTcR40s0/NbDkwjPCPWR4rCe0XK4EngdbAnWb2Xcx/ErAzgJmNN7O3Yr7Tgf8D9snimK4xsxVRz1qY2X2EktvbhAfd7yuxl2I00EdSPaAvcBOwV9y3T9xfFa41s+Vm9j7wPsH5Q+XXPxfcaGbfmNmXwH/48XqdCNxmZp+Z2RLgCuC4tKqcQWa2NO3cXm9m35vZKwSn+0TUPxP4L7ArgJlNNbMR8drMA26j8uu5BkmbEh4oF5jZu9HmU2Y2y8xWm9lQwrXtlaXJE4EHzGyCma2Ix/uT2O6SorxzVfS40y8OFgCtK6kP3ZLwep3iixi2xkbaQ2MZoVRWJcxsKaFkdC4wW9ILkrbLQk9KU9vE9tdV0LPAzMriespxzEnsX55KL2lbSf+S9LWkbwntIK0rsA0wz8y+ryTOfcCOwF3xn71SzGwa4QHbHehDKAHOktSV6jn98s5ZZdc/F1Ql7waEtqcUX2Wwl379yruem0l6UtLMeD0fpfLrSUzbEBgOPG5mTybCT5H0nqRvJH1DuK5Z2STteOODbgHVv7eLCnf6xcGbhNffIyqIM4vQIJuiQwyrDksJ1RgpNk/uNLOXzexAQol3MsEZVqYnpWlmNTVVhf9H0NXFzDYCriTUm1dEhd3UJDUn1JPfDwyStEkV9IwGjia0K8yM26cALQk9sKqsJwMVXf+1rqekta5nNfLKJu9VrO3Ea5LHDTH9zvF6nkTl1zPFXYR6+zU9kyRtRbhnf02obmwBTEzYrEzrWscrqRnhbbw27u28406/CDCzxYT67HskHSGpqaSGkvpLuilGewK4StKmklrH+I9WM8v3gL6SOkjamPD6CoCkNpIOizf6CkIptiyDjReBbSWdIKmBpF8COxBKuvlmQ0K7w5L4FnJe2v45hPrnqnAnMN7MzgReINRHAyBpkKTXKkg7muBgxsTt1whdZF9PvL2kU1WNFV3/94FukrpLakyo965JXpny/o2kjvHh+GdCu0WueoNtSGxUldQWuDSbRJLOIbxNnWBmqxO7mhEc+7wY7zRCST/FHKCdpEblmH4cOC2ezw0Ix/t2rEpc73GnXySY2W2EPvpXEW7WrwiO5LkY5Y/AOELvhw+BCTGsOnmNAIZGW+NZ21HXI/QCmkXoubAP8KsMNhYAh8S4Cwg9UA4xs/nV0VRFfktoNP2OUKIbmrZ/EDAkvtofW5kxSYcTGtPPjUEXAz0knRi32xN6IZXHaILjSjn91wkl7zHlpgil26uixt9WppEKrr+ZfUpo6B1JqLtO/67jfmCHmNdzVJ0HCD2OxhB6c31PeKjlimsJjaaLCQ/cZ7JMdzzhYTZL0pK4XGlmHwG3Et6g5wA7sfb1e5XQRvS1pHXuVzMbBfwBeJrQO6wzcFx1DqwY8Y+zHKcSJL0H7B8fdI6zXuNO33Ecp4Tw6h3HcZwSwp2+4zhOCeFO33Ecp4TwwZHWQxo2a2GNN0nvil04Om/arNAS1qG+su3m7TiZmTBh/Hwz27S66etvtJXZqnU+/s6ILZ/3spkdVN28qoI7/fWQxptsTveBOR9SvNo8e07vQktYh+aN/dZ2akaThkr/4rxK2KrlbNC10h7DAHz/3j3Zfi1cY/w/w3EcJy8IVHw16O70Hcdx8oGAevULrWId3Ok7juPkiyJsW3Kn7ziOkxe8esdxHKe08JK+4zhOiSC8pO84jlM6yEv6juM4JUUR9t4pvncPx3GcOkFsyM1mqcyS9ICkuZImpoVfIOkTSZMSEy5ViJf0Hcdx8oHIZfXOQ8DdwMNrzEs/BQ4nTDO5QtJm2Rhyp+84jpMvctSQa2ZjJG2dFnwecKOZrYhx5mZjy6t3HMdx8kKVqndaSxqXWM7OIoNtgT6S3pY0WtLu2ajykn6JcMkBXdijY0u+WbaSsx97F4C+27Ti5N4d6LBJUy548n0+nbukMNpueJxRYz+iVcvmjHr48oJoSGfk2I+44tbhlK1ezcmH78lvBvQrtKSi0+R6KkFA/awbcueb2W5VzKEB0BLoDewODJPUySqZDtFL+lVA0kGx0WSqpIzeSVKZpPfi8nwivGN8Ik+RNFRSoxg+KDUxtqTGkkZIuibX2l/5aA5XPjdprbDpC5Zx7b8m8+HMb3OdXZU4pv8ePHLLOQXVkKSsbDWX3jSMp+78FW8Nu4qnXxnP5M9muybXU3Wk7JbqMQN4xgL/A1YDlY7W6U4/SyTVB+4B+gM7AMdL2iFD1OVm1j0uhyXC/wLcbmZdgEXAGWn2GwFPA+PN7Npc6/9w1rd89/2qtcK+XLScGd9kN953PundvTMtNmpaaBlrGD9pOp3at2brdq1p1LABRx3YgxdHf+CaXE8VyV3vnXJ4DtgPQNK2QCNgfmWJ3OlnTy9gqpl9ZmY/AE8SWs4rRZIIF2d4DBoCHJGI0iDam2JmxVG/UcLMnreYtm1artnesk1LZs9bXEBFxafJ9WRJjkr6kp4A3gS6Spoh6QzgAaBT7Mb5JHBqZVU74HX6VaEt8FViewawR4Z4jSWNA1YRWtafA1oB35jZqkTatok0lwEjzWxgeZnHhp2zATZo2abaB+FUTqb/m0J/WFlsmlxPluSu987x5ew6qaq23OlnT6ZbKNNTtYOZzZLUCXhV0odApkrzZNrXgZ9I2tbMPs2UuZkNBgYDbNh+u0qf5k712XKzFsycs2jN9qw5i9i89cYFVFR8mlxPFtSsvj5vePVO9swA2ie22wHzE422hwGY2az4+xnwGrAroZ6thaQGibSzErbGAAOBf0vaMq9H4VRKjx22YtqX8/hi5nx+WLmKZ0ZMoH/fnV2T66k69epnt9QiXtLPnneALpI6AjOB44ATko2ukloCy+LXca2BvYCbzMwk/Qc4mlj3BvwjadzMnpa0KfCSpL5m9k0uxV95UFd2brcxGzduwOOn787Db3/Jd9+v4vx9OrFxk4b88fAdmDZvKVek9fCpDc4fNIS33p3GwsVL2P2oa7jk9P4cd0jh5t1t0KA+N112LL+48B7KyowTD+vN9p23KJieYtTkerKhOMfTVxb1/k5E0sHAHUB94AEz+1Pa/j2B/yN0naoH3GFm98d9nQgOfxPgXeCk+HAYBCwxs1tivEGERt9+ZvZ9Jh0btt/OfGL0ivGJ0Z2a0qShxlej7/wa6m3cwTbY+7dZxf3+xYtqlFdV8P+MKmBmLwIvVrB/LLBTOfs+I/QASg8flGF7UHo8x3HWM3w8fcdxnFKiOKt33Ok7juPkiyIcT9+dvuM4Tr4owi6b7vQdx3Hygbx6x3Ecp7Twkr7jOE7pIHf6juM4pUGYLdGdvuM4TmkgoXru9B3HcUoGL+k7juOUEO70HcdxSgh3+o7jOKWCyDwLR4Fxp78e0nnTZkU1smXPK/9daAnrMPrqAwstYS2KbdTPYtNTFxHKWUlf0gPAIcBcM9sxbd9vgZuBTc3M58h1HMcpFPXq1ctqyYKHgIPSAyW1Bw4EvsxaU7YRHcdxnKohKaulMsxsDLAww67bCXNsZz0xir/jOY7j5IOq1em3ljQusT04zotdvvkwRetMM3u/KtVI7vQdx3HyRBWc8fyqzJwlqSnwe6BfVTW503ccx8kDuWzIzUBnoCOQKuW3AyZI6mVmX1eU0J2+4zhOnsjXMAxm9iGw2Zp8pOnAbt57x3Ecp1Aodw25kp4A3gS6Spoh6YzqyvKSvuM4Tp7IVfWOmR1fyf6ts7XlTt9xHCdP+DAMjuM4JUKeG3KrjTt9x3GcfFF8Pt+dvuM4Tl4Q2Q6xUKu403ccx8kTXr3jOI5TShSfz3enX4pccsPjjBr7Ea1aNmfUw5cXRMOff7kL++7QhgVLVnDozaMBuOigruy/4+asNmPBkh+44ol3mfvtilrXNnvuN1x+0xPMX/gdqieOPbg3pxzVp9Z1JCmGa5bOyLEfccWtwylbvZqTD9+T3wyo8ogAdVoPFGdJv/gqnGoJSdMlfSjpvdRAR5I2kTRC0pT42zKGD5B0d1yvJ2mIpAeUdkUlnRLtpRaTtGPct7ukiZKmSro9keZRSUfE9daSPpB0Sj6P/Zj+e/DILefkM4tKeeadrzhz8Ntrhf39P9M47JbRHHHrGF77aA7n99u2INrq16/HZeccygsPXMbQv17A48+/wdQvKvyyPe8UwzVLUla2mktvGsZTd/6Kt4ZdxdOvjGfyZ7NdT4JsP8yq7QdDyTr9yE/NrHtioKPLgVFm1gUYFbfXEJ38vUBD4EwzW2s4UzN7ONrrDpwGTDGziXH3vTGsC9BN0lqzfEhqAbwM3GVmD+f0KNPo3b0zLTZqms8sKmXcZwtZvOyHtcKWrli1Zr1Jo/pY1oPF5pbNWm1Ety7tAGjWtDGdO7RhzvxvCyMmUgzXLMn4SdPp1L41W7drTaOGDTjqwB68OPoD15OGO/3i53BgSFwfAhyRtv9OoBVwipmtrsTW8cATsGaig8Zm9k58UDySZntD4CVgiJndV7NDWL8Z2H87XvvDARzaoy13vvRJoeUw8+uFfDx1Jrts16HQUoqK2fMW07ZNyzXbW7Zpyex5i11PGqqnrJbapJSdvgGvSBov6ewY1sbMZgPE380S8U8AegLHmdkqKiC+ERxLdPpAW+CrRJQZMSzFncCrZvbXCmyeLWmcpHEL5lc6ptJ6yx3/nsy+14/knxNmctLeWxdUy9LlK7jwuiFcft7hNG/WuKBaig3L8BpWyOrrYtPzowYv6RcTe5lZD6A/cL6kvpXEnwBsBfTKwvaewCIzmxy3M13V5F06CjhSUuvyDJrZYDPbzcx2a9W63Gh1hn9NmEm/nbcoWP4rV5Vx0bVDOHS/HvTrs1PBdBQrW27WgplzFq3ZnjVnEZu33tj1JMnhgGu5pGSdvpnNir9zgWcJznyOpC0A4u/cRJLJhNL7UEndYpyjE4223RNxj+PHUj6Ekn37xHY7YFZi+zHgAeBFSc1ycXzrI1u1/vHQ9+vWhs/mLimIDjPjqluH0alDGwYcvU9BNBQ7PXbYimlfzuOLmfP5YeUqnhkxgf59d3Y9CUR428hmqU1KsstmdKz1zOy7uN4PuA54HjgVuDH+/iOZzszGSjoXeEFSXzMbDgxPs10fOBrYI5HuK0krJO0OjANOBm5Js32zpDbA05IONbOVuT3qHzl/0BDeencaCxcvYfejruGS0/tz3CG985VdRm49qQe9tmlFy2aNGH31Adz18if03b4NHTdthhnMXLSMa4Z/WKuaUkyYNJ3nR45n245bcOQ5twEw8PT+7LPH9gXRA8VxzZI0aFCfmy47ll9ceA9lZcaJh/Vm+86FezMrNj2B4hx7R5nqwuo6kjoRSvcQHnyPm9mfJLUChgEdCLPLH2NmCyUNIExQ8OuY/jTgUqCPmS1Is30AMMjM9k4L34NQmm8M/AsYaGYm6VFguJk9F9sChgD1gZPSewel6N6jp40c83amXQWh55X/LrSEdRh99YGVR6pFmjcurvJVsekpRpo01PiqTGGYTuPNt7WtTr0rq7if3nRQjfKqCiV55c3sM2CXDOELgP0zhD8EPJTYfhB4sBzbI4GRGcLfBrplCD8psW5AXvvoO45TSxSg6iYbSrZO33EcJ58IqFdPWS2V2gofg86VNDERdrOkyfGDzmfjtz6V4k7fcRwnT+SwIfch4KC0sBHAjma2M/ApcEU2htzpO47j5Ilcddk0szHAwrSwVxLfDL1F6BVYKSVZp+84jpN3qlan31pxDLDIYDMbXIXcTgeGZhPRnb7jOE4eEKrKJCrzq9t7R9LvgVWE730qxZ2+4zhOnsh37x1JpwKHAPuX18U7HXf6juM4eSKfH2dJOgj4HbCPmS3LNp035DqO4+SDLHvuZPNckPQE8CbQVdIMSWcAdxNG6B0Rh4K5NxtZXtJ3HMfJA2HsndyU9M3s+AzB91fHljt9x3GcPFGMX+S603ccx8kT2XxtW9u403ccx8kHKs6J0d3pOzVm2EV9Ci1hHZ77aFblkWqRk3r4dIulRmo8/WLDnb7jOE5eKM7x9N3pO47j5Iki9Pnu9B3HcfKCvCHXcRynZMhlP/1c4k7fcRwnT7jTdxzHKSGK0Oe703ccx8kXXtJ3HMcpFYp0YnR3+o7jOHkgTKJSfF7fnb7jOE6eqFeERX13+o7jOHmiCH2+O33HcZx8oPVtwDVJG1WU0My+zb0cx3GcukMRVulXWNKfBBjhw7IUqW0DfNjA9ZRLbnicUWM/olXL5ox6+PJCy1lDWdlqzvnd/6P1Jhtx45UnF0zHvDkLeWLIC2u2F85fzAEH78ne+/YomKZivGYjx37EFbcOp2z1ak4+fE9+M6Cf60kjVw25kh4gTIA+18x2jGGbAEOBrYHpwLFmtqhSTeXtMLP2ZtYh/rZP216vHb6krnFOydTyraSBkjaRNELSlPjbMsYfIOnuuF5P0hBJDyjt3U3SBnHfh9Fu38S+3SVNlDRV0u2J8EclHRHXW0v6QNIp+Tz+Y/rvwSO3nJPPLKrF0y++yVbtNi20DDZtswkXXnYyF152Mr/+7Yk0bNSAbjtvU1BNxXbNyspWc+lNw3jqzl/x1rCrePqV8Uz+bLbrSSBCD55s/rLgIeCgtLDLgVFm1gUYFbcrJauJ0SUdJ+nKuN5OUs9s0hUrZvaJmXU3s+5AT2AZ8CyVnMTo5O8FGgJnmpmlmT4X+MHMdiJcoNsSD4Z7gdOALkA3SQem2W4BvAzcZWYP5+5o16V398602KhpPrOoMnMXLOat8Z/w8/2L69aa+umXtGrdgpabVFjbmXeK7ZqNnzSdTu1bs3W71jRq2ICjDuzBi6M/cD1p1FN2S2WY2RhgYVrw4cCQuD4EOCIrTZVFiCXcnwKp9+1lBAdWV9gfmGZmX1D5SbwTaAWcYmarM9jagfCwwMy+BpYCu0pqDzQ2s3fig+KRNNsbAi8BQ8zsvtwc1vrF3Q++yDkn/6zoGr4+mPAJO/foWmgZRcfseYtp26blmu0t27Rk9rzFrieJwnj62SxAa0njEsvZWeTQxsxmA8TfzbKRlU1Jf08zOwf4PhpfCDTKxvh6wnHAE3G9opN4AuGt4DgzW1WOrfeBIyTVl9QZ2BVoD7QFvkrEmxHDUtwJvGpmfy1PpKSzUzfEgvnzsz+69YCx4ybTcuNmdO3ctvLItciqVWV8PHEaO3XfttBSio51X3IL2z2x2PQkNWSzAPPNbLfEMjhfmrLpsrlSUj1C4y2SWgGZSrnrHZIaAYcBV2QRfQKwHdALeKOcOPcBXYHxwOfAm8AqyFhpl7xLRwFHSrrNzDJ69HgTDAbo3qPnunf4eszET77kjXcm89aET/lh5SqWLVvBH+98iqsuOqaguj79+HO2bNeGDTdqVlAdxciWm7Vg5pwf2wxnzVnE5q03dj0JRN4/zpojaQszmy1pC2BuNomyKenfAzwNbCrpWuB14C/V11lU9AcmmNmcuD0nnjwynMTJwLHAUEndYpyjE43B3c1spZldFNsLjgRaAlMIJfv2CVvtgOQkro8BDwAvSio5D3P2if0YPvgyhv6/33L1wGPZdcdOBXf4AO+P/4RdvGonIz122IppX87ji5nz+WHlKp4ZMYH+fXd2PWnUq6eslmryPHBqXD8V+Ec2iSot6ZvZw5LGAwfEoGPMbGK1JBYfx/Nj1Q78eBJvJMNJNLOxks4FXpDU18yGA8NT+6PDNjNbJqk/sMTMPo37VkjaHRhHaB+5Jc32zZLaAE9LOtTMVub6YFOcP2gIb707jYWLl7D7Uddwyen9Oe6Q3vnKbr3khx9WMuWTLzjylwdUHrkWKLZr1qBBfW667Fh+ceE9lJUZJx7Wm+07b+F6EiSqbnJgS08A+xLq/mcA1xD81DBJZwBfAlmVlJSpLixDhjsDexOqJN4ws8I3i9cQSU0J9eydzGxxDGsFDCN8g/Al4QG3UNIAYDcz+3WMdxpwKdDHzBYkbHYGXiRUf80ATjezr+K+PQil+cbAv4CBZmaSHgWGm9lzsafPEKA+cFKG3kFAqN4ZOebt3J6QGjBtzpJCS1iHN2emd3QoLCf1KK5ezs0b+8f4ldGkocab2W7VTb9Jxx3swEGPZRV32IAeNcqrKlR65SX9ntCI+SyhmupxSY+Z2Q35FpdPzGwZoSdOMmwBoTdPetyHCP1kU9sPAg9miDeNUKefKb+3gW4Zwk9KrBuQ1z76juPUHkXQlrwO2TzuTwJ6RieJpD8RGirXa6fvOI6Tb4qtCzJk5/S/SIvXAPgsP3Icx3HqBqH3TqFVrEtFA67dTqjDXwZMkvRy3O5H6MHjOI7jlIfWv0lUUj10JgEvJMLfyp8cx3GcusN6Vb1jZvfXphDHcZy6xHpXvZMidkP8E2FcmcapcDPzb9Mdx3EqoBhL+tl8kfsQoXuiCF+wDgOezKMmx3GcOoGyXGqTbJx+UzN7GUI/dDO7ijDqpuM4jlMOEtSvp6yW2iSbLpsr4pei0+IQBDPJcghPx3GcUqYYq3eycfq/AZoDFxLq9jcGTs+nKMdxnLpAEfr8rAZcSw3y8h0/TqTiOI7jVIBQvodWrhYVfZz1LGuP+b4WZnZUXhQ5juPUBXI4ymYuqaikf3etqXCqRH2pqEZJbLNx48oj1TIHNi6uZqf2fQYWWsJaLHrH/71rg/WqTt/MRtWmEMdxnLqECAW0YqN4iouO4zh1jGL8IjebfvqO4zhONain7JZskPQbSZMkTZT0hKRq1atm7fQlbVCdDBzHcUqRMF2isloqt6W2hG7zu5nZjoTZ9Y6rjq5Knb6kXpI+JEzwjaRdJN1Vncwcx3FKiVyW9AnV8U0kNQCaArOqpSmLOH8FDgEWAJjZ+/gwDI7jOJWSmhy9soUw4fm4xHJ20o6ZzQRuIczdPRtYbGavVEdTNg259czsi7RXkLLqZOY4jlMqCGiQfe+d+RVNjC6pJXA40BH4BnhK0klm9mhVdWVT0v9KUi/AJNWXNBD4tKoZOY7jlBpVKOlXxgHA52Y2z8xWAs8Ae1ZHUzYl/fMIVTwdgDnAyBjmOI7jlIOU02EYvgR6S2oKLAf2B8ZVx1A2Y+/MpZqtxI7jOKVMrny+mb0taTgwAVgFvAsMro6tbGbOuo8MY/CY2dkZojuO4ziRXH6cZWbXANfU1E421TsjE+uNgSOBr2qaseM4Tl1GUOsTpGRDNtU7Q5Pbkh4BRuRNkeM4Tl2gan3wa43qjL3TEdgq10Kc2mXk2I+44tbhlK1ezcmH78lvBvQrmJbZc7/h8pueYP7C71A9cezBvTnlqD4F07Pih5Wc9bv/Y+XKVZStXs3+e+3EOSceWOs67vrDifxs7x2Zv+g79jzuzwDc/+fT6LJVGwA2bt6ExUuW0/fEG2tdGxTXPVSMeiCMqV9sZFOnv4gf6/TrAQuBy/MpKpdIugg4i/C2dZ+Z3SFpE2AosDUwHTjWzBZJGkD4zPnXkuoRJoQvA84wM0vY3IwwQfzu0ebAGL5htNuZ0NjynJn9Pu5rDDwKdAfmAb80sy8lHQD82syOiPFuAHYGjjSzH/JxTsrKVnPpTcN49u5fs2WbFux36s3077sT23XaIh/ZVUr9+vW47ICXNx4AACAASURBVJxD6dalHUuXfc8vfnUHe/bswjZbbV4QPY0aNuDeP59F0yYbsGpVGWdcdi979uzKTtt1qFUdT/zrLe4bNpp7rz1lTdgZVz64Zv36gUfy7ZLltaopRbHdQ8WmB4LDKcaSfoX99OPcuLsAm8alpZl1MrNhtSGupkjakeDwexGO4xBJXQgPrVFm1gUYRdpDLB73vUBD4Mykw48sA34P/C4t3IC/mFlXoAfwU0mpIuLZwNdmtg1wD3BDBr2DCA+SX+TL4QOMnzSdTu1bs3W71jRq2ICjDuzBi6M/yFd2lbJZq43o1qUdAM2aNqZzhzbMmf9twfRIommTMNTUqlVlrCorK8hkGGPfncaib5eVu//IA3rw9Mvja1HRjxTbPVRselLkeBiG3GiqaGd0ds+aWVlcyp1Jq0jZHnjLzJaZ2SpgNKEh+nBgSIwzBDgiLd2dQCvgFDNbnW7UzJaY2RvA9xnCR8f1FYRuVe3i7mSew4CfJdNK+h2wH3CYma1lN9fMnreYtm1artnesk1LZs9bnM8ss2bm1wv5eOpMdqnlUnU6ZWWrOeGCOznwpD+yR/cu7Ni1sHrS2XPXzsxd8B2ffTWvIPkX2z1UbHpS5GrAtVySzRe5/5PUI+9K8sNEoK+kVvGjhoOB9kAbM5sNEH+T0yydAPQEjosPimoRP5s+GHg1BrUl9nqKpfilklrEfX0Jk83/3MwyFu0knZ0al2Pe/Jr9o2d6dhfDXA9Ll6/gwuuGcPl5h9O8WWFn46pfvx6P33URLz50BZM+/Yqp078uqJ50ftFvN55+pVrf5uSEYruHik1PKv/69bJbapNys4sjuQHsTXD8n0iaIOldSRNqR17NMLOPgb8Qehu9BLxPqGuviAmEhupe1c1XUkNC3f6tZvZFKjiTxPg7hVCVtH95Ns1ssJntZma7bdp60+pKA2DLzVowc86iNduz5ixi89Yb18hmTVm5qoyLrh3Cofv1oF+fnQqqJcmGzZvQc6dOvDmheEYeqV+/Hof8dBeeHVG4f8Niu4eKTU+KevGr3MqWWtVUwb7/xd8jgK6EUusxwNHxd73AzO43sx5m1pfQCD0FmCNpC4D4OzeRZDJwLDBUUrcY52hJ78Wle0X5xfaA+4GJZpaciHQG4S0DSY2AZmaWev+cTTi/d0vqW8NDrpQeO2zFtC/n8cXM+fywchXPjJhA/7475zvbcjEzrrp1GJ06tGHA0fsUTEeKRYuX8F1sIP1+xUr+995Utm5XswdtLtm3V1emfDGHWXO/KZiGYruHik0P/NiQW2x1+hX13hGAmU2rJS15QdJmZjZXUgfgKOAnhG6npwI3xt9/JNOY2VhJ5wIvSOprZsOB4VlmeQPhI7bfpoU/H/N6h/BQWWtYVDObLOkXwDOS+ptZ3lqhGjSoz02XHcsvLryHsjLjxMN6s33nwvVymDBpOs+PHM+2HbfgyHNuA2Dg6f3ZZ4/tC6Jn/sLvuOb2YaxebaxebRzYZyf69Kp9LX//4wD26tmFVi2aM/Ff13Pj4Bd59Pk3Oapfz4I14KYotnuo2PSkKHQVUyZUXtuspBnAbeUlNLNy9xUTkv5LaJRdCVxsZqMktSI0pnYgDGR0jJktTHbZjGlPAy4F+pjZgjS7MwgTGTQEFhOqZlYAnwMfA6neN3ea2YOSmhC6bO4CzCe0GUzP0GWzP6Hn0D5mNj3TMfXsuZu98Xbh6nPT+fqbvLY7V4ulK6rdHJMXdjukuHo5L3rn7sojlThNGmp8RcMdV0aH7Xa2393/fFZxf713xxrlVRUqKunXB5qTuS56vcHM1vnKJzrwderPzewh4KHE9oOEvvqZ7LbLFE4558vMlgO/yBA+ksRQF2b2b/zjN8dZ7xHFWdKvyOnPNrPrak2J4zhOXULQoAi/zqq0Tt9xHMepOutjSb/c7oOO4zhO5dR2d8xsKNfpm9nC2hTiOI5T1yhCn1+tUTYdx3GcShDZDXlQ2xSjJsdxnPUf5faLXEktJA2XNFnSx5J+Uh1ZXtJ3HMfJA+GL3JzW79wJvGRmR8ev+ptWx4g7fcdxnDyRK5cvaSPCwIwDYM2gjdUaft2rdxzHcfKElN0CtE6NohuXs9NMdSJMvvRgHPTy75KaVUeTl/Qdx3HyQpXGyp9fyTAMDQgTM11gZm9LupMw+dMfqqrKS/qO4zh5INV7J5slC2YAM8zs7bg9nPAQqDLu9B3HcfJErnrvmNnXwFeSusag/YGPqqPJq3ecGrN5i8LOcrU+8PGIWwotYS1+NfzDQktYh5sOKcxQ2nlD5HoqxAuAx2LPnc+A06pjxJ2+4zhOHsj1x1lm9h5Q4+GX3ek7juPkidqe9Dwb3Ok7juPkieJz+e70Hcdx8oKA+l7SdxzHKR2K0Oe703ccx8kPQkVYweNO33EcJ094Sd9xHKdECF02i8/ru9N3HMfJB/KSvuM4TkmxXs2R6ziO41SfMIlKoVWsizt9x3GcPOG9dxzHcUqIIqzdcadfqowc+xFX3DqcstWrOfnwPfnNgH6up4j1zJ77DZff9ATzF36H6oljD+7NKUf1Kaimn27Tir06bgLAG58v5D9TFxRMyyU3PM6osR/RqmVzRj18ecF0pFOMJf06MZ6+pAckzZU0MRF2c5w1/gNJz0pqkdh3haSpkj6R9LNE+JLE+sGSpkjqkCG/GyXNkPRNWviZkuZJei8upyX2nRHtTZF0UiJ8RkqbpF6SPpe0cy7OS3mUla3m0puG8dSdv+KtYVfx9CvjmfzZ7Hxm6XpqSP369bjsnEN54YHLGPrXC3j8+TeY+sXXBdOzxUYbsFfHTfjLq1P588gp7LTFRmzavFHB9BzTfw8eueWcguWfiVSdfjZLbVInnD7wEHBQWtgIYEcz2xn4FLgCQNIOwHFAt5jmb5LqJxNK2h+4CzjIzL7MkN8/gN7laHnMzLrH5cForzVwJbB7THe9pI3T8uwODAWOMbMPsjrqajJ+0nQ6tW/N1u1a06hhA446sAcvjs5rlq6nhmzWaiO6dWkHQLOmjencoQ1z5n9bMD2bb7gBny9cxsoyY7XBlPlL6b7lRgXT07t7Z1ps1LRg+WckywlUaruHT51w+mY2BliYFvaKma2Km28B7eL64cCTZrbCzD4HpgK9Uukk9QHuA35uZtPKye9NoCrFrP7AS2b2jZktAF4FkvUF3YBngRPMbFwV7FaL2fMW07ZNyzXbW7Zpyex5i/OdrevJETO/XsjHU2eyy3brvITWGrO/XcE2rZvRrFF9GtYX3TbfkJZNGxZMT7GiLJfapFTq9E8nlKIB2hIeAilmxDCADQil+H3NbHI18zpW0n7AZOA3ZjYz2v+qnDwB/gkcFx8mGZF0NnA2QPsONftnN7MM9mtkska4nuxZunwFF143hMvPO5zmzQo3Y9nX361gxCfzuKBPR1asWs3Mb5ZTtu5pK2lC9U5ub5xYKzEOmGlmh1THRp0o6VeEpN8Dq4DHUkEZoqVu15XAWOCMamb3HNAxVimNBh7MIk+AkcCZksq9HmY22Mx2M7PdNm29aTXlBbbcrAUz5yxasz1rziI2b71xBSnyi+vJjpWryrjo2iEcul8P+vXZqdByGDt9ETeOmsrtoz9j6coy5n23otCSio48lPQvAj6uiaY67fQlnQocApxoPxbfZgDtE9HaAbPi+mrgWGB3SVdGG40SDbNXV5Sfmc03s9SdP5hQh19ZngDnEd4y7qrK8VWXHjtsxbQv5/HFzPn8sHIVz4yYQP++eW07dj01xMy46tZhdOrQhgFH71NQLSmabxCawlo2aUj3LTfina++qSRFCZJDry+pHfBz4O81kVRnq3ckHQT8DtjHzJYldj0PPC7pNmBLoAvwv9ROM1sm6RDgv5LmmNn9QPcs89zCzFLdPI4AJsX1l4BrYy+d+oSZ7C9OJC0jNC6/IulqM7uuiodbJRo0qM9Nlx3LLy68h7Iy48TDerN95y3ymaXrqSETJk3n+ZHj2bbjFhx5zm0ADDy9P/vsUbjJxM/+yVY0a1SfstXG0PdmsXzl6oJpOX/QEN56dxoLFy9h96Ou4ZLT+3PcIeX1tag9cly9cwdwGbBhTYzUCacv6QlgX6C1pBnANYTeOhsAI+I8lW+Z2blmNknSMOAjQrXP+WZWlrRnZgvjQ2OMpPlm9o+0/G4jvBFsFPO718z+CFwsqT/BiS8gVhOZ2TxJNxDq4gCuNrO1WgbNbLmkw2Kec8zs/3J0ejLSb69u9NurWz6zqBKup2J67tiRj0fcUmgZa3Hba58VWsIa7hl0aqElZKQKLr+1pGQnjsFmNniNnVAQnWtm4yXtWxNNdcLpm9nxGYLvryD+n4A/ZQhvnlj/CuhYTvqLWbukngq/FLi0nDT3EXoFpYe3S6wvAgpfWes4Tm7I3uvPN7PdKti/F3CYpIOBxoQC56NmdlIFaTJSp+v0HcdxCkWors/urzLM7Aoza2dmWxOqgl+tjsOHOlLSdxzHKTp8PH3HcZzSIh8+38xeA16rbnp3+o7jOHlBqAiL+u70Hcdx8kQR+nx3+o7jOPmgEOPqZIM7fcdxnHxRhF7fnb7jOE6eKMZJVNzpO47j5Amv03ccxykVvJ++4zhOaeHVO47jOCWC8JK+4zhOSVGEPt+d/vpImRlLvl9VecRaopi0FCvNGxfXv9rVB3QptIR16HnlvwstIfcUodcvrjvRcRynDpHrOXJzgTt9x3GcPFF8Lt+dvuM4Tv4oQq/vTt9xHCcPpCZRKTbc6TuO4+QD/zjLcRyntChCn+9z5DqO4+SHMIlKNkullqT2kv4j6WNJkyRdVF1VXtJ3HMfJEzms3lkFXGJmEyRtCIyXNMLMPqqqIS/pO47j5AFVYakMM5ttZhPi+nfAx0Db6ujykr7jOE6+yL6k31rSuMT2YDMbnNGktDWwK/B2dSS503ccx8kTVeiyOd/MdqvUntQceBoYaGbfVkeTO33HcZw8kcsum5IaEhz+Y2b2THXtuNN3HMfJB4J6OXL6Cl187gc+NrPbamLLnX4JcskNjzNq7Ee0atmcUQ9fXmg5zJ77DZff9ATzF36H6oljD+7NKUf1cT0J/Jqty59/uQv77tCGBUtWcOjNowG46KCu7L/j5qw2Y8GSH7jiiXeZ++2KWtW1Njkr6u8FnAx8KOm9GHalmb1YVUN5670j6QFJcyVNTAsfJGmmpPficnBi3xWSpkr6RNLPEuFLEusHS5oiqUOO9Q6QdHe+4hcTx/Tfg0duOafQMtZQv349LjvnUF544DKG/vUCHn/+DaZ+8bXrSeDXbF2eeecrzhy8dlvm3/8zjcNuGc0Rt47htY/mcH6/bWtVU5LUJCrZLJVhZq+bmcxsZzPrHpcqO3zIb5fNh4CDytl3e7pwSTsAxwHdYrq/SaqfTCRpf+Au4CAz+zJvyus4vbt3psVGTQstYw2btdqIbl3aAdCsaWM6d2jDnPnVaqOqk3rAr1kmxn22kMXLflgrbOmKH+d2aNKoPma1KmkdctVlM5fkzemb2RhgYRWSHA48aWYrzOxzYCrQK7VTUh/gPuDnZjYtPbGkXpLGSno3/naN4QMkPSPppfiGcFMizWmSPpU0mvD6tA7l2Y20j3Y/kXRNIs3FkibGZWAM+4ukXyXiDJJ0SVy/VNI7kj6QdG0VzlmdY+bXC/l46kx22S6nL3LVptj0FCPFdo4G9t+O1/5wAIf2aMudL31SUC25KunnkkJ9nPXr6OAekNQyhrUFvkrEmcGPHx9sAPwDOMLMJpdjczLQ18x2Ba4G/pzY1x34JbAT8Mv4SfMWwLUEZ38gsEM17PYCToz2j5G0m6SewGnAHkBv4CxJuwJPRg0pjgWektQP6BJtdQd6SuqbLkLS2ZLGSRq3YP78cqSu3yxdvoILrxvC5ecdTvNmjQstp+j0FCPFeI7u+Pdk9r1+JP+cMJOT9t66oFpyNQxDLimE0/9/QGeCg5sN3BrDMx156uVsJTAWOKMCuxsTnOhE4HZCNVGKUWa22My+Bz4CtiI45dfMbJ6Z/QAMrYbdEWa2wMyWA88Ae8flWTNbamZLYngfM3sX2EzSlpJ2ARbFKqp+cXkXmABsR3gIrH0izAab2W5mtlur1q0rOA3rJytXlXHRtUM4dL8e9OuzU6HlFJ2eYqTYz9G/Jsyk385bFFRDSVXvlIeZzTGzMjNbTaiuSVXhzADaJ6K2A2bF9dWEkvHukq4sx/T1wH/MbEfgUCBZ7Eg235fxY6+lbGr8KrKbnt6o+BoOB44mlPifjGECbki0cWxjZvdnoavOYGZcdeswOnVow4Cj9ym0nKLTU4wU6znaqnWzNev7dWvDZ3OXVBA7v2RbtVPb1Tu13mVT0hZmNjtuHgmkevc8Dzwu6TZgS0Jp93+pdGa2TNIhwH8lzcngGDcGZsb1AVlIeRu4U1Ir4FvgGOD9DPEqsnugpE2A5cARwOmEB9RDkm4kOPQjCV2tIDj6+4DWQOo/5WXgekmPmdkSSW2BlWY2N4tjqBbnDxrCW+9OY+HiJex+1DVccnp/jjukd76yq5QJk6bz/MjxbNtxC448J3RBHnh6f/bZY3vXE/Frti63ntSDXtu0omWzRoy++gDuevkT+m7fho6bNsMMZi5axjXDP6w1PZkoqUlUJD0B7EsYU2IGcE101DdJ6k4oFU8HzgEws0mShhGqX1YB55tZWdKmmS2UdBAwRtJ8M/tHYvdNwBBJFwOvVqbPzGZLGgS8SahmmgDUzxC1IruvA48A2wCPm9m4eOwP8eMD6++xaid1jBsCM1MPPjN7RdL2wJuxbm8JcBKQN6d/z6BT82W6WvTcsSMfj7il0DLWUGx6wK9ZJi55dMI6YcPf/ipDzAJSfD4fWaH7NDlVpnuPnjZyTLXGWsoLS75fVXmkEqd54+L6DrIYr9k+140otIS1+PKuw8ZnMx5OeVTl/3TTDRvWKK+qUFx3ouM4Tp1B1CvC+RLd6TuO4+SB1Be5xYZPouI4jlNCeEnfcRwnTxRjSd+dvuM4Tp4oqS6bjuM4JU0BPrzKBnf6juM4eaBYG3Ld6TuO4+QJr95xHMcpIYqxpO9dNh3HcfJELkfZlHRQnLtjqqRqz5npTt9xHCdf5Mjrx1kE7wH6E+b+OD7ONlhl3Ok7juPkAQH1pKyWLOgFTDWzz+L8H08SZhusMl6nvx7y/rsT5m+6YcMvcmCqNVBs03AVmybXUzHFpgdyp2mrmiSeMGH8y00aKtsZjxpLGpfYHmxmgxPbmWYW3KM6utzpr4eY2aa5sCNpXG2N7JctxabJ9VRMsemB4tFkZgfl0FxFMwtWCa/ecRzHKX4qmlmwSrjTdxzHKX7eAbpI6iipEXAcYbbBKuPVO6XN4Mqj1DrFpsn1VEyx6YHi1FQjzGyVpF8TpletDzxgZpOqY8tnznIcxykhvHrHcRynhHCn7ziOU0K4068jZPOJtqQySe/F5flEeEdJb0uaImlobChC0iBJv43rjSWNkHRNms3pkj6MNsfFsE1i3Cnxt2UMHyDp7rheT9IQSQ9Ia3+dIumUhM73JJmkHeO+3SVNjMd5eyLNo5KOkNQ16lku6QtJ30oamANNG8R9qWPtm9i3jiZJF0n6RtKXMf/WkibFeNXVsJmk1yQtlXRHInxDSS/G6z9J0p9i+rlxe3jUNjP+fiDpdUkvJGyk7H4i6WeJ8CWJ9YMT98hcSRMT+26M53p14rodLOlMSfMkzZK0QtLslH1JZ8T4UySdlLA/W1KLGKeXpM8l7Zx+P2dL8vzmI/76hjv9OoCy/0R7uZl1j8thifC/ALebWRdgEXBGmv1GwNPAeDO7NoPdn0abqb7RlwOjor1RcTtpT8C9QEPgTEtrWDKzh1M6gdOAKWaWcjD3xrAuQDdJB6ZpmQP8AFwIdAKWAc/WVBNwLvCDme0EHATclnDK6ZrOAs4C/g0MBI4AXiN8XPNoDTQsA34P/C4t3IC/mFlXoAfwU2Bi1NkS+NrMtgEeAf5nZjsTugB2ifneA/QEtohp/hbvqaS2/YG74v574m+SfwB/B1Yk7rEX475/A/OAjYA9o/3NgCvjMfUGbk7YLYt5dgeGAseY2Qc4OcGdft2g2p9oR0ezHzA8Bg0hOKkUDaK9KWaW7SBPh0c7mewB3Am0Ak4xs9WV2DoeeCJqbQ80NrN3okN8JM32hsBLwBAzuw/YH5hmZl/kQNMOBEeNmX0NLAV2LUfTL4C3CM6rGdAZmAZ0rIkGM1tiZm8A32cIHx3XVwDvEh7eCwmONpXn1UC/uP4x0ETS74CDgZvN7Fsz+xyYSrinAJDUB7gP+LmZTTOzMdF2UsObwBIy0wl40sxWJOyfT7hWEM5tM+AOM5sWw7oRHtYnAPUkjZX0bvztGnUNkPSMpJfiG8JNCc2nSfpU0mhgr0yi4lvEOnYj7aPdT5R4u5V0cXxbmyhpYAz7i6RfJeIMknRJXL9U0jvx7SpTganWcadfN8j0iXbbDPEaSxon6S1JKYfTCvjGzFaVk/YyYJWZDSwnbwNekTRe0tkxrI2ZzQaIv5sl4p9AKFUel8gzI/GBdCzR6WdxnHcCr5rZX+P2cYm0NdX0PnCEpPqSOgO7Ej6WyaSpAdAXaBQ1CfgyBxoqRaHK6GDg1RjUMKUvFgiWxqqT1FvA6cArwGdpx5A6rxsQSvFHmNnkLCRsEB3cA1ELhHN1nqRhktpG+52jrpT9x1j7q9N/AufEh8lkoK+Z7Up4cP05Ea878EtgJ+CXktpL2gK4luDsDyQ8VDJRkd1ewInR/jGSdpPUk/BGtwfh7eQsSbsSCkW/TKQ9FnhKUj/C21SvaKenEtWChcKdft0g20+0O8QqmBOAO6Lzqizt68BPJG1bTt57mVkPQtXS+Vnc1BMIY5r0qiQehKqARQlnU5nWUcCRCvXnjYDDgKeyyCcbTfcRqo7GA7cAbwKrytH0HaHK7AB+LO1X9k1MVc5LRiQ1JFSH3Brfbsrj4qhrEuGh0D5DnNR5XQmMJa3KrxzuBb4lOLjZwK3Ac8DDhKqc0cCDaWlS9ndl7Ws5EjhTUj1gY4ITnQjcTngLSDHKzBab2ffAR4RzuAfwmpnNiw+6oeXorcjuCDNbYGbLgWeAvePyrJktNbMlMbyPmb0LbCZpS0m7EO7ZLwlvVf0Ib14TgO2IVWqFxJ1+3SDTJ9rzEw1qhwGY2az4+xmhjnlXwsBULSQ1SKRNft49hlAv/W9JW6ZnnLA5l/A63guYE0tbxN+5iSSTCSWhoZK6xThHJ7R2T8RNltTLO86k1seAB4AXCVUnE8xsTtxXI01mttLMLop11UcSSslTytNkZvdHHWcR/uH7A3NzdF7WIb4V3Q9MNLNkI+TKlL74INyEUPq9geCYDyaUiPuUc15XR127S7oyYedFoLOkqxPp5gLEqqn7gF5mNp/wltOe8NHU7tH+tBiWst+RtQcQO4/wFnAXcD3wHzPbETgUaJyItyKxXsaPD9dsPkCqyG56eqPiQZCHA0cTSvxPxjABNyTaOLaJ90VBcadfN8j0ifbwxM32vKSWkjYAkNSa8I/+UayH/g/hhgU4lfC6vQYze5rQ0PZSrBog2mkmacPUOqFUM5HwefipFdgbS2gYfUFSBzNLan0v2qsfNT2ZSPcVsEKht4yAkzPYvpnwoLoTGJbYVSNN8VibRm39gSVm9ml5mhQaKiGM+NiWUF3RkFA9UO3zUgE3EJzWb9PCv0sc9x8JTvYworOMb1HnAKdL6impI6E0+r+ErmXAIcCJks6IpeeDCe0l1yXy2jyxfiQwMT7cnifck0cT6vO7AH8jPAgBmkQ9u0pKvVGUxTQ7EwoSM2P4gErOA8DbwL6SWsW3n2PKibdxBXYPVOjx1YRQgHiDcF8dIalpvN+PBP4b4z+ZOMZU+9jLhPPaHEBS28R9UTjMzJc6sBD+CT8llKB+n2H/nsCHhLrpD4EzEvs6Ef7JpxKqQzaI4YOA3ybiDSLc+I0T6d6Py6RUvoR2glGEkvAoYJMYPgC4O2HvNMIreasMeg8AXs8QvkfMaxqxvjyGP0qodwZoSnAiwxL7a6SJUAf9CaEBdATQviJNBGfwDfA5oUFZBMfwdQ3PywxCI+p3cb0rsDWhJPoR8F5c3iKU5FcSesjMITQAz4r7pwKfJ+w+FONOA/onwpck1tvH4xmTsD2DUPVzG6Fx2wi9pz4m9Aa6mVAQmB11fJayT3gLWh21nJKwPx9oEeO0jJrmEBzv9cD0cs7bv4B9E+fwU0KV0p3JeIn4P4lxMtkdBrwQr/k1iTQXx+OZCAxMs/ch4c0hGXZRDP+QUCXYudC+wodhcBzHKSG8esdxHKeEcKfvOI5TQrjTdxzHKSHc6TuO45QQ7vQdx3FKCHf6Tp1EP44oOlHSU6k+9tW0ta+kf8X1w1TOKKZxf4vkOCxVyGPNiKbZhKfFeUjS0RXFSYu/tRIjZDqlhTt9p66SGlF0R0K/8XOTOxWo8v1vZs+b2Y0VRGkBVNnpO05t4U7fKQX+C2wTS7gfS/obYWiE9pL6SXpT0oT4RpD6evIgSZMlvQ4clTKktce+byPpWUnvx2VP4EbC8ATvSbo5xss40qKk3yuM4jiS8JFVhUg6K9p5X9LTaW8vB0j6r8LIkofE+PUl3ZzI+5yankhn/cedvlOnURhTqD/hi0gIzvVhCyMrLgWuAg6wMGjcOOBiSY0JY8ccShiTZvN1DAf+Cow2s10I49hPIoyRPy2+ZVyqckZaVBix8TjC+EdHEcakqYxnzGz3mN/HrD0I2tbAPsDPgXvjMZwBLDaz3aP9s+IwC04JU9nIf46zvtJEUmq8mv8SBiPbEvjCzN6K4b0Jw+6+EYbNoRHhU/ntCEMUTIEwKxdwNuuyH2H4AMysDFisH4cTTpEcaRGgOeEhsCFhxMZlMY/nqZwdJf2RUIXUnDC2S4phFgY6myLps3gM/YCdE/X9G8e8P80i7OgGgwAAAVNJREFUL6eO4k7fqasstzDz1hqiY1+aDCIMoXt8WrzuZDdKYzakRlr8v7Q8BlYjj4cI4wu9L2kAsG9iX3mjQl5gZsmHA5K2rmK+Th3Cq3ecUuYtYC9J2wDE0RO3JQxz3FFhvgEIs3dlYhRhCOBU/flGhIHQNkzEKW+kxTGEsf+bxJFKD81C74bA7Dhy5Ilp+45RmF+3M2EgvE9i3ufF+EjaNo4O6ZQwXtJ3ShYzmxdLzE8oDjsNXGVmnyrMAvaCpPmEiWR2zGDiImBwHA64DDjPzN6U9EbsEvnvWK+/PfBmfNNYApxkZhMkDSWMePkFPw7RWxF/IAwb/AWhjSL5cPmEMKJkG+BcM/te0t8Jdf0TFDKfx7pTNDolho+y6TiOU0J49Y7jOE4J4U7fcRynhHCn7ziOU0K403ccxykh3Ok7juOUEO70HcdxSgh3+o7jOCXE/wc6DXKsD1zGBgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEWCAYAAABv+EDhAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzdd3wU1drA8d+TQockJEASCE2KAtKbqKgUBRvYsPd+r9cO1quoFwt2RV9FxV5AFAUEFVFQQZQqghQB6SEkIQSpIbvP+8dMwmbZJJuQJRvyfPnMJ1POnPNsYc7MOWdnRFUxxhhT+USUdwDGGGPKh1UAxhhTSVkFYIwxlZRVAMYYU0lZBWCMMZWUVQDGGFNJWQVgKhQRGS4iH7jzjUVkp4hElnEZa0WkX1nmGUSZN4tImvt64g8hn50i0rwsYysvIrJURE4u7ziOZFYBmALcg1+aiNT0WXediMwox7ACUtX1qlpLVT3lHcuhEJFo4DngVPf1ZJY2L3f/NWUXXdkTkXdE5H/FpVPVtqo64zCEVGlZBWACiQJuO9RMxGHfseI1AKoBS8s7kHAgIlHlHUNlYf85TSBPA3eLSGygjSLSS0Tmiki2+7eXz7YZIjJCRGYBu4Hm7rr/ichst4likojEi8iHIrLDzaOpTx4visgGd9t8ETmxkDiaioiKSJSIHOfmnTftFZG1broIEblXRFaLSKaIjBORuj75XC4i69xtDxT1xohIdRF51k2fLSI/i0h1d9vZbrPFdvc1H+Oz31oRuVtEFrv7jRWRaiLSCljhJtsuIt/7vi6/9/U6d76FiMx088kQkbE+6VREWrjzMSLynoiku/E+mFchi8hVbuzPiEiWiPwtIgOLeN1rRWSoG/8uEXlLRBqIyFQR+UdEvhOROJ/0n4rIFjfGH0Wkrbv+BuBSYFjed8En/3tEZDGwy/1M85viRGSKiDzrk/9YERlT1GdlgqCqNtmUPwFrgX7A58D/3HXXATPc+bpAFnA5zpXCxe5yvLt9BrAeaOtuj3bXrQKOAmKAP4GVbjlRwHvA2z4xXAbEu9vuArYA1dxtw4EP3PmmgAJRfq8hr8wn3OXbgTlAI6Aq8DrwsbutDbAT6O1uew7IBfoV8v684ubdEIgEern7tQJ2Af3d8oe5r7mKz/v6G5DsvofLgJsCvY5Ar8st8zp3/mPgAZwTuGrACT7pFGjhzr8HfAnUdvNcCVzrbrsK2A9c776Om4HNgBTxvZiDc7XSENgKLAA6ua//e+Bhn/TXuOVWBV4AFvlsewf3u+WX/yIgBaju+1105xPdMvvgVCBrgNrl/f+lok/lHoBN4TVxoAJoB2QD9ShYAVwO/Oa3zy/AVe78DOBRv+0zgAd8lp8Fpvosn+V7gAgQUxbQwZ0fTvEVwP8BXwER7vIyoK/P9iT34BcFPAR84rOtJpBDgArAPeDuyYvFb9t/gXF+aTcBJ/u8r5f5bB8JvBbodQR6XRSsAN4DRgONAsShQAucg/o+oI3Ptht9PsergFU+22q4+yYW8b241Gf5M+D/fJb/A3xRyL6xbt4x7vI7BK4Argn0XfRZPhfYAGTgU+nZVPrJmoBMQKq6BJgM3Ou3KRlY57duHc5ZYZ4NAbJM85nfE2C5Vt6CiNwlIsvc5oPtOFcNCcHELSI3AicDl6iq113dBJjgNs1sx6kQPDhns8m+8arqLqCwTtgEnDPu1QG2FXhf3LI3UPB92eIzvxuf11xCwwABfnObnK4pJNYqFPys/D+n/HhUdbc7W1RMQX2GIhIpIk+6TW47cA7keTEVJdD3xtdknIpthar+XExaEwSrAExRHsZpIvA9aGzGOaD6aoxztpun1LeYddv77wGGAHGqGotzJSJB7vsYMEhVs302bQAGqmqsz1RNVTcBqTjNDnl51MBpfgokA9iL05Tlr8D7IiLi5rspQNri7HL/1vBZl5g3o6pbVPV6VU3GOat/Na/d3y/W/RT8rPw/p1C5BBiEcyUZg3NFAwc+w8K+H8V9b0bgVN5JInLxIcZosArAFEFVVwFjgVt9Vk8BWonIJW5H3YU47eiTy6jY2jht8OlAlIg8BNQpbicRSXFjvUJVV/ptfg0YISJN3LT1RGSQu208cKaInCAiVYBHKeT/hXtWPwZ4TkSS3TPd40SkKjAOOENE+oozrPMunCaY2SV69U456TgH6svcMq7Bp9IRkQtEpJG7mIVz4PT45eFxYxohIrXd134n8EFJ4ymF2jivPROnEnvcb3saUKLfKohIb+Bq4Ap3ellEGha9lymOVQCmOI/itIsDoM4Y9TNxDnCZOM0RZ6pqRhmV9w0wFafDch3OGXdxTQMAfXHOksfLgZFAecMqXwQmAt+KyD84nZk93NezFPg38BHO1UAWsLGIcu4G/gDmAtuAp3D6GlbgdF6/jHP2fRZwlqrmBPm6/V0PDMV5j9tSsCLpBvwqIjvd13Wbqv4dII//4FxNrAF+dl/j4Rg58x7OZ7cJp8N/jt/2t4A2bpPcF8VlJiJ13DxvUdVNbvPPW8Db7pWWKSVxO1eMMcZUMnYFYIwxlZRVAMYYU0lZBWCMMZWUVQDGGFNJ2U2XKiCJqq5SpXZ5h5Hv2NYpxSc6zKIibHCIOTQLFszPUNV6pd0/sk4T1dw9QaXVPenfqOqA0pZVWlYBVEBSpTZVWw8p7zDyTfnhufIO4SDxtaqUdwimgqseLf6/eC8Rzd0T9P/TvYteCeqX7mXNKgBjjAkJgTC/G7pVAMYYEwoCRJTpw+rKnFUAxhgTKmH+Q2WrAIwxJiSsCcgYYyovuwIwxphKSLArAGOMqZzErgCMMabSslFAxhhTGVknsDHGVE6CNQEZY0ylZVcAxhhTGVkTkCknfY87hifuOp/IiAje/3I2L7w7rcD2lMQ4Xn7oMhJia5G1Yzc3PvQum7dup12rhjx7z0XUrlUNr8fLs29/w4RpC0oVw8zflvG/UV/g8XgZckZPbrqkb4Ht+3JyGfrERyxZuYG4OjV58eEraJRYl/25Hu5/eixL/9qIx+Nl8KldufnSfuzL2c/Ft40iJyeXXI+XASd14ParQ3MDxe9m/8l9z47H4/Vy+aBe3HHVqSEppyLHZPEUQ4DI8O4EDu/qKcyIyAARWSEiq0Tk3kLSeERkkTtN9FnfTER+FZG/RGSsiFRx1w8Xkbvd+WoiMk1EHj6UOCMihKeHDeGC216l55D/cd6pXWjdLLFAmkdvO4dPvvqNEy55gpFvTuWhf58NwJ69+7l5+Hv0unAE59/6Ko/feR51alUvcQwej5fhL37OW0/ewNfv3MPk6Qv4a+2WAmk+nfIrMbWr8/2HD3D1BScx8vXJAEydsYic/blMGTOML16/k08m/cLGLduoEh3F+8/9i8lvDWXSm3fz02/LWfjn2tK9ScXEPnTkOD598V/MGfcgn307n+VrUsu8nIock8UTJJHgpnJiFUCQRCQSeAUYCLQBLhaRNgGS7lHVju50ts/6p4DnVbUlkAVc65d/FeAzYL6qPnIosXZp25Q1GzJYtymT/bkePp+2gNNPal8gTevmSfw4dwUAP81bycDexwKwev1W1mxIB2BLRjYZ2/4hIa5WiWP4ffl6miQn0Dg5nirRUZzRpxPfzVpSIM13s5ZwzmndABhwUnt+WfAXqoqIsHtvDrkeD3v37Sc6OopaNaoiItSsXhWA3FwP+z0ehLL/zzN/6VqapyTQtFECVaKjOLd/Z6bMXFzm5VTkmCyeYLhNQMFM5cQqgOB1B1ap6hpVzQE+AQYFs6OICNAHGO+uehcY7JMkys3vL1UNeGVREkn1YtiUlpW/vDkti6R6MQXSLF25ibP6dATgzFM6UKdWdeJiahZI07lNE6Kjo/h7Y0aJY0jLyCapfmz+cmK9WNIysgtNExUZSa1a1cjasYsBJ3WgRrUqHHfecHpf9BjXDTmZ2DpObB6Pl7Oue4Ye5zzECV1a0bFNkxLHVpzU9GwaNojLX05uEEdqenYRe4ReuMVk8QTJrgCOGA2BDT7LG911/qqJyDwRmSMieQf5eGC7quYWsu8wIFdVby+scBG5wc13XnFPGZIAXyjVgsv/fXECx3duwcwP7uH4zi3YlJaFJ9eTv71BfB1ee/QKbnn0A9R/5yAE2sc/roBpEBYvW09kRASzxw9nxkcP8NanM1i/OROAyMgIJr15Nz9/+jC/L1/Pyr/L/jI/cOxlXkyJhFtMFk+QwvwKwDqBgxfo6xToyNhYVTeLSHPgexH5A9hRzL4/A8eJSCtVXRmocFUdDYwGiKhRv8gj8uat2w86G9rid/a9JSObK4a9CUDN6lU465SO7Ni1F4DaNasx9oWbGfF/k5m3ZG1RRRUqsV4sqVu3HygvfTv14+sETJNUL5Zcj4edO/cSW6cGE6cv4MTuRxMdFUl8XG26tG3GHys20Dg5Pn/fOrWq06NjC378bTmtmiWVKsbCJNePPegKKjEhpog9Qi/cYrJ4glDOZ/fBsCuA4G0EfB9+2wjI8OnwPRtAVTe7f9cAM4BOQAYQKyJRPvtu9snrR+B2YKqIJB9qoAv+XMdRjevRODme6KhIzu3fmak/FmwPrRtTM/+M/I6rTuPDSXMAiI6K5P2nr+eTKb/y5fSFpY6h/dEprNuUzobUTHL25/LV9wvp26tdgTR9e7VlwjdzAfh65mJ6dmqBiJDcIJY5C53+gN179rFw2TqOalyfzO072bHTufrZuy+H2fNX0rxx/VLHWJjObZqwen066zZlkLM/l8+nLWBg7/bF7xhC4RaTxROkiMjgpnJiVwDBmwu0FJFmwCbgIuAS3w5bEYkDdqvqPhFJAI4HRqqqisgPwPk4bf1XAl/6Zq6qn4lIPeBrEemtqtspJY/Hy7CR4/jspX8TGSl8OHEOy9ds4b4bz2DRsvVM/fEPTujSkof+fTaqMHvhKoaOHAfAOf0706tTC+rG1OSSM3sC8K9H3mfJyk0liiEqMpKHbz2Xq4eNxuP1csHA7rRqlsgLY6bSrnUK/Y5vx5AzenDX4x/R59IRxNapwQv/vQKAywafwD1PfcLAq0eiwPkDunH0UcksX72ZoU9+jNfrxetVTj+5A32Oa1vat6nw2KMiGTlsCOfd+goej3Lp2T055qiyvcqo6DFZPMEI/98BSGnadysrETkdeAGIBMao6gi/7b2A1wEvztXVC6r6lrutOc7Bvy6wELjMrSiGAztV9Rk33XCcDuNTVXVvoDgiatTXcHoo/Cp7KLw5AlWPlvmq2rW0+0fENNaqJ9wdVNq9U247pLJKy64ASkBVpwBTitg+Gzi2kG1rcEYS+a8fHmB5uH86Y0wFY88DMMaYyir8m4DCOzpjjKnIyrATuLg7EYhIYxH5QUQWishit8m66PBK8ZKMMcYEo4x+CBbknQgeBMapaiecQSqvFpevVQDGGBMKUqa3ggjmTgQK5P3YJoaCQ80Dsj4AY4wJleB/CJYgIvN8lke7P/7ME+hOBD388hgOfCsi/wFqAv2KK9QqAGOMCZFAt2UpREYxw0CDuRPBxcA7qvqsiBwHvC8i7VTVW1imVgEYY0wIOE+ELLNbQQS6E4F/E8+1wAAAVf1FRKoBCcDWwjK1PgBjjAkFESQiuCkI+XcicG8dfxEw0S/NeqCvU7QcA1QD0ovK1K4AjDEmRMrqCkBVc0XkFuAbDtyJYKmIPArMU9WJwF3AGyJyB07z0FVazK0erAIwxpgQKcMmoIB3IlDVh3zm/8S5/1jQrAIwxpgQKcsKIBSsAjDGmFAQAo/dCSNWAVRA7Y9OYdrM58s7jHwtbx5b3iEcZMHz55V3CAXUrhZe/9VqV48u7xCOeILYFYAxxlRWERHhPdDSKgBjjAkRuwIwxpjKyPoAjDGm8rIrAGOMqYSsE9gYYyqxIG/zUG6sAjDGmFAQawIyxphKyyoAY4yppKwCMMaYSsg6gY0xpjIL7+O/VQDGGBMSYreCMMaYSsuagIwxprIK7+O/VQBHkhm/LmP4SxPweJWLzujBvy/rV2D7vpxc7hjxIX+s3EhcnRq8MvxKUpLqsiF1G30uf5KjGtcDoFObJjxx9xAAJk5fyKj3p+HxeulzXBseuPnsUsV2yrFJPHZJNyIjhA9/XMWor5YW2P7IxV04/pgGAFSvEkVCnWq0/tc4ABrWrcGz1/QkuW5NUOXS539gQ8auUsWR56e5y3ny/ybi8Xo5b0B3rr+oT4Ht8xav4cnXJrJyTSpP338pp/VuD8Cy1Zt47KXP2bl7H5ERwg0X92XgyR1LFUM4f17B+G72n9z37Hg8Xi+XD+rFHVedGrKyKmI8YFcAYUtE1gL/AB4gV1W7ikhdYCzQFFgLDFHVLBG5CuiqqreISATwtrvftb7P3BSRK4A7fYrpAByrqktEpJu7XzVgkqre4e7zATBeVb8QkQTge+AZVX2vJK/H4/Hy4POf8eFzN5FUL5azbnie/ie0o1XTxPw0Y7+aQ0zt6vz08QNMnL6AJ16bxKuPXAlAk4bxfD1maIE8s7J38fj/TeSrN+8iPrYWd4z4kJ/nr+SELq1KEhoRIjxxeXeGPD2d1G27+frhgXy7cCMrN2fnp3n44/n589f2a027xnH5yy/fcDwvTPqDH5duoUbVKIp5zGmxPB4vI0ZN4I0nb6BBQgwX/uclTjmuLS2aNMhPk1Q/lhF3D+Gd8TML7Fu9ahWeGHYRTRrWY2tmNhf8+0WO79qaOrWqlziGcP28go1/6MhxTBh1C8kNYulz5dMM7H0sRzdPKvOyKmI84Bz8w70CCO8eitA7RVU7qmpXd/leYLqqtgSmu8v5xPk0XwOigev8H7isqu+5+XUErgb+UtUl7ubX3HUtgbYi0t8v71icBz6/XNKDP8CiZetp2jCBJskJVImO4qy+nfj25yUF0nz78xLOH9AdgNNP6sCsBX8VeTBdvzmTZin1iI+tBcAJXVsxdebikoZGp+bx/J32D+vTd7Lf4+WLX9dyWqdGhaYf3KMpE35dC0Cr5BgiI4Qfl24BYPe+XPbkeEocg68/VqwnJTmBlKR4qkRHcfpJHflhdsErkoaJdWndPPmg/8BNG9WjSUPnzLt+fAx1Y2uRlb2zxDGE8+cVjPlL19I8JYGmjZz4z+3fmSkhKqsixpMnrxIobiovlb0C8DcIeNedfxcY7Lf9RSAeuEJVvcXkdTHwMYCIpADVVHWuW2m875d3beBr4F1VfaM0gW/J2E5y/dj85aR6MaSlZ/ulyc5PExUVSe2a1cjKdppSNqRuY+C1z3DBf0bx6++rAWjSKIHV67eyIXUbubkevv1pCZu3ZpU4tqS4Gmzetjt/OTVrN0lxNQKmbRRfk8b1avHzn2kANE+szY7dObx1S2+mPXI6D13YmYhD/A+TlrGDpHoH3qsG9WJIy8wuYo/AFi9fT+5+DylJ8SXeN5w/r2CkpmfTsMGBq7TkBnGkppf8PTxS48kjERLUVF4qbRMQoMC3IqLA66o6GmigqqkAqpoqIvV90l8CLANOVtXcojJ2rxSGAAPcVQ2BDT5JNrrr8rwIvKaqLxWR5w3ADQCNUhof/GICnBj6HycDnT2KCPXj6zDn04eIi6nJ4hUbuP7+MXz33j3E1q7BiDvP59/D3yUiQujSthnrUzMLC7FQgY7XSuAz2cE9mjB53jq8bqxRERH0aFWffg9PYVPmLl7/14lceGJzPv5xdYnj8C394BhL9p8wPXMH9438hMeHXliqoX7h/HkFF3+g2EJSVFDCLZ4DMYRBEEWozBXA8aq62T3ITxOR5cWkXwAcDXQHZhWTtheQpap5eQb6Fvh+Y6cD54jIc6qaEShDt4IaDdCxc5eDvu1J9WLZvHV7/nJqejb1E2ICpkmqH0turod/du0ltk4NRISqVZyvQvvWKTRpGM+aDVvpcHRj+h/fjv7HtwPgw4mziYgs+Rd687bdJNc9cMafFFeDLVl7AqYd1KMp973/24F9s3azZH0W69OdZpavF2ygy1EJfEzpK4AGCTGkph94r9LSs6lft07Q++/ctZeb/zuGW686jQ7HNClVDOH8eQUjuX4sm9IOXF1sTssi0S/+wync4gEqxM3gKm0TkKpudv9uBSbgHNjTRCQJwP271WeX5Thn9WNFpK2b5nwRWeROvkNBLsJt/nFtBFJ8lhsBm32WPwTGAFNEpGZpXk+Ho1P4e2M66zdnkrM/l0nTF9L/+LYF0vQ/vh3jv3YOrlNm/k6vzi0QETK378TjcVq01m3O4O+NGTRJdpo1MrL+AWD7P7t5/4tZXHxmzxLHtujvTJo3qE3jhJpER0YwuEdTvl248aB0RyXWIbZmFeatOlAHLlqTSUyNKsTXrgrACcckFug8Lo12rVNYvymDjanbyNmfy5SZizjluDZB7ZuzP5dbH3mXs/t14bTeHUodQzh/XsHo3KYJq9ens25TBjn7c/l82gIGuiOlykO4xQPuA8EkuKm8VMorAPcgG6Gq/7jzpwKPAhOBK4En3b9f+u6nqrNF5CbgKxHprarjgfF+eUcC5wM9fPbbICL73JFA84DLgWf88n5aRBoAn4nIWaq6vySvKSoqksduP4/L734dj9fLhaf3oHWzJJ59ayrHtk7h1BPaceEZPbh9xIecePEIYmvXYNTwywH4ddFqnh0zlajISCIjhMfvOp/YOk49NPylCfy5yqmrbr/qVJqn1C80hsJ4vMr9H8zl47v7EhkhfPzTalZszmbYOe1Z9Pc2vl3kVAbn9GzKF27nbx6vKo+Mnc+nw/ohwOJ12/hgxqoSx+ArKjKSB24ZzA33v4HX6+Wc07rTomkiL7/7DW1bNaLPcW35Y8UGbnvkXXb8s5sZc5bxyvvfMvGNu/lm5u/M/2MN23fs4otv5wIwYuiFHHNUw2JK9YshjD+vYOMfOWwI5936Ch6PcunZPTnmqPIbcRNu8TjCfxSQHOqQuopIRJrjnPWDUwl+pKojRCQeGAc0BtYDF6jqNt9hoO7+VwNDgRNVNdMv737AcFU9wW99D5yz/GrAZOB2VVW/YaCC0/kcCVzmP8ooT8fOXXTazDmH/kaUkZY3jy3vEA6y4PnzyjuEAmpXC69zrdrVo8s7hLBXPVrm+4wQLLFqia20yZUvB5V25cgBh1RWaYXXt/IwUdU1OGP0/ddnAn0DrH8HeMdn+W2cMf2B8v4O+C7A+l+BtgHWX+Yzr8AVQbwEY0y4K+fmnWBUygrAGGNCTYAIeySkMcZUTnYFYIwxlVS4dwJbBWCMMaFgfQDGGFM5CWIPhDHGmMrKrgCMMaaSsj4AY4ypjKwPwBhjKifnXkDhXQNYBWCMMSES5sd/qwCMMSZUwv2XwOE9RskYYyoqKdtHQorIABFZISKrROTeQtIMEZE/RWSpiHxUXJ52BVABqUKuN3zu4vrm3X3KO4SD3PHFkuITHUavDSn9swNMxZT3PIAyycu5zfwrQH+c54vMFZGJqvqnT5qWwH04D7vK8nuiYUB2BWCMMSER3Nl/kFcA3YFVqrpGVXOAT3CeYe7reuAVVc2C/IddFckqAGOMCZESPBEsQUTm+Uw3+GVV3HPFAVoBrURklojMEZEBFMOagIwxJhSkRJ3AGcU8EKa454qDczxvCZyM89jZn0Sknapu998xj10BGGNMCOT9DqCMmoCKe654XpovVXW/qv4NrMCpEAplFYAxxoRIGVYAc4GWItJMRKoAF+E8w9zXF8ApbrkJOE1Ca4rK1CoAY4wJkRL0ARRJVXOBW4BvgGXAOFVdKiKPisjZbrJvgEwR+RP4ARjq/8xyf9YHYIwxIVKWt4JQ1SnAFL91D/nMK3CnOwXFKgBjjAkFuxmcMcZUTs4DYcK7BrAKwBhjQiQizC8BrAIwxpgQCfPjv1UAxhgTCiIV+HkAIlKnqB1VdUfZh2OMMUeOMO8CKPIKYCnOT419X0LesgKNQxiXKYWZvy7j0VFf4PV4GXJGT26+tG+B7ftycrn7iY9YsmIDsTE1efmhK2iUVJcvps3njU9+yE+3fE0qk0bfSeOG8Vz4n1H567ekZzOof2ce+s85JY5t8R+ref+jaXi9ysm9O3DWGb0KbJ/+wwK+mz6fiAihWrUqXHPlQBo2rMc/O3fz8iufs+bvVE48vj1XXn5aicsOpEPDOlzVozERIny/Mp0v/9hSYPtJLeK5rFsK23btB+CbZWl8/1dG/vbq0RE8d86x/LY+i7fnrC9VDOH8eQXju9l/ct+z4/F4vVw+qBd3XHVqSMqpqPFA+D8PoNAKQFVTCttW0YlIa2Csz6rmwEPAe+76psBaYIh7W9WrgK6qeouIRABvAx7gWnfsbV6+VYHRQGd3+62q+qO7rZu7XzVgkqre4a7/ABivql+4v977HnhGVd8ryWvyeLw8/OLnvPfMTSTWi2HwTc/T7/i2tGyamJ9m3JRfqVOrOj989ACTpi/kqdGTefnhKxjcvwuD+3cBYPmazdz4wBjatHTuM/XVW3fn73/2Dc8xoHf7koQFgNfr5d33v+Geuy+mbt06PPTo23Tu2JKGDevlp+nVsy19T+kMwIKFK/nwk+kMu+sioqOjOO+ck9i4KZ2NG9NLXHYgInBNzyaM+GYlmbtzeOKsNsxbv51N2XsLpJv997ZCD+5DOjfizy3/lDqGcP68go1/6MhxTBh1C8kNYulz5dMM7H0sRzdPCkl5FS0ecG8FEfAWPuEjqF8Ci8hFInK/O99IRLqENqzQUtUVqtpRVTsCXYDdwATgXmC6qrYEprvL+cRp0HsNiAau8z34u24CclT1WGAA8JwcaAR8Dbga594cbUWkv1/esTi/5Hu5pAd/gN+Xr6dJwwQaJ8dTJTqKM/t0YtqsgvfE/27WEs4b0A2AgSe1Z/b8v/B/CZOmL+Ssvp0Pyv/vjelkZu2kW/vmJQ2N1Ws206B+HPXrxxEVFUnP7m2Yv/CvAmmqV6+aP79v3/78zrNqVavQulUK0dFl113VIqEmaf/sY+vOfXi8yuw12+jWOC7o/ZvF1yC2WhSLN2eXOoZw/ryCMX/pWpqnJNC0UQJVoqM4t39npsxcHJKyKmI8eSIkuKnc4isugYiMwrm/xOXuqt04B7MjRV9gtaquw7m/9rvu+neBwX5pXwTigStU1Rsgr1Pf3iIAACAASURBVDY4FQequgXYBXQSkRSgmqrOdSuN9/3yrg18Dbyrqm+U5kVsSc8mqV5s/nJSvVjS0gseoNJ80kRFRVK7VjWysncVSPPVD4s4q0+ng/KfNH0BZ5zSsVSdWllZ/1C37oEupbp1a5OVdfDZ87Tp87hr2Kt8Mu57Lr8kdJfvdWtUIXNXTv5y5u4c4mpGH5SuR5M4Rg5qyx2nHEV8zSqAc1Z3ebcUPpi38ZBiCOfPKxip6dk0bHCg0kxuEEdqeukrxCMtHgCCvA9QeXYUB3MF0EtVbwT2AqjqNqBKSKM6vC4CPnbnG6hqKoD71/eJOpfgXC1c5N6XI5DfgcEiEikiRwGdcO7gV9y9vF8EvlfVlwoLUkRuyLtXeGZmRoAUBz8hzP+LpcWkWfTnOqpVjaZ1gMvmyd8v4qy+Bx9oghHo2WWBvvP9+3bl2ZH/4sIL+vDlpFmlKisYAf+/+QU5f8N2bvl0McO+XMofm3fwrxObAXDqMfVZtDG7QAVSOuH7eQXj4Ivf8h3yGG7x+MZQFvcCCpVgKoD9bru3AohIPBDo7LfCce+qdzbwaRDJFwBNcJ7MU5g3gDRgPvAM8AuQS/H38p4OnOP2AQSkqqNVtauqdo2PPzhZYr1YUtMP3PY7NX079RPqFJomN9fDPzv3ElunRv72Sd8Hbk5YtmoTuR4vx7YuXbdQ3bjabNt2YNDYtm3/EBtbu9D0PXu0Yf7ClaUqKxiZu3Lyz+gB4mtUIWv3/gJpdu7z5D92c/rKdJrHO+9Tq3q1OO2Y+rx8fnsu65ZC76MSuLhLoxLHEM6fVzCS68eyKS0rf3lzWhaJCTEhK6+ixQPOf/oIkaCm8hJMBfAK8BlQT0QeAX4GngppVIfPQGCBqqa5y2kikgTg/vV9pNpyYAgwVkTaumnOF5FF7tTRvQ/3bW7/wjlAHPAXxd/L+0NgDDBFRGqW5oW0b53C2o3pbEjNJGd/LpO/X0i/Xu0KpOnbqy2ffT0XgKkzF3Nc5xb5Z5Rer5epM34P2JwwcfrCQzqbbN4smS1bs9iavp3cXA9zfvuTzp0K3qZ8y5Zt+fOLFq8isUHwbfIltTpjF4l1qlKvVhUiI4Rezesyb0NWgTSx1Q80CXVNiWXTdqeD+OUf1/DvTxfzn/GL+WDuBn5cncHH80veHBTOn1cwOrdpwur16azblEHO/lw+n7aAgSHqcK6I8eSJiJCgpvJSbM+aqr4nIvOBfu6qC1Q1vJ64XXoXc6D5B5z7a18JPOn+/dI3sarOFpGbgK9EpLeqjgfG5213D96qqrtFZCCwU1VXutv2uSOB5uH0pzzjl/fTItIA+ExEzlLVgqekxYiKimT4bedy5dDReL1eLhjYnVbNEnl+zFSObZ1Cv+PbceHpPbjz8Y845ZIRxNSpwUsPXZG//2+/ryGxXgyNk+MPynvKjEWMefL6koRTQGRkBFdceipPP/sJXq+X3id2oFHDenw2YSbNmibRuVMrpk2fx9I/1xIZGUHNmtW44bqz8ve/4+5X2LN3H7m5HuYvXMk9d11UYARRSXkVxsxZz/2ntiZCYMZfGWzcvpcLOiWzJmM38zdsZ2CbBnRJicWrys59ubz689+lLi+QcP68go1/5LAhnHfrK3g8yqVn9+SYo8pvxE24xQPl37wTDAnUdnZQIpH2wAk4zRazVLX8u9cPkYjUwGmXb66q2e66eGAczm8c1uNUdtt8h4G66a4GhgIn+t5v2233n4LTRLYRuEZVN7jbeuCc5VcDJgO3q6r6DQMVnM7nSOCyAKOMAOjQqYt+PeOXsn1DDsGsvwP1SZSv9+duKu8QCnhtSIfyDqGAuJpHUjdeaFSPlvnFPKaxSHWbtdH+wz8MKu24qzofUlmlVewVgIg8gNMBOgGnWesjEflQVZ8IdXChpKq7cUb0+K7LxBkV5J/2HeAdn+W3ccb0+6dbDbQupLxfgbYB1l/mM6/AFf5pjDEVU5hfAAR1L6DLgC7uARMRGYHTyVmhKwBjjAm1CnsvIB/r/NJFUcxzJo0xprJzRgGVdxRFK+pmcM/jtPnvBpaKyDfu8qk4I4GMMcYURir2A2HyRvosBb7yWT8ndOEYY8yRo8I2AanqW4czEGOMOZJU6CagPO7QxhE497mplrdeVVuFMC5jjKnwwv0KIJhfAr+DM+RRcH45Ow74JIQxGWPMEUGCnMpLMBVADVX9Bpxx7qr6IM7dQY0xxhRCBCIjJKipvAQzDHSf+wvV1e5tEDZR8C6ZxhhjAgj3JqBgKoA7gFrArTh9ATHANaEMyhhjjgRhfvwP6mZwv7qz/3DgoTDGGGOKIJTvrZ6DUdQPwSYQ+FkeAKjquSGJyBhjjgQV4G6gRV0BjDpsUZgSiYqQsLqbY88mB9+SuLy1T4wtPtFhdPTNY8s7hALS3rOL+cOhwvYBqOr0wxmIMcYcSQSIrKgVgDHGmENT4X8JbIwxpnSOmApARKqq6r5QBmOMMUcK55GQ4V0DFPtLYBHpLiJ/4DzcHBHpICIvhzwyY4yp4CIkuKnc4gsizUvAmUAmgKr+jt0KwhhjipX3YPjipvISTBNQhKqu87uU8YQoHmOMOSIIEBXmTUDBVAAbRKQ7oCISCfwHWBnasIwxpuIL8+N/UBXAzTjNQI2BNOA7d50xxphCiFTgW0HkUdWtwEWHIRZjjDmihPnxP6gngr1BgHsCqeoNIYnIGGOOEGU5wkdEBgAvApHAm6r6ZCHpzgc+Bbqp6ryi8gymCeg7n/lqwDnAhqAiNsaYSkqgzB724va/vgL0BzYCc0Vkoqr+6ZeuNs6t+389OJeDBdMEVOAuViLyPjAtyLiNMaZyKtsx/t2BVaq6BkBEPgEGAX/6pXsMGAncHUympbkVRDOgSSn2M2Hku9l/ct+z4/F4vVw+qBd3XHVqmZfx42/L+d+oL/B4vQw5vQc3XtK3wPZ9ObkMe/IjlqzcSGydmrz40OU0SqxLzv5c/vvceJas3ECECA/eMpgeHVsA8NxbU5jw7Tx2/LOH36c8UerYZs1bwVOvfYnXq5wzoDvXDin405b5f6xh5OsT+evvLTx17yX0P7F9ge07d+1l8I3P0KdXO+7/1+BSx+HrlGOT+d9lXYmMED6cuYqXJy8tsP3RS7py/DENAKheNYqE2tVodfNYjj+mAY9e0jU/XYukGG569SemLgjthfrh+A5V5HjAeSZAkBJExLe5ZrSqjvZZbkjBlpeNQI8CZYl0AlJUdbKIBFUBBPNL4CwR2eZO23HO/u8PJvNwICK3icgSEVkqIre76+qKyDQR+cv9G+euv0pERrnzESLyroiMEb8fQYhIfRGZISK7ROQFn/W1RWSKiKxwyxvhs62aiIwXkVUi8ouINHbX9xORL3zSPSEiX4lIyO737PF4GTpyHJ+++C/mjHuQz76dz/I1qWVexvAXP+fNJ69n6tvDmPz9Qv5au6VAmvFTf6VO7RpM/+B+rj6/N0+PngzAuK/mAPDVW0N55+kbeeL/JuH1egE45bi2fPbq7Ycc2+OvTODVx65lwut38fWMRaxel1YgTWL9WB6760IGntIxYB6vvP8NXY9tfkhx+IoQ4ckrunPJM99z4r2TOKdnU1olxxRI89BH8+j736/o+9+veGvacqbMXw/ArGVp+evPe3Iae3JymbFkc5nFFsjh+A5V5HjAaQIqwS+BM1S1q880OkB2/vL7ZkUkAngeuKskMRZZAbgHvg5APXeKU9XmqjquJIWUFxFpB1yPc/nUAThTRFoC9wLTVbUlMN1d9t1PgNeAaOA6VfXvBN8NPADc47degadUtTXQGThFRPq7224AtqhqC5y2vINOX0VkONANOE9Vc0r1ooMwf+lamqck0LRRAlWiozi3f2emzFxcpmUsXr6eJg3jaZwcT5XoKM7o04npswue0X43awnnnuqcuQ44qT2/LPgLVWXVujR6dW4JQHxcberUqsYfKzYC0KlNE+rH1zmk2Jas3EBKcgKNkuKJjo5iwEkdmDGnYGwNG9SlVbOkgMP4/vxrI5lZOzmuc6tDisNX56Pi+XvrP6xL38l+j5cv5qxjQOeUQtOf07Mpn/+y9qD1Z3VrwveLN7MnJ7S/1Twc36GKHE+eMrwVxEbA9wvRCPCt5WsD7YAZIrIW6AlMFJGuFKHICsA98E1QVY87FfqEsDB1DDBHVXerai4wE6cTexDwrpvmXcD/Gv5FIB64QlW9/pmq6k5VnQXsDbB+pju/D1iI80HhV+Y44DTffUXkHqAPcLaqFsi3rKWmZ9OwQVz+cnKDOFLTs8u0jC0Z2STVP/BQlsSEGNL8ykjL2EGimyYqMpJaNauTtWMXRx+VzHezlpDr8bAhNZMlKzeSmr69zGLbmpFNYr0DZ9f1E2JIy9wR1L5er5dn35jMndedUWbxACTG1WBz5q785c3bdpEYVz1g2kbxNWlcrxY//7nloG2DezZlwpy/yzS2QA7Hd6gix5NHRIKagjAXaCkizdzWgYuAiXkbVTVbVRNUtamqNgXm4BxLDnkU0G8i0llVFwQTZZhZAowQkXhgD3A6MA9ooKqpAKqaKiL1ffa5BFgGnOxWGqXiNiudjtMhAz5teKqa4zYf5R0he+NUVl1VdXch+d2AcxVBSuPGpQ0Lt/wA+R9SlgEKOXiVfxkB40A4f2B3Vq/byjk3vUDDBnF0btuUqMhgbltV6tCCbqkdO/kXTuh2NIn1yvaJYyV5+wf3bMrkuevx+r1/9WOqc3SjWH74I7TNP3CYvkMlEG7x5JVfVl9bVc0VkVuAb3CGgY5R1aUi8igwT1UnFp1DYEU9EzjKPQCeAFwvIquBXTjfVVXVzqUp8HBS1WUi8hROv8VO4HeguIP6AuBonGajWaUpV0SigbHAs6q6Lm91oBDdv3/hNLH1Bb4IkA63TXA0QJcuXQ/pSiy5fiyb0rLylzenZZGYEFPEHiWXWC+G1K0Hztq3ZGRT36+MxHoxbNm6naR6seR6POzctYfYOjUQER7496D8dENueYkmDRPKLLYGCTFs8Tk73JqRHXSz0uJl61iwdC3jJv/C7r372L/fQ41qVbj9mtMPKabUrN0kx9fMX06uW5MtWXsCph3csyn3vvvbQesH9WjC1PkbyPWE/kL9cHyHKnI8ecryl8CqOgWY4rfuoULSnhxMnkXVT3nfsMFAa5yz2QuA892/FYKqvqWqnVW1N7AN52CbJiJJAO7frT67LAeGAGNFpK2b5nwRWeROgXsFXW7/wVvAElX1fa5yfhueewlXU1XzjkKpOO/vKBHpfYgvuVid2zRh9fp01m3KIGd/Lp9PW8DA3u2L37EEjj06hbWbMtiQmknO/ly++n4hfY9rWyBN315t+fxb5wr165mL6dmpJSLCnr057N7jPHri53kriIyMpGXTxDKLrW2rRqzfnMHGLdvYvz+Xr2f+zkk92wS17xP3XMI3793P1Hfv487rzuTMfl0O+eAPsHBNJs0b1KZxQi2iIyMY3LMJ3yw8eBTPUYl1iKlRhXmr0g/ads5hav6Bw/MdqsjxQIk7gctFUU1AAqCqqw9TLCEhIvVVdas76uZc4DicoaxXAk+6f7/03UdVZ4vITcBXItJbVccD44Ms8gmcH8z5D8Oa6JY1F6eC+davzOUich7wuYgMVNWQ9WBFRUUyctgQzrv1FTwe5dKze3LMUUllW0ZkJA//51yuuWc0Ho9y/sDutGyWyAtvf82xrRrR9/h2XHB6D+5+/CP6XvY4sbVr8Px/nQeVZ27fyTXDRiMRQmJCDM/cd3F+vk+9PolJ0xeyZ99+ThjyKENO78GtV51WWBiFxnbfzYO4+cE38Xq8DD61Gy2aJPLKe9/QtlUjTu7ZliUrNnDHY++xY+duZv66jFc/mMaE10s0wKJEPF7lvvd+45NhfYkU4eMfV7FiUzbDzu3A739n8s1CpxP8nOOa8uWvaw/aPyWhJsl1azJ7edpB20LhcHyHKnI8ecq7Gao4Uli/rohsBJ4rbEdVLXRbOBGRn3A6dPcDd6rqdLdPYBzODe7WAxeo6jYRuQqnHf4Wd9+rgaHAiaqa6ZfvRqAGzkihbJzmm33A3zh9CHmjeF5U1bdFpDrwAc5opAzgIlVdKyL9gFtUdbCb70CcEUgnqeraQK+pS5euOuvXIvt2DquMf8LvQXG794XXHcu73flZeYdQQNp7l5d3CGGverTMV9UiR9EUpfHR7fWet4Jrmr/lhGaHVFZpFXUFEAnUomT9U2FHVU8MsC4T54Dtv/4d4B2f5beBtwvJt1Gg9RTyfqnqHuC8AOu/w+d2G6o6FfuhnTEVnhD+VwBFVQCpqvroYYvEGGOOJAJRYf5U+GL7AIwxxpRcRb8COKiJxBhjTPAq7ANhVHXb4QzEGGOONGF+/C/V3UCNMcYUQwjibpvlzCoAY4wJBanATUDGGGNKz/klsFUAxhhTKYX34d8qAGOMCZkwvwCwCsAYY0Ij6Hv9lxurAIwxJgRsFJAxxlRi1glsjngJtauWdwgHq13eARQ0a+Q55R1CAY2u+6S8QzjI7y+cW94hlC3BmoCMMaYysiYgY4ypxOwKwBhjKqnwPvxbBWCMMSEhQKRdARhjTOUU5sd/qwCMMSY0BAnzRiCrAIwxJkTsCsAYYyohZxhoeNcAVgEYY0woiF0BGGNMpWW3gjDGmErIeSBMeUdRNKsAjDEmRGwUkDHGVFJh3gJkFUBl9d3sP7nv2fF4vF4uH9SLO6461eIp53hmz1/Bs29MwutVBvXvxlUXnFxg+4Ila3jujcmsWruFEcMupu/xxwIwb/Fqnntzcn66dRvTGTH0Yk4+ru0hxXNy20QeubgzkRHCxz+t4ZWpywpsf/jCTvRqXR+A6lUiia9Tjba3fu7EMHoIyzdmA7Bp226uGfVTqWKY+dsy/jfqCzweL0PO6MlNl/QtsH1fTi5Dn/iIJSs3EFenJi8+fAWNEuuyP9fD/U+PZelfG/F4vAw+tSs3X9qPfTn7ufi2UeTk5JLr8TLgpA7cfvWAUsUWDLsCOAxEZAxwJrBVVdu5654GzgJygNXA1aq63d12H3At4AFuVdVv3PU7VbWWO3868CLQV1XX+5X3JHAZUEtVY33WXwc8AWxyV72oqm+7264F7nXXP6KqH7jrNwLtVHW7iHQHxgKDVHVxmb1BfjweL0NHjmPCqFtIbhBLnyufZmDvYzm6eVKoirR4gihz5GtfMuqxa2kQH8OVd46id49jaN64QX6axHqxPHz7BXww4ccC+3ZtfxQfvXQbANn/7ObcG56mZ6eWhxRPhAj/u7Qrlzz3A6lZe/jqwf58u2gTf6XuyE/zyNiF+fNX92lJ28Zx+ct7czyc9ug3hxSDx+Nl+Iuf8+7TN5FYL4Zzb3qevr3a0rJpYn6aT6f8Skzt6nz/4QNM/n4hI1+fzEsPX8HUGYvI2Z/LlDHD2LM3hwFXPcVZfTvTsEEc7z/3L2pWr8r+XA8X/edlTupxNJ3aND2kWAOpCH0A4X630mC9A/hX49NwDqztgZXAfQAi0ga4CGjr7vOqiET67igifYGXgQH+B3/Xl0DPQmL5UFU7ulPewT8BuB/o5u73mIjE+JXZEefgf0EoD/4A85eupXlKAk0bJVAlOopz+3dmysyQFmnxFGPpXxtISYqnUWI80dFR9O/dgZm//lkgTXKDurRsllTkHSanz/qD47q0plq1KocUT8dmdVm79R/WZ+xiv8fLl7+t59SODQtNP6h7E778bd0hlenv9+XraZKcQOPkeKpER3FGn058N2tJgTTfzVrCOad1A2DASe35ZcFfqCoiwu69OeR6POzdt5/o6Chq1aiKiFCzuvP8itxcD/s9ntCdpYsQEeRUXo6ICkBVfwS2+a37VlVz3cU5QCN3fhDwiaruU9W/gVVA97z9RORE4A3gDFVdXUh5vwBbShDiQOBrVd2uqpnA94Bvm0JbYAJwiarOK0G+pZKank3DBgfO1pIbxJGanh3qYi2eIqRn7qBBwoFzggbxMaRn7ihij8Cm/fQ7p/XucMjxJMVVJzVrd/7ylqw9JMVVD5i2Yd0apCTUZNayrfnrqkZH8tWDpzLxvn6cVkTFUZS0jGyS6udfYJNYL5a0jOxC00RFRlKrVjWyduxiwEkdqFGtCsedN5zeFz3GdUNOJrZOTcC5sjjrumfocc5DnNClFR3bNClVfMGQIKfyckQ0AQXhGpyza4CGOBVCno3uOoCqOGf3J6vq8lKWNURE+gDLgTtUdZOb/4ZCygSYBFzkViwBicgNwA0AKY0blzI0h6oGyP+QsjwkFk/ZlJmxbQer1qZxXOdWZRDRwYUHCBGAQd0bM2X+Brw+CXoMm0ha9l4aJ9Rk7N19WL4pm3XpO0sUQeD3RIpPg7B42XoiIyKYPX44O/7ZzUW3jaJXl1Y0To4nMjKCSW/ezY6de7j5v2NY+XcqrZqVffOe0wQU3m1AR8QVQFFE5AEgF/gwb1WAZHnfov3AbJz+gdL4AmjmNjvNBN4OokyA74DrRKTQz0NVR6tqV1XtWi+hXinDcyTXj2VTWlb+8ua0LBITYorYI7QsHqifEFPg7DYtM5uEunVKlMe0nxdz8nFtiYqKLD5xMVKzdpMUVyN/OTGuOlu27wmY9uzuTfjCr/knLXsvAOszdvHLiq20axwbaNciJdaLJXXr9vzlLenbqR9fp9A0uR4PO3fuJbZODSZOX8CJ3Y8mOiqS+LjadGnbjD9WbCiwb51a1enRsQU//lbac73ihfsVwBFdAYjIlTidw5fqgVOFjUCKT7JGwGZ33gsMAbqJyP1uHlVEZJE7PVRUeaqaoar73MXROG3+xZUJcDPO1cfLJXl9pdW5TRNWr09n3aYMcvbn8vm0BQzs3f5wFG3xFKJNy0as35zJpi3b2L8/l2k//k7v7m1KlMe3P5ZN8w/A72u30axBbVISahIdGcGg7o2Z9vumg9I1b1CbmBpVmL86M39dTI1oqkQ5h5a4WlXo1iKBlZtL3pzV/ugU1m1KZ0NqJjn7c/nq+4X07dWuQJq+vdoy4Zu5AHw9czE9O7VAREhuEMuchU5/wO49+1i4bB1HNa5P5vad7NjpVGR79+Uwe/5KmjeuX+LYghbmNcAR2wQkIgOAe4CTVHW3z6aJwEci8hyQDLQEfsvbqKq7ReRM4CcRSVPVt4COQZaZpKqp7uJgYKk7/zXwiIjEApFAX+BOn109OB3T34rIQ6r6aAlfbolERUUyctgQzrv1FTwe5dKze3LMUeUz4sbiccuMjGTYTWdz68Nj8Hi9nN2vK0c1acBrH3zLMS0bcVKPNixduYFhj7/Pjp17+Hnucl7/cBrjXnW+RpvTtpGWnk3nds3KJB6PV/nvR/P58PaTiIiIYOysNazcvIO7B7Xj97XbmPa7c/4yuEcTJs4tePbfIqkOT13eDa8qESK8MnVZgdFDwYqKjOThW8/l6mGj8Xi9XDCwO62aJfLCmKm0a51Cv+PbMeSMHtz1+Ef0uXQEsXVq8MJ/rwDgssEncM9TnzDw6pEocP6Abhx9VDLLV29m6JMf4/V68XqV00/uQJ9DHC5blHBvApJAbWgVjYh8DJwMJABpwMM4o36qAnmnJnNU9SY3/QM4/QK5wO2qOtVd7zsMNAX40d3+pV95z+FcKSTjnMm/pqr/c4eeDsQ5oGcCN6vqCnef63EqJIBHVfU9d73vMNA4t8xRqvp6Ya+3S5euOuvXkPcVmzK0akvJ2r9D7eQHJxef6DD7/YVzyzuEAhrFVZ2vql1Lu/8xx3bS976cEVTa7kfFFluWe1L7Is5J5Juq+qTf9juB63COa+nANapa5NCsI+IKQFUvDrD6rSLSjwBGBFhfy2d+AxDwdEpV76TgGXze+qHA0EL2eQNndJH/+kY+81nAsYXFbYypYMroAsAdqv4K0B+nSXmuiExUVd+xwguBrm4rxs3ASODCovI9ovsAjDGmvDjN+8H9C0J3YJWqrlHVHOATnCHt+VT1B5/mbt+h74WyCsAYY0LBfR5AMBOQICLzfKYb/HIrbii5v2uBqcWFeEQ0ARljTDgqQQtQRjF9AMUNJT+QUOQyoCtwUnGFWgVgjDEhIUXetqOEihtK7pQo0g94AGf04z7/7f6sCcgYY0KkBE1AxZkLtBSRZiJSBWfY+MSCZUkn4HXgbFXdGiCPg1gFYIwxIRDsb8CCOf679zW7BfgGWAaMU9WlIvKoiJztJnsaqAV86v5wdWIh2eWzJiBjjAmVMvwdmKpOAab4rXvIZ75fSfO0CsAYY0LEHghjjDGVVJjfCcIqAGOMCYngO3jLjVUAxhgTItYEZIwxlZBgVwDGGFNphfnx3yqAisijyq59ucUnPEx27/OUdwhhLymuWnmHUMDC584p7xAOMviV2eUdQtkL8xrAKgBjjAmRcH8gjFUAxhgTIuF9+LcKwBhjQifMawCrAIwxJgTyHggTzqwCMMaYULAfghljTOUV5sd/qwCMMSY0yvSBMCFhFYAxxoRImB//rQIwxphQCPZhL+XJKgBjjAmVMK8BrAIwxpgQsWGgxhhTSVkfgDHGVEYCEVYBmMPlhznLePjFz/F4lYvP7Mktlxd8RvS+nFxu/98HLF6xkbg6Nfi/R68kJSk+f/umLVmccvkT3Hn1AG66pA8Ab4ydwceT5iACRzdP4tn7L6Fa1eig4vnxt+WMeOULPF4vF5zegxsv7ltge05OLkOf+oilKzcSW6cmL/z3chol1iVnfy4PPT+eJSs3ICI8+O/B9OjYAoDL7nyV9MwdVHVjePupG4iPq10h4wH7zEqie7O63Na3BRERwuTfU/nw1/UHpTnl6Hpcc3xTFFi1dSePTlpGi/q1uOvUltSsGoXXq7z3yzq+X55e4vJLJ7xrgIhQZSwiY0Rkq4gs8Vs/XEQ2icgidzrdZ9t9IrJKRFaIyGk+63f6zJ8uIn+JSOMyjvcqERkVqvSho1fkDQAAFvpJREFU5vF4efC58bz/zI388MG9fPndAlb+vaVAmk8mzyGmdg1mjX2Q6y88mcf/b1KB7cNfnsApPY7JX05N386Y8T/y1Vt3Mv39e/F4lYnTFwQdzyMvfc4bT1zPlDHDmPz9QlatLRjPp1N/JaZWDb57/36uOq83T78xGYBxX80BYPKbQ3ln5I08+dokvF5v/n7P3H8pE0ffxcTRdwV9IAm3ePJiss8sOBECd/Zvyd2fLubyN3+jX5v6NI2vUSBNo7jqXNazMTd/sJAr3prLS//f3pmHV1Vdffj9EaYAYQwzUVBwAoWA4KwoSAEnUBAs1YI4tlV5sH5Wax1KnYrWqaKfitWqddbKVxBFKgoCyjwpowgiYZQpEAJJ1vfH3hdOwk1yI0QC2e/z3Oeeu8/aa68z3L32cPY645cCkL07l/tHL+SqkdO49e253NylJTWqlH7bN/ZCmEQ+B4tScwDAS0D3QvY9Zmbt/GcMgKQTgP5Aa59vhKSkaCZJXYCngO5mtq/7L8fM/mYFzZulcmTTVCpXqsglXdP5eNK8fDIfT5pH3x4dAbigc1smzViCmQEw9vO5HNGkHse0aJQvT05uHjuzd5OTk0tW9i4aptZKyJ65C1dyZNN6HNGkHpUrVeSCc9P5ZPKCfDLjJ8+nd7eTAeh+zklMmensWbpiLaeltwKgXp0UUmpUZd7iVSU/KWXYHgjXrCQc37gmP2zOImPLTnLyjPHfrOPMVqn5ZC5q25j3Z64m078rY/OO3QB8vymLVZuyANiYuYtNO3ZTu1piPaL9RQl+Dhal5gDM7HPgxxJkuQR4w8yyzWw5sBToFNsp6SzgeeACM1tWMLOkTpImS5rlv4/16QMlvSdprO85/DWSZ5CkxZI+A86IZ1Rhej1pXu8iSfdE8gyVNN9/hvi0hyX9JiJzr6Rb/fZtkqZJmivpvhKcsz1krN9C4wZ19vxuVL82Geu35JNZE5GpWDGJmtWrsmnLdnZkZTPitfEMHZTfXzeuX5vr+5/LKZfdR/ted5NSPZlzOh2XkD1rN2yhUf3aEXtqsXbDlgIyW2ncwMlUTEoipXoym7Zu57ijmzB+8nxycnP5PmMjCxavYs26zXvy3TH8DS6+7lGefmXcnsrwULMHwjUrCfVTqrBua/ae3+u3ZZNao0o+mbQ61Uirm8yIAek8e2V7OrWou4+e4xunUDFJ/OAdQmlT1nsAB2sO4HeSrgKmA7ea2SagKTA1IrPKpwFUAT4AOpvZwkJ0LgTONrMcSV2BB4DL/L52QDqQDSyS9BSQA9wHdAC2AJ8Cs0qotxPQBtgBTJM0GjBgEHAKzrl/6R3MG8DjwAif93Kgu6RuQCuvS8AoSWd7B7oHSdcB1wE0S4sz+hXnP1VwGXq8/50kHh05lmsv70z1avn/UJu37uDjSfOZ8tbd1ExJ5oY//YN3P5rOZb84eV9FxZuzz40eryIQok+PTny7ch2X3vg4TRvWIb11c5KSXFvlkTsG0Kh+LTJ37OSme1/m3+Nm7GmRHkr2FGZUuGYlIb8tSRVEszrJ3PT6bBqkVOHvA9L59chpe3oE9apX5q4Ljuf+MQvjHmtpEEJB7MszwDDc1RsGPApcTfyeUOw67QYmA4OBWwrRWwt4WVIrny/axxtvZlsAJH0NHAmkAhPMbL1PfxM4poR6x5nZRp//PeBML/O+mW2PpJ9lZk9KaiCpCVAf2GRmKyXdDHRjr/OpgXMI+RyAmT0HPAfQrn2Hfe7fxg1qkbFu057fa9ZvplFqzbgyTRrUJicnl63bd1K7ZjVmfb2C0RNmc/8zo9iamYVUgSpVKlG/TgppjetSr04NAHqcfRIz5i1PqDJplFqLNev3tgDXrN9Cg3r5hyIa1a9FxrrNNKpfm5zcXLZtz6J2zWpI4s7fXLJHrt9NT9K8aeqePAA1qlXlovPSmbtwZUKVSVmzB8I1Kwnrt2XToOZeZ1c/pQobMnflk1m3LZuvV28lN8/I2LKT7zfuoFmdZBau2Ua1ykn8tc+JPD9xOV+v3lqisveHsl39l+4cQFzMbK2Z5ZpZHm5IJzbMswpIi4g2A1b77Txci7mjpDsLUT0M+NTM2gAXAdGXsGZHtnPZ6/gSaQgUpbdgfqPoa/4O0Afoh+sR4OUfjMyJtDSzkQnYlY+2xx3B8u83sHL1RnbtzuGDT2Zx/hlt8smcf0Yb3v5wGgCjJ8zhjPatkMR7I25m6jv3MPWdexjc9xxuurIrgy47iyYNazNrwQqydu7CzJg0YwktmzdMyJ4Tj0vjux828H2Gs2f0p7PocnrrfDLnndaa9z+eDsDYz+ZyWrqzJ2vnLnZkuUv2xfRFJCUl0bJ5I3Jyc/lxi3seYHdOLp9O/Waf8e9DxR4I16wkLMzYRrM6yTSuVZWKFUSX4xswaemGfDITl2wg/Qg3PFUruRLN6iazenMWFSuIB3q3YeyCtUxY9HM9/ZP48E+5GgKS1NjMMvzP3kDsKaFRwL8k/Q1ogmsFfxXLZ2Y7JF0ITJS0Nk4lWQv4wW8PTMCUL4EnJNUDtgJ9gTlx5IrSe76kukAW0AvXk8kDXpL0EK5y7w1c6eXfwDm9VOAcn/YRMEzSa2aWKakpsNvM1iVwDHuoWDGJYUMvY8DQZ8nLy6PfBadw7FGNGf7CGNoedwTdzmxD/wtP5ZZhr3JGv79Qu2Y1Rtx7VZE627duTs9z29L96keomFSB1sc0Y8DFpydmT1ISd990KYNvf47cPKNPj060at6IJ/4xljbHNqPL6W3o2/MUbnvwX3S98gFqpVTjsbvcadq4OZPBtz+HKoiGqbUYfscVgHsEcfDtz5OTk0tuXh6ntz+Gy3ueekjaA+GalYRcMx4bt4RHLz+JChKj52Xw3YYdDD6zOQvXbOOLpRv5avmPdGpRh1cGdyTXjGcmfMvWnTl0O6EhbdNqUTO5Ej3aOOfzwJiFLF2XWUyp+09ZXwmsnzIhk5Bi6XWgM66yWwvcY2YjJb2CG5M34Dvg+phDkPRHXCWaAwwxsw99eqaZ1fDbabjhkSFm9kGkvNOAl4H1wH+BK82suaSBwMlm9jsv9x/gETObIGkQcAeQAcwGkmJyCertCVQHWgL/MrP7fJ6h/jgAXjCzxyP65gEbzOzcSNotwDX+Zybwq3gT3THate9g4yd+Wdjun50d2bkH24QyT7UqScUL/YyUxWt26TNTDrYJ+Zh+17kzzOwnT1a0a9/Bxn2e2P+0QUql/Srrp1JqDiBQegQHcOgRHEDxHI4O4JMEHUD9g+QAwkrgQCAQKBVEhfAUUCAQCJQ/YiuByzI/+1NAgUAgECgbhB5AIBAIlBJlvQcQHEAgEAiUEmX9MdDgAAKBQKA0OMiLvBIhOIBAIBAoBQ6FSeDgAAKBQKCUCENAgUAgUE4p6z2A8BhoIBAIlBIH8oUwkrr7d48slfSHOPurSHrT7/9SUvPidAYHEAgEAqXFAfIA/u2ITwM9gBOAK/xbFKMMxoWZbwk8BjxcnN7gAAKBQKAUEFBBSuiTAJ2ApWb2rZntwkUWvqSAzCW4wJXgQs93UTFvpAlzAIcgc2bN3JBao9KKA6AqFdhQrNTPS1mzKdhTNGXNHjhwNh25P5lnzpzxUXIlpRYvCUBVSdMjv5/zL4GK0RT4PvJ7Fe6tg8ST8W8w3ALUo4hzERzAIYiZ1T8QeiRNPxgRCIuirNkU7CmasmYPlB2bzKx78VIJU9QbE0sik48wBBQIBAJln6LemLiPjKSKuJdZ/ViU0uAAAoFAoOwzDWglqYWkykB/3FsUo4wCfu23+wD/tWJe+BKGgMo3zxUv8rNT1mwK9hRNWbMHyqZN+4Uf0/8d7hWyScCLZrZA0p+B6WY2ChgJvCJpKa7l3784veGNYIFAIFBOCUNAgUAgUE4JDiAQCATKKcEBHCYUt0zcy+RKmu0/oyLpLfzS8SV+KXlln36vpN/77aqSxkm6p4DO7yTN8zqn+7S6XnaJ/67j0wdK+rvfriDpZUkvFlysIumqiJ2zJZmkNn5fR0nz/XE+FsnzqqReko719mRJWiFpq6QhB8CmKn5f7FjPjuzbxyZJt0jaLGmlLz9V0gIv91NtaCBpgqTtkh6PpKdIGuOv/wJJ9/v86/zvd7xtP/jvuZImSRod0RHTu0jSLyLpmZHtnpF7ZJ2k+ZF9D/lznRe5bj0lXSNpvaTVkrIlZcT0Sxrs5ZdI+lVEf4ak2l6mk6Tlkk4qeD8nSvT8lob8oUxwAIcBSmyZOECWmbXzn4sj6Q8Dj5lZK2ATbkl5VH9l4F1ghpndF0fvuV5n7NnrPwDjvb7x/ndUn4BngUrANQWfVDCzf8bsBAYBS8wsVtk869NaAa0lnV/AlrXALuBm4ChgB/D+/toE3ADsMrMTge7A3yIVdEGbrgWuBT4EhgC9gAm4RTqv7ocNO4A/ArcXSDfgYTM7FmgPnAvM93bWAdb48ACvAF+Z2Um4RwZb+XKfBjoAjX2eEf6eitrWBXjK73/af0f5AHgByI7cY2P8vg+B9UBN4HSvvwFwpz+mU4HhEb25vsx2wJtAXzObS+CAExzA4UEiy8Tj4iud83BLx8EtJe8VEano9S0xs7g9izhEl6QX1AfwBG6F4lVmlleMriuA172taUBVM5vmK8dXCuhOAcYCL5vZ80AXYJmZrTgANp2Aq7QxszXAdiC9EJsuA6biKrLqwNHAMqDF/thgZplm9gWwM076Z347G5iFc+Q/4irdWJl3A9389jdAsqTbgZ7AcDPbambLgaW4ewoASWcBzwMXmNkyM/ucAs+Xm9kUIJP4HAW8YWbZEf2/xV0rcOe2OvC4mS3zaa1xjvuXQAVJkyXN8t/HersGSnpP0ljfc/hrxOZBkhZL+gw4I55Rvnexj15Pmte7SJFer6Shvhc3X9IQn/awpN9EZO6VdKvfvk3SNN/ritd4OqgEB3B4EG+ZeNM4clUlTZc0VVKs8qkHbDaznELy/g+QY2ZDCinbgI8lzZB0nU9raGYZAP67QUT+l7jWZv9ImXHxzulyvANI4DifwD37/KT/3T+Sd39tmgP0kpQk6WggHbfoJp5NFYGzgcreJgErD4ANxSI3rNQT+K9PqsTe8AC7gO1+eCXWO7ga+Bj4tsAxxM5rFVzrvpeZLUzAhCq+snvR2wLuXN0o6S1JTb3+o71dMf2vkX8l6/8B13vHshA428zScU7sgYhcO6AfcCLQT1KapMbAfbiK/3ycg4lHUXo7AQO8/r6STpbUAdfTOwXXa7lWUjqugdQvkvdy4G1J3XC9rE5eTwdFhg7LAsEBHB4kugT8CD9M80vgcV+RFZd3EnCapGMKKfsMM2uPG376bQI3+ExcjJVOxciBGy7YFKl4irN1PNBbbry9MnAx8HYC5SRi0/O44aUZwCPAFCCnEJu24YbVurK3F1DcmpuSnJe4SKqEGzJ51Pd6CmOot2sBzkGkxZGJndfdwGQKDAsWwrPAVlxllwE8Cvwb+CduuOcz4B8F8sT0p5P/Wn4CXCOpAm5F69t+zuExXO8gxngz22JmO4GvcefwFGCCma33Tu/NQuwtSu84M9toZlnAe8CZ/vO+mW03s0yffpaZzQIaSGoiqS3unl2J6211w/XIZgLH4YfdygrBARwexFsmviEyGXcxgJmt9t/f4sak03GBomrLLR2P5Y0uMf8cN479oaQmBQuO6FyH67J3Atb6Vhj+e10ky0JcC+lNSa29TJ+Ire0istEWfGHHGbX1NeBFYAxueGWmma31+/bLJjPbbWa3+LHt3rjW85LCbDKzkd6Oa3F//h7AugN0XvbB95ZGAvPNLDqBuZu94QEqA3VxreIHcZV0T1xL+axCzmuet6ujpDsjesYAR0u6O5JvHYAfvnoe6GRmG3C9nzTcAq2OXv8ynxbT34L8wc1uxPUOngKGAZ+aWRvgIqBqRC47sp3LXkebyAKnovQWzG8UHbj5Hdzq2364HgFe/sHInEhLf1+UGYIDODyIt0z8nciNN0pSHUlVACSl4v70X/tx609xNy+4peQfRJWb2bu4SbqxfvgAr6e6pJTYNq61M5/8S9Lj6ZuMm1QdLekIM4vaOtvrS/I2vRHJ9z2QLffUjYAr4+gejnNaTwBvRXbtl03+WKt523oAmWa2uDCb5CY5wUWmbIob0qiEG0L4yeelCB7EVWC/L5C+LXLcf8FVuBfjK07fu7oeuFpSB0ktcK3UryJ27QAuBAZIGuxb1T1x8yt/jpTVKLLdG5jvHd0o3D3ZBzf+3woYgXOKAMnennRJsZ5Grs9zEq5R8YNPH1jMeQD4EugsqZ7vFfUtRK5WEXrPl3tyLBnXmPgCd1/1klTN3++9gYle/o3IMcbm0z7CndcaAJKaRu6LsoGZhc9h8MH9IRfjWlZ/jLP/dGAebix7HjA4su8o3B9+KW7IpIpPvxf4fUTuXtyfoGok3xz/WRArFzevMB7XQh4P1PXpA4G/R/QNwnXb68WxtyswKU76Kb6sZfjxdZ/+Km6cGqAarkJ5K7J/v2zCjVkvwk2ejgPSirIJVzFsBpbjJqOFqyTW7Od5WYWbgN3mt48FmuNaqF8Ds/1nKq6Fvxv3pM1a3OTxar9/KbA8ovclL7sM6BFJz4xsp/nj+TyiexVueOhvuIlxwz2F9Q3uqaLhuEZBhrfj25h+XO8oz9tyVUT/BqC2l6njbVqLq4SHAd8Vct7+A3SOnMPFuGGnJ6JyEfnTvEw8vW8Bo/01vyeSZ6g/nvnAkAL65uF6FNG0W3z6PNyw4dEHu66IfkIoiEAgECinhCGgQCAQKKcEBxAIBALllOAAAoFAoJwSHEAgEAiUU4IDCAQCgXJKcACBwxLtjXw6X9LbsWf4f6KuzpL+47cvViHRVv3+2tG4MCUoY0/k1UTSC8i8JKlPUTIF5JsrEskzUH4JDiBwuBKLfNoG91z6DdGdcpT4/jezUWb2UBEitYESO4BA4GAQHECgPDARaOlbvt9IGoELz5AmqZukKZJm+p5CbNVmd0kLJU0CLo0pUv7Y/Q0lvS9pjv+cDjyEC5EwW9JwLxc3IqSkP8pFm/wEt6CrSCRd6/XMkfRugV5NV0kT5SJgXujlkyQNj5R9/f6eyMDhRXAAgcMauRhHPXArMcFVtP80FwFyO3AX0NVcQLvpwFBJVXGxbC7CxchptI9ix5PAZ2bWFheHfwEuxv8y3/u4TYVEhJSLLNkfF4/pUlyMnOJ4z8w6+vK+IX+AtubAOcAFwLP+GAYDW8yso9d/rQ/1EAgAxUcoDAQOVZIlxeLnTMQFSmsCrDCzqT79VFyo4C9cGB8q45brH4cLk7AE3NvGgOvYl/NwIQwws1xgi/aGQI4RjQgJUAPnEFJwkSV3+DJGUTxtJP0FN8xUAxdrJsZb5oKwLZH0rT+GbsBJkfmBWr7sxQmUFSgHBAcQOFzJMvdGsT34Sn57NAkX9veKAnLtSCyaZCLEIkL+b4EyhvyEMl7CxTuaI2kg0Dmyr7DolTeZWdRRIKl5CcsNHKaEIaBAeWYqcIaklgA+yuMxuNDMLeTelwDurWTxGI8LWxwbb6+JC9KWEpEpLCLk57h3FyT7iKoXJWBvCpDhI1wOKLCvr9z7hI/GBelb5Mu+0csj6RgfxTIQAEIPIFCOMbP1viX9unyobOAuM1ss93az0ZI24F6K0yaOiluA53wI41zgRjObIukL/5jlh34e4Hhgiu+BZAK/MrOZkt7EReZcwd6wwkXxJ1yo4xW4OY2oo1mEi3zZELjBzHZKegE3NzBTrvD17PsaykA5JkQDDQQCgXJKGAIKBAKBckpwAIFAIFBOCQ4gEAgEyinBAQQCgUA5JTiAQCAQKKcEBxAIBALllOAAAoFAoJzy/2g9eK8qRzLyAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Visualize Confusion Matrix\n",
    "from sklearn.metrics import plot_confusion_matrix\n",
    "np.set_printoptions(precision=2)\n",
    "\n",
    "class_names = [\"0-50K\",\"50K-70K\",\"70K-90K\",\"90K-120K\",'120K-150K','150K and above']\n",
    "# Plot non-normalized confusion matrix\n",
    "titles_options = [(\"Confusion matrix, without normalization\", None),\n",
    "                  (\"Normalized confusion matrix\", 'true')]\n",
    "\n",
    "\n",
    "for title, normalize in titles_options:\n",
    "    \n",
    "    disp = plot_confusion_matrix(classifier, X_test, y_test,\n",
    "                                 display_labels=class_names,\n",
    "                                 cmap=plt.cm.Blues,\n",
    "                                 normalize=normalize)\n",
    "    disp.ax_.set_title(title)\n",
    "    \n",
    "    print(title)\n",
    "    print(disp.confusion_matrix)\n",
    "\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.711340206185567"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Mean accuracy on the given test data and labels\n",
    "SGD.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [],
   "source": [
    "#save the model to disk\n",
    "filename='finalized_model.sav'\n",
    "pickle.dump(SGD, open(filename, 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.6701030927835051\n"
     ]
    }
   ],
   "source": [
    "#load the model from disk\n",
    "loaded_model = pickle.load(open(filename, 'rb'))\n",
    "result = loaded_model.score(X_test, y_test)\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using Count vectorizer and TFIDF transformer with SGD did much better than previous Word2Vec and RandomForest. Looks  like TFIDF was able to retrieve valuable information. \n",
    "\n",
    "**Parameters**\n",
    "SGD = Pipeline([\n",
    "    ('vect', CountVectorizer(max_df=0.5, min_df=0.01, ngram_range=(1,3), stop_words='english')),\n",
    "    ('tfidf', TfidfTransformer()),\n",
    "    ('clf', SGDClassifier(alpha=0.0001, max_iter=500, n_jobs=3))\n",
    "])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "3tiCnAPtde3l"
   },
   "source": [
    "### Random Forest Classifier with Grid Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "NxQSFu9tde30"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performing grid search...\n",
      "pipeline: ['vect', 'tfidf', 'rf']\n",
      "parameters:\n",
      "{'rf__criterion': ('gini', 'entropy'),\n",
      " 'rf__max_features': ('auto', 'sqrt', 'log2'),\n",
      " 'rf__min_samples_split': (2, 6, 8),\n",
      " 'rf__n_estimators': (30, 40, 50),\n",
      " 'tfidf__smooth_idf': (True, False),\n",
      " 'tfidf__use_idf': (True, False)}\n",
      "Fitting 5 folds for each of 216 candidates, totalling 1080 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:   23.2s\n",
      "[Parallel(n_jobs=-1)]: Done 192 tasks      | elapsed:  1.7min\n",
      "[Parallel(n_jobs=-1)]: Done 442 tasks      | elapsed:  3.7min\n",
      "[Parallel(n_jobs=-1)]: Done 792 tasks      | elapsed:  7.3min\n",
      "[Parallel(n_jobs=-1)]: Done 1080 out of 1080 | elapsed: 10.3min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done in 617.912s\n",
      "\n",
      "Best score: 0.546\n",
      "Best parameters set:\n",
      "\trf__criterion: 'gini'\n",
      "\trf__max_features: 'sqrt'\n",
      "\trf__min_samples_split: 8\n",
      "\trf__n_estimators: 40\n",
      "\ttfidf__smooth_idf: False\n",
      "\ttfidf__use_idf: False\n"
     ]
    }
   ],
   "source": [
    "pipeline = Pipeline([\n",
    "    ('vect', CountVectorizer(max_df=0.5, min_df=0.01, ngram_range=(1,2), stop_words='english')),\n",
    "    ('tfidf', TfidfTransformer()),\n",
    "    ('rf', RandomForestClassifier(n_jobs=3))\n",
    "])\n",
    "\n",
    "# uncommenting more parameters will give better exploring power but will\n",
    "# increase processing time in a combinatorial way\n",
    "parameters = {\n",
    "    \n",
    "    'tfidf__use_idf':(True, False),\n",
    "    'tfidf__smooth_idf':(True, False),\n",
    "    'rf__n_estimators':(30,40,50),\n",
    "    'rf__criterion':(\"gini\",\"entropy\"),\n",
    "    'rf__min_samples_split':(2,6,8),\n",
    "    'rf__max_features':(\"auto\",\"sqrt\",\"log2\")\n",
    "    \n",
    "}\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    # multiprocessing requires the fork to happen in a __main__ protected\n",
    "    # block\n",
    "\n",
    "    # find the best parameters for both the feature extraction and the\n",
    "    # classifier\n",
    "    grid_search = GridSearchCV(pipeline, parameters, n_jobs=-1, verbose=1)\n",
    "\n",
    "    print(\"Performing grid search...\")\n",
    "    print(\"pipeline:\", [name for name, _ in pipeline.steps])\n",
    "    print(\"parameters:\")\n",
    "    pprint(parameters)\n",
    "    t0 = time()\n",
    "    grid_search.fit(X_train, y_train)\n",
    "    print(\"done in %0.3fs\" % (time() - t0))\n",
    "    print()\n",
    "\n",
    "    print(\"Best score: %0.3f\" % grid_search.best_score_)\n",
    "    print(\"Best parameters set:\")\n",
    "    best_parameters = grid_search.best_estimator_.get_params()\n",
    "    for param_name in sorted(parameters.keys()):\n",
    "        print(\"\\t%s: %r\" % (param_name, best_parameters[param_name]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Performance similar to SGD. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf = Pipeline([\n",
    "    ('vect', CountVectorizer(max_df=0.5, min_df=0.03, ngram_range=(1,3), stop_words='english')),\n",
    "    ('tfidf', TfidfTransformer()),\n",
    "    ('rf', RandomForestClassifier(max_features='sqrt', min_samples_split=8, n_estimators=50, n_jobs=3))\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "         steps=[('vect',\n",
       "                 CountVectorizer(analyzer='word', binary=False,\n",
       "                                 decode_error='strict',\n",
       "                                 dtype=<class 'numpy.int64'>, encoding='utf-8',\n",
       "                                 input='content', lowercase=True, max_df=0.5,\n",
       "                                 max_features=None, min_df=0.03,\n",
       "                                 ngram_range=(1, 3), preprocessor=None,\n",
       "                                 stop_words='english', strip_accents=None,\n",
       "                                 token_pattern='(?u)\\\\b\\\\w\\\\w+\\\\b',\n",
       "                                 tokenizer=None, vocabu...\n",
       "                 RandomForestClassifier(bootstrap=True, ccp_alpha=0.0,\n",
       "                                        class_weight=None, criterion='gini',\n",
       "                                        max_depth=None, max_features='sqrt',\n",
       "                                        max_leaf_nodes=None, max_samples=None,\n",
       "                                        min_impurity_decrease=0.0,\n",
       "                                        min_impurity_split=None,\n",
       "                                        min_samples_leaf=1, min_samples_split=8,\n",
       "                                        min_weight_fraction_leaf=0.0,\n",
       "                                        n_estimators=50, n_jobs=3,\n",
       "                                        oob_score=False, random_state=None,\n",
       "                                        verbose=0, warm_start=False))],\n",
       "         verbose=False)"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = rf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6391752577319587"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "SGD outperforms RandomForest in this case. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "5lqFFadrde31"
   },
   "source": [
    "## Model 3: Use Pretrained GloVe Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "xQaSjclkde32"
   },
   "source": [
    "#### Getting the Total Vocabulary\n",
    "\n",
    "Sort vocab by frequency, take top 200, set that = sub set. Use that to get vectors."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "vUF42ZNzde32"
   },
   "outputs": [],
   "source": [
    "total_vocabulary = set(word for jd in data for word in jd)\n",
    "\n",
    "#get glove for the limit_vocabulary "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "YSDuTyA0de35",
    "outputId": "865fbce3-dc5d-4747-ebf3-e24bf182361b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 13272 unique tokens in the dataset.\n"
     ]
    }
   ],
   "source": [
    "max_length=len(total_vocabulary)\n",
    "print('There are {} unique tokens in the dataset.'.format(len(total_vocabulary)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "okhX8DYAde37"
   },
   "outputs": [],
   "source": [
    "#Get the appropriate vectors out of the GloVe file\n",
    "glove = {}\n",
    "with open('glove_6B_50d.txt', 'rb') as f:\n",
    "    for line in f:\n",
    "        parts = line.split()\n",
    "        word = parts[0].decode('utf-8')\n",
    "        if word in total_vocabulary:   \n",
    "            vector = np.array(parts[1:], dtype=np.float32)\n",
    "            glove[word] = vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "1wK-2uUdde3-",
    "outputId": "f816c5de-66d5-4527-eb6d-e5d7467ab3dd"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 5.3101e-01, -5.5869e-01,  1.7674e+00,  4.4824e-01,  2.2341e-01,\n",
       "       -3.4559e-01, -7.7679e-01, -9.6117e-01,  1.1669e+00,  7.4279e-02,\n",
       "        8.1470e-01, -5.9428e-02,  6.4599e-02,  1.5176e-03,  9.9179e-02,\n",
       "        3.6602e-01, -9.8724e-01, -8.3913e-01,  1.5917e-01, -7.7603e-01,\n",
       "        7.3474e-01, -6.4861e-01,  4.6174e-01,  8.8162e-03,  5.1738e-01,\n",
       "       -6.5976e-01, -7.4010e-01, -1.3928e-01,  8.1094e-02,  2.0657e-01,\n",
       "        3.5652e+00, -8.2264e-01,  5.7360e-01, -1.7268e+00,  6.2356e-03,\n",
       "        6.7672e-02, -2.3411e-01,  3.5163e-02,  2.6507e-01, -2.9966e-01,\n",
       "        7.4323e-01, -4.5027e-01,  1.9406e-01,  4.8611e-01, -4.3075e-01,\n",
       "       -2.5210e-01,  1.2774e+00,  1.5815e+00,  6.5838e-01, -2.0978e-01],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Each word has a 50 dimension vector space\n",
    "glove['data']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "zpGfK25hde4B"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(next(iter(glove)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Oedt1e-Zde4G"
   },
   "outputs": [],
   "source": [
    "class W2vVectorizer(object):\n",
    "    \n",
    "    def __init__(self, w2v):\n",
    "        # Takes in a dictionary of words and vectors as input\n",
    "        self.w2v = w2v\n",
    "        if len(w2v) == 0:\n",
    "            self.dimensions = 0\n",
    "        else:\n",
    "            self.dimensions = len(w2v[next(iter(glove))])\n",
    "    \n",
    "    # Note: Even though it doesn't do anything, it's required that this object implement a fit method or else\n",
    "    # it can't be used in a scikit-learn pipeline  \n",
    "    def fit(self, X, y):\n",
    "        return self\n",
    "            \n",
    "    def transform(self, X):\n",
    "        return np.array([\n",
    "            np.mean([self.w2v[w] for w in words if w in self.w2v]\n",
    "                   or [np.zeros(self.dimensions)], axis=0) for words in X])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "E0Ao18wWde4I"
   },
   "source": [
    "To be able to use in a pipeline, need fit and transform"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "NwerYehyde4J"
   },
   "source": [
    "### Random Forest with Grid Search"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "UEbKJIBkde4J"
   },
   "source": [
    "http://nadbordrozd.github.io/blog/2016/05/20/text-classification-with-word2vec/\n",
    "TF-IDF embedding, focus on important words."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(data, target_class, test_size=0.1, random_state=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "qVfegx6Ode4K"
   },
   "outputs": [],
   "source": [
    "#Use pipeline\n",
    "from sklearn.ensemble import RandomForestRegressor, RandomForestClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.svm import SVR\n",
    "from sklearn.linear_model import LinearRegression, SGDClassifier\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.model_selection import cross_val_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Performing grid search...\n",
      "pipeline: ['Word2Vec Vectorizer', 'rf']\n",
      "parameters:\n",
      "{'rf__criterion': ('gini', 'entropy'),\n",
      " 'rf__max_features': ('auto', 'sqrt', 'log2'),\n",
      " 'rf__min_samples_split': (4, 6, 8),\n",
      " 'rf__n_estimators': (30, 50, 80)}\n",
      "Fitting 5 folds for each of 54 candidates, totalling 270 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 4 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  42 tasks      | elapsed:   41.2s\n",
      "[Parallel(n_jobs=-1)]: Done 192 tasks      | elapsed:  2.9min\n",
      "[Parallel(n_jobs=-1)]: Done 270 out of 270 | elapsed:  4.1min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done in 243.937s\n",
      "\n",
      "Best score: 0.467\n",
      "Best parameters set:\n",
      "\trf__criterion: 'gini'\n",
      "\trf__max_features: 'log2'\n",
      "\trf__min_samples_split: 8\n",
      "\trf__n_estimators: 80\n"
     ]
    }
   ],
   "source": [
    "pipeline =  Pipeline([('Word2Vec Vectorizer', W2vVectorizer(glove)),\n",
    "              ('rf', RandomForestClassifier(n_jobs=3))])\n",
    "\n",
    "parameters = {\n",
    "    'rf__n_estimators':(30,50,80),\n",
    "    'rf__criterion':(\"gini\",\"entropy\"),\n",
    "    'rf__min_samples_split':(4,6,8),\n",
    "    'rf__max_features':(\"auto\",\"sqrt\",\"log2\")\n",
    "    \n",
    "}\n",
    "                \n",
    "if __name__ == \"__main__\":\n",
    "    # multiprocessing requires the fork to happen in a __main__ protected\n",
    "    # block\n",
    "\n",
    "    # find the best parameters for both the feature extraction and the\n",
    "    # classifier\n",
    "    grid_search = GridSearchCV(pipeline, parameters, n_jobs=-1, verbose=1)\n",
    "\n",
    "    print(\"Performing grid search...\")\n",
    "    print(\"pipeline:\", [name for name, _ in pipeline.steps])\n",
    "    print(\"parameters:\")\n",
    "    pprint(parameters)\n",
    "    t0 = time()\n",
    "    grid_search.fit(X_train, y_train)\n",
    "    print(\"done in %0.3fs\" % (time() - t0))\n",
    "    print()\n",
    "\n",
    "    print(\"Best score: %0.3f\" % grid_search.best_score_)\n",
    "    print(\"Best parameters set:\")\n",
    "    best_parameters = grid_search.best_estimator_.get_params()\n",
    "    for param_name in sorted(parameters.keys()):\n",
    "        print(\"\\t%s: %r\" % (param_name, best_parameters[param_name]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf = Pipeline([('Word2Vec Vectorizer', W2vVectorizer(glove)),\n",
    "              ('rf', RandomForestClassifier(n_jobs=3, criterion='gini',  max_features='log2', min_samples_split=4, n_estimators=150))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "         steps=[('Word2Vec Vectorizer',\n",
       "                 <__main__.W2vVectorizer object at 0x144a3e7b8>),\n",
       "                ('rf',\n",
       "                 RandomForestClassifier(bootstrap=True, ccp_alpha=0.0,\n",
       "                                        class_weight=None, criterion='gini',\n",
       "                                        max_depth=None, max_features='log2',\n",
       "                                        max_leaf_nodes=None, max_samples=None,\n",
       "                                        min_impurity_decrease=0.0,\n",
       "                                        min_impurity_split=None,\n",
       "                                        min_samples_leaf=1, min_samples_split=4,\n",
       "                                        min_weight_fraction_leaf=0.0,\n",
       "                                        n_estimators=150, n_jobs=3,\n",
       "                                        oob_score=False, random_state=None,\n",
       "                                        verbose=0, warm_start=False))],\n",
       "         verbose=False)"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5360824742268041"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ZpTsS1EWde4Z"
   },
   "source": [
    "The score is lower than the first Word2Vec model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SGD Classifier in the pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "4FHOTQx4de4Z"
   },
   "outputs": [],
   "source": [
    "SGD =  Pipeline([('Word2Vec Vectorizer', W2vVectorizer(glove)),\n",
    "              ('SGD', SGDClassifier(alpha=0.0001, max_iter=80, n_jobs=3))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LErTi1sdde4c"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Applications/anaconda3/lib/python3.7/site-packages/sklearn/linear_model/_stochastic_gradient.py:557: ConvergenceWarning: Maximum number of iteration reached before convergence. Consider increasing max_iter to improve the fit.\n",
      "  ConvergenceWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "         steps=[('Word2Vec Vectorizer',\n",
       "                 <__main__.W2vVectorizer object at 0x13c70d898>),\n",
       "                ('SGD',\n",
       "                 SGDClassifier(alpha=0.0001, average=False, class_weight=None,\n",
       "                               early_stopping=False, epsilon=0.1, eta0=0.0,\n",
       "                               fit_intercept=True, l1_ratio=0.15,\n",
       "                               learning_rate='optimal', loss='hinge',\n",
       "                               max_iter=80, n_iter_no_change=5, n_jobs=3,\n",
       "                               penalty='l2', power_t=0.5, random_state=None,\n",
       "                               shuffle=True, tol=0.001, validation_fraction=0.1,\n",
       "                               verbose=0, warm_start=False))],\n",
       "         verbose=False)"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SGD.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "i8qyNGZtde4e"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.21649484536082475"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SGD.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "SGD performs worse than RandomForest in this case. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "MfC_eg3pde4g"
   },
   "source": [
    "## Model 4: Deep Learning Models with Word Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "xPhRNfUHde4g",
    "outputId": "f061e79d-c2bd-4847-f5ff-a12378980948"
   },
   "outputs": [],
   "source": [
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.layers import Input, Dense, LSTM, Embedding\n",
    "from keras.layers import Dropout, Activation, Bidirectional, GlobalMaxPool1D\n",
    "from keras.layers.convolutional import Conv1D\n",
    "from keras.layers.convolutional import MaxPooling1D\n",
    "\n",
    "from keras.models import Sequential\n",
    "from keras import initializers, regularizers, constraints, optimizers, layers\n",
    "from keras.preprocessing import text, sequence\n",
    "from keras.layers import Flatten\n",
    "\n",
    "from keras.utils import to_categorical\n",
    "\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  Word Embedding"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will map each job description into a real vector domain, a popular technique when working with text called word embedding. This is a technique where words are encoded as real-valued vectors in a high dimensional space, where the similarity between words in terms of meaning translates to closeness in the vector space.\n",
    "\n",
    "Keras provides a convenient way to convert positive integer representations of words into a word embedding by an Embedding layer.\n",
    "\n",
    "We will map each word onto a 64 length real valued vector (embedding size). We will also limit the total number of words that we are interested in modeling to the 1000 most frequent words, and zero out the rest. Finally, the sequence length (number of words) in each review varies, so we will constrain each job description to be 500 words, truncating long reviews and pad the shorter reviews with zero values. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "MlfCJ3Sede4m"
   },
   "outputs": [],
   "source": [
    "tokenizer = text.Tokenizer() \n",
    "tokenizer.num_words = 1000  #limit the number of words to keep most common 1000 words\n",
    "tokenizer.fit_on_texts(list(salary_df['combined_text']))\n",
    "list_tokenized_jd = tokenizer.texts_to_sequences(salary_df['combined_text'])\n",
    "\n",
    "X_t = sequence.pad_sequences(list_tokenized_jd, maxlen=400, padding = 'post') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(963, 400)"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_t.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "0dKkpHICde4n",
    "outputId": "d58d6c38-af96-4a5b-a538-f554925fa067"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "963"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(list_tokenized_jd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "mq_ZD2-Ede4j"
   },
   "outputs": [],
   "source": [
    "#Encoding to categorical classes\n",
    "y = to_categorical(target_class.values, num_classes=6, dtype=\"float32\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "i8B6kWwTde4q",
    "outputId": "c09f418c-928a-43cc-c939-7aa53053697b"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(963, 6)"
      ]
     },
     "execution_count": 104,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "n-t9KkH9de4t"
   },
   "source": [
    "#### Train-Test Split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "h1Fkq34-de4u"
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X_t, y, test_size=0.2, random_state=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "f_WUVyTLde4v",
    "outputId": "da10402f-f598-44e7-c926-1b3a3c489541",
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(193, 400)"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "gu7pPgf5de4x",
    "outputId": "b5116b1f-3152-4a9f-d034-c0739f7c6734"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(193, 6)"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create a base model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "nwe00hXOde4_",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "\n",
    "embedding_size = 64  #each word onto a 32 length real valued vector\n",
    "model.add(Embedding(input_dim = 1000, output_dim = embedding_size, input_length = 400)) #try 100, 200, single LSTM and single dense 100\n",
    "model.add(LSTM(100))\n",
    "#model.add(LSTM(64))\n",
    "#model.add(GlobalMaxPool1D())\n",
    "#model.add(Dropout(0.5))\n",
    "model.add(Dense(64, activation='relu'))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(16, activation='relu'))\n",
    "#model.add(Flatten())   #add Flatten layer \n",
    "model.add(Dense(6, activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ROzc4IHWde5A",
    "outputId": "aa9d8204-8392-4f42-c7d9-ab1a4cd4b09d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_14\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_3 (Embedding)      (None, 400, 64)           64000     \n",
      "_________________________________________________________________\n",
      "lstm_6 (LSTM)                (None, 100)               66000     \n",
      "_________________________________________________________________\n",
      "dense_30 (Dense)             (None, 64)                6464      \n",
      "_________________________________________________________________\n",
      "dropout_4 (Dropout)          (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_31 (Dense)             (None, 16)                1040      \n",
      "_________________________________________________________________\n",
      "dense_32 (Dense)             (None, 6)                 102       \n",
      "=================================================================\n",
      "Total params: 137,606\n",
      "Trainable params: 137,606\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "I8NdwrUTde5D"
   },
   "outputs": [],
   "source": [
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ZiBp9FQTde5E",
    "outputId": "7c34871d-05c2-4b20-89e1-1525fe42afc6",
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 770 samples, validate on 193 samples\n",
      "Epoch 1/20\n",
      "770/770 [==============================] - 6s 8ms/step - loss: 1.7918 - accuracy: 0.1636 - val_loss: 1.7894 - val_accuracy: 0.1762\n",
      "Epoch 2/20\n",
      "770/770 [==============================] - 5s 7ms/step - loss: 1.7891 - accuracy: 0.1857 - val_loss: 1.7890 - val_accuracy: 0.1606\n",
      "Epoch 3/20\n",
      "770/770 [==============================] - 5s 7ms/step - loss: 1.7886 - accuracy: 0.1896 - val_loss: 1.7889 - val_accuracy: 0.1554\n",
      "Epoch 4/20\n",
      "770/770 [==============================] - 5s 7ms/step - loss: 1.7841 - accuracy: 0.2104 - val_loss: 1.7863 - val_accuracy: 0.1710\n",
      "Epoch 5/20\n",
      "770/770 [==============================] - 5s 7ms/step - loss: 1.7794 - accuracy: 0.2104 - val_loss: 1.7836 - val_accuracy: 0.1969\n",
      "Epoch 6/20\n",
      "770/770 [==============================] - 5s 7ms/step - loss: 1.7662 - accuracy: 0.2234 - val_loss: 1.7784 - val_accuracy: 0.1917\n",
      "Epoch 7/20\n",
      "770/770 [==============================] - 5s 7ms/step - loss: 1.8039 - accuracy: 0.2468 - val_loss: 1.7740 - val_accuracy: 0.2176\n",
      "Epoch 8/20\n",
      "770/770 [==============================] - 6s 8ms/step - loss: 1.7502 - accuracy: 0.2688 - val_loss: 1.7701 - val_accuracy: 0.2021\n",
      "Epoch 9/20\n",
      "770/770 [==============================] - 5s 7ms/step - loss: 1.7267 - accuracy: 0.2299 - val_loss: 1.7568 - val_accuracy: 0.1813\n",
      "Epoch 10/20\n",
      "770/770 [==============================] - 5s 7ms/step - loss: 1.6990 - accuracy: 0.2455 - val_loss: 1.7560 - val_accuracy: 0.1865\n",
      "Epoch 11/20\n",
      "770/770 [==============================] - 5s 7ms/step - loss: 1.6644 - accuracy: 0.3052 - val_loss: 1.7348 - val_accuracy: 0.2228\n",
      "Epoch 12/20\n",
      "770/770 [==============================] - 5s 7ms/step - loss: 1.6235 - accuracy: 0.2922 - val_loss: 1.7664 - val_accuracy: 0.2228\n",
      "Epoch 13/20\n",
      "770/770 [==============================] - 6s 8ms/step - loss: 1.5767 - accuracy: 0.3247 - val_loss: 1.7688 - val_accuracy: 0.1813\n",
      "Epoch 14/20\n",
      "770/770 [==============================] - 6s 8ms/step - loss: 1.5613 - accuracy: 0.3247 - val_loss: 1.8453 - val_accuracy: 0.2176\n",
      "Epoch 15/20\n",
      "770/770 [==============================] - 6s 8ms/step - loss: 1.4938 - accuracy: 0.3429 - val_loss: 1.8100 - val_accuracy: 0.2124\n",
      "Epoch 16/20\n",
      "770/770 [==============================] - 6s 8ms/step - loss: 1.4998 - accuracy: 0.3442 - val_loss: 1.8899 - val_accuracy: 0.2539\n",
      "Epoch 17/20\n",
      "770/770 [==============================] - 5s 7ms/step - loss: 1.4692 - accuracy: 0.3610 - val_loss: 1.9637 - val_accuracy: 0.2539\n",
      "Epoch 18/20\n",
      "770/770 [==============================] - 6s 8ms/step - loss: 1.4687 - accuracy: 0.3494 - val_loss: 1.8398 - val_accuracy: 0.2539\n",
      "Epoch 19/20\n",
      "770/770 [==============================] - 6s 7ms/step - loss: 1.4636 - accuracy: 0.3610 - val_loss: 1.9818 - val_accuracy: 0.2280\n",
      "Epoch 20/20\n",
      "770/770 [==============================] - 5s 7ms/step - loss: 1.4253 - accuracy: 0.3922 - val_loss: 1.9809 - val_accuracy: 0.2591\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(X_train, y_train, epochs=20, batch_size=50, validation_data=[X_test, y_test])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "XDpePK3ude5H"
   },
   "source": [
    "Hyperparameter testing (layer size, learning rate) \n",
    "Grid Search: https://github.com/autonomio/talos\n",
    "https://machinelearningmastery.com/grid-search-hyperparameters-deep-learning-models-python-keras/\n",
    "Google Collab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "zb4F2jC3de5I",
    "outputId": "7b2298e2-4ba2-496a-f6af-114003a8e467"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train: 0.417, Test: 0.259\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEJCAYAAACE39xMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd3wU1fr48c+TnpCQAAk1dOlViAqCAiIIKCoWRK+KXhV716vee233/mz3a6+IiNjAXhEVUJAOBqQTCU0ILQEkkN7O748zwSWkkt3sbvK8X699bZkzc56dnX129syZM2KMQSmllP8L8HYASiml3EMTulJK1RKa0JVSqpbQhK6UUrWEJnSllKolNKErpVQtoQndS0TkIxH5rIrzLBWRZz0Vk6eJSLyIGBHp5+1YSiMinZ34uns7Fl8nImHOujrP27FUlT/HXhFN6GVwPvDyblOrWcWNwPVVnGcU8Hg16/VpnvjREpERzmcW6c7leoOI7BWR27wdh/JNQd4OwIc1c3l8HvBWideyS5tJRIKNMfkVLdwYk17VgIwxB6s6j1Kq7tA99DIYY/YW34BDJV8zxqS7/EW/VER+EZEcYLyINBGRj0Vkl4hkicg6Efmb6/JLNrk4e6YviMj/ichBZ0/sSRGREmWedXm+V0QeEJEpInJERHaKyB0l6ukqIotEJEdENojIMBEpEJFxZb13ETldROaIyAERSReR+SJyisv04r+s14rIl8573CIiY0ssp7+IrHLqTgT6lLfOReQj4DTgXpd/Qk2daT1E5AcRyRCRfSLygYjEucx7sojMc9bDERH5TUQGikhn4Hun2BFnmRPLi6NETENF5FcRyRWRPSLyPxEJLjF9uYhkisgh5zPq5ExrJCLTRCTNWQebReTmcurqLCLfOu8vQ0QSRWS4y/SlQBPgFed95JSzrDARec7ZBjNFZJmInOUyPVRE3hGR7SKSLSKbRORu1+3NKXe9iKx33v9eEZlUoqq48raBMmKbICJJzjpJEpHbiut12bZuFJEfndi2ichlJZZR/HlnO9vpZBGJ8nTsPs8Yo7cKbsAldlUd93pnwABbgAuBtkBzoA1wN9AbaAfcChQAA13m/Qj4zOX5UiAd+DfQAfgbUAiMKVHmWZfne4E04CbgJOA+J54+zvQgIBn4AegJDAR+BYqAceW83+HAFc776wK86dQT7UwPc+rZAYxz6n4eyAGaOWWigYPAh0A3bHNRsjNfvzLqjQZWAK8DTZ1bANAS+BP4L9DJWa8/AAtc5k0G3nGmnwRcDJwCBAKXO/W2c5ZZv4z6iz/P7s7zNs57esWZdqGzHp5wpocCR4AnnWV3Aa4COjjT33LWd4KzrLOAi8pZ7wnADUB3Zxt4HMgF2jnTGwKpwEPO+2hSzrI+BxY4n3l74B5nWV2c6fWAR5062zrr6AjwN5dl3AlkAXcAHZ31eW9lt4Ey4rodSAHGOPUWr9PrSyw3DbjOqfdx7Hehh1OmvrMePnHW1VnAVuBDT8buDzevB+APNypO6LdWYhlfAa+6PC8toc8tMc+CEvOUltDfKTHPTuA+5/EFQB7Q2GX6WU7MZSb0UmIPwCbnS5znxV+IR13KhAL5LmXucL6UYS5lrqechF7ae3Re+x/wXYnXmjrL6gmI82W8rIxljnDKRlbwPksm9OeA9YC4lLnJSRTB2B9vA5xWxvJmAW9Uc9tbVfx5unzmt1UwT1cnATYp8foPwPPlzPciMMPlM08DHiujbIXbQCnziBP/pSVefxBYWWK5r5QosxCY7Dy+HdgPhJf4jIuwP/5uj91fbtqG7h6Jrk9EJAj4F/aHoAUQgt1gvj9+1mOsKfF8N9C4GvN0BrYbY1Jdpi+rYHmISDPs3vAgZ1mBQATQqqy6jTG5InLQpe4uwG/GGNdmgSUV1V2GvsAZIpJRyrT2xpg1IvIC8IGIXA/8jP2xTD7B+op1ARYb5xvvWAiEA22NMZucZqJ5IjIH+An41Bizyyn7GvCR2F49s4FvjDELy6pMROoDjwEjscdrgrDJp+R6r0hfbFLbUqIFJRS7l15c3+3ANc7yw7E/Ur87k+OBWOc9lae8baCkeGyT0bsi8o7L60GucTlKbitLgAHO4+Jty/U41kLsD0YX597dsfsFTejukVni+b+wzSx3YffwMrF7e6EVLKfkwVSDTaZVnaf42Ig4z6tqGjaB34H9W5qL/cKEVLFudwnA/sP5ZynT9gIYYx4S2/NoFLbJ6HERudYY82E16i1v/Rmn3stFpA9wDraZ50kROdcYM9cY87WItHZiGgr8KCLvGWPKakd/CdtE8g9gM/bA+0ccv94rEoD9bE4uJf5MABEZDzyDbYpZhm1uuduJEyr/+ZW3DZQWF8C12KY1V0WVrA8q/lw8Ebtf0ITuGQOBL40x0wBEJADbjvdHDcexEWgjInHGmDTntVPLm8E5ODUAGG+M+d55LZ6q77lsAC4SkVBjTPHeV2X6n+dx/I/YSuxf6m3GmMKyZjTG/I7dw3zB2QO8DtuGn+cUqejHsaQNwAgREZe99IHYRLvdpd6VToxPichcbDv6XGdaKjAVmOrsxU8WkVuNMaUlsIHAFGPMlwAiUg/bNu/6D7C09VPSSuzedqwxpqx/RQOxxyCOHiAWkQ4u03dimzWGYpv+3KF4me2MMR9XULYfdsfC9flG5/EG4FIRCXfZSx+ITchJwC4PxO4X/PrXyIdtAs4R28uj+KBicy/E8R12D/tdEekpIgOAp7Ebfql7OE7iSgaudnpdnAZMp4xumuV4D5tUJovtaTMSu+dZke1APxFpLSKxzg/MS9gmiGkicoqItBOR4SLytoiEiEi0iLwsIoOc+U4H+mO/+MXLBDhPROKcRFkZr2APKL7krIsLsE1RLxhj8kWkk4g84XzOrUXkbGz79QYAsb2UzheRk0SkG/YA4O9lJHOw283FItJLRHph13vJna7twCARaSEijUpbiDFmLfag6IciMkZE2jrr7QERGe1S12liez11FJH/YnsYFS+jCHuw9x8icruIdHB6ltxVyXVXWlxF2AOc/3aW2VFs76VrROT+EsXHie1F1VFEHgVOx24HAO9i9+inikh3ERmCbd6abozZ6YnY/YUmdM94FNs+NxuYhz0iX6WzQt3BGFOAPTAag+1tMRn4jzO5zC5vwNVAHPaA3AfAq8CeKtZ9CBiNPWj5G/YL9kAlZn0auwe6EXtgq4kxZgf2Cx2KXafrgJeBDOzBv3zsP4j3sYnqU+we8gNOLFuBJ7A9GfZhm78q8x62A+c6da/G/jC/g23nxqm/GzZ5bsKu38nAC870fGyzxhpgvvO+LiqnytudZS4BZgBzgOUlyvwL+29vK3ZPtCx/w+7hPo/91/INdi93hzP9FeBr7Lpahl1/r5R4/y9gm2RuwzYdzsQelzlhxphXgZux/57WAr8Af3fej6tHnPewBttEc6UxZo2zjMPYJq4m2O36M+znfZMnY/cHcuzxHlXbOXvcS7E9OdZ7Ox6lXIlIGPbf4GhjzAxvx+NvtA29lhORS7F9uDdjmw9eBJZrMleq9tGEXvtFA09hu4wdwHbluterESmlPEKbXJRSqpbQg6JKKVVLeK3JJTY21rRp08Zb1SullF9asWLFfmNMXGnTKkzoItIS26e4Kbbv5yRjzEslyhT3FR6FHefiGudkizK1adOGxMTE8ooopZQqQUTKPEGxMnvoBdhRylY6w1OuEJHZxpgNLmVGYkeH64A9OeENXE5SUEop5XkVtqEbY/YU720bY45gT/hoUaLYBcB7xloKxDgDPCmllKohVTooKiJtsAP+lByxrwV2nIZiKRyf9IsHtk8UkcS0tLSSk5VSSlVDpQ+Kir0e4+fAXc6pt8dMLmWW4/pDGmMmAZMAEhISjpuen59PSkoKOTnlnZVeO4SFhREfH09wcHDFhZVSqhIqldDFXnLrc+wVQb4opUgKdmD5YvHYcbmrJCUlhaioKNq0aUOJcZxrFWMMBw4cICUlhbZt23o7HKVULVFhk4vTg+VtYKMx5vkyin2DHZ1PnMH8040xVRrMCSAnJ4dGjRrV6mQOICI0atSoTvwTUUrVnMrsoQ/Aju+8VkRWOa/9E+cqKs54yjOxXRY3Y7stXnuiAdX2ZF6srrxPpVTNqTChO5fMKjf7OGNo3+quoJRSymet/gj+3A4BgRAQDAFBEOjcH30cbKcffRwEgUF/Pa7fHGJaVlhVVengXC4OHTrEtGnTuOWWW6o036hRo5g2bRoxMTEeikwp5RPSU+DLG6u/nAF3wrD/VFyuijShuzh06BCvv/76cQm9sLCQwMCyr/o1c+ZMT4emlPIFG50h2m/9FRq0gaJ8KMyHokL7uKjAeV7g8tiZfvT1fIhp7ZHwNKG7ePDBB9myZQu9e/cmODiYyMhImjVrxqpVq9iwYQMXXnghO3fuJCcnhzvvvJMJEyYAfw1jkJGRwciRIxk4cCCLFy+mRYsWfP3114SHh3v5nSml3CJpBsR1hriOzgtVvX63Z/lsQn/82/Vs2F2yu3v1dG1en0dHdytz+tNPP826detYtWoV8+bN49xzz2XdunVHuxZOmTKFhg0bkp2dzSmnnMLFF19Mo0bHXtYxOTmZ6dOn89ZbbzF27Fg+//xzrrzySre+D6WUF2QegD8WwcB7vB1JmXw2ofuCU0899Zh+4i+//DJffvklADt37iQ5Ofm4hN62bVt69+4NQN++fdm+fXuNxauU8qDfZ4Ipgi6jKy7rJT6b0Mvbk64p9er9dXH4efPmMWfOHJYsWUJERASDBw8utR95aGjo0ceBgYFkZ2fXSKxK1RkFuRAYAjXd9TdpBkS3gma9arbeKtALXLiIioriyJEjpU5LT0+nQYMGREREkJSUxNKlS2s4OqUU6bvguU6wfFLN1pt7BLbMhc7n1vwPSRX47B66NzRq1IgBAwbQvXt3wsPDadKkydFpI0aMYOLEifTs2ZNOnTrRr18/L0aqVB31w4OQ/SckvgOnTqi55Jo8Gwpzfbq5BTShH2fatGmlvh4aGsr3339f6rTidvLY2FjWrVt39PX77rvP7fEpVWclz4aN30CTHrBvLexdU3PNH0kzICIWWvn2jpw2uSilfF9+Nsy8D2I7wpWf2zb01R/VTN0FubBpFnQaac/+9GGa0JVSvm/hC/Z0+3Ofg6gm0GE4rP0MCgs8X/fWXyDvCHQ53/N1VZMmdKWUb9u/2Sb0HmOh7Zn2tV7jIDMVts71fP1J30JIFLQb5Pm6qkkTulLKdxkDM++FoHAY/v/+er3DcAhv4Plml6JCSJoJHYZBUGjF5b1ME7pSynet/wK2zoOhD9umlmJBodDtIkj6DnLce0b5MXYshaz9Pt+7pZgmdKWUb8o5DD/8E5r1hoS/Hz+91zgoyLY9XzwlaQYEhto9dD+gCd1F8WiLJ+LFF18kKyvLzREpVYfNfRIy9sF5L5TeuyT+FGjYznPNLsbY0RXbDYbQKM/U4Waa0F1oQlfKR+xZDcvfhFOugxZ9Si8jAj0vg+0L7TjlnoghfYffNLeAnlh0DNfhc4cNG0bjxo355JNPyM3NZcyYMTz++ONkZmYyduxYUlJSKCws5OGHH2bfvn3s3r2bIUOGEBsby9y5NXDkXanaqqgIZtwDEY3grIfLL9vzMpj3FKz5BM5w8yiISTNAAmz/cz/huwn9+wdh71r3LrNpDxj5dJmTXYfPnTVrFp999hnLly/HGMP555/P/PnzSUtLo3nz5nz33XeAHeMlOjqa559/nrlz5xIbG+vemJWqa1a+C7sSYcwkCK/gKmAN20LLfrDmYxh4t3uHAtg4A1qdDvX85zutTS5lmDVrFrNmzeLkk0+mT58+JCUlkZycTI8ePZgzZw4PPPAACxYsIDo62tuhKlV7ZO6HOY9B64HQc2zl5ul1GaQlwZ5VFZetrP2bIW2jXzW3gC/voZezJ10TjDE89NBD3Hjj8dcPXLFiBTNnzuShhx5i+PDhPPLII16IUKlaaPYjkJdhzwit7N52tzHw/QOw+mNofrJ74kj61t53Ptc9y6shuofuwnX43HPOOYcpU6aQkZEBwK5du0hNTWX37t1ERERw5ZVXct9997Fy5crj5lVKnYA/FsOqD+H026Fx58rPF94AOp4D69w4FMDGGba7ZExL9yyvhvjuHroXuA6fO3LkSK644gr69+8PQGRkJB988AGbN2/m/vvvJyAggODgYN544w0AJkyYwMiRI2nWrJkeFFWqqgrz4bt77QUkzry/6vP3uhw2fgtbfoaOw6sXy+Hdtg2/ogOyPkiMMV6pOCEhwSQmJh7z2saNG+nSpYtX4vGGuvZ+lSrTopdh9sMwbjp0HlX1+Qvy7IUv2g+BS6ZUL5blb9mRHW9dDnGdqrcsDxCRFcaYhNKmaZOLUsq70lNg3tPQceSJJXOAoBDoXjwUQHr14tn4LTTq4JPJvCKa0JVS3vXDg/biyyOfqd5yeo6DghzYUI2hALIO2hOV/Kx3S7EKE7qITBGRVBFZV8b0BiLypYisEZHlItK9OgF5qwmoptWV96lUuTbNsnvEg+6HBq2rt6z4BGjY3vZJP+F4fgBTCF3Oq14sXlKZPfSpwIhypv8TWGWM6QlcDbx0osGEhYVx4MCBWp/sjDEcOHCAsLAwb4eilPe4XoWo/+3VX56IHbBr+wI4tPPElrFxBtRvAc3LGG7Ax1XYy8UYM19E2pRTpCvwlFM2SUTaiEgTY8y+qgYTHx9PSkoKaWlpVZ3V74SFhREfH+/tMJTyngXPwaE/YPy3tg3cHXqOhblPwNpP4Ix7qzZvXiZs+Qn6jK+5i0+7mTu6La4GLgIWisipQGsgHjguoYvIBGACQKtWrY5bUHBwMG3btnVDSEopn7Y/GRa9ZMdiKb4KkTs0aAOt+tuTjAbeU7XEvHmObYP30+YWcM9B0aeBBiKyCrgd+A0otXe/MWaSMSbBGJMQFxfnhqqVUn7HGNvnvORViNyl52Ww/3fY/VvV5ts4A8Ib2vFb/FS1E7ox5rAx5lpjTG9sG3ocsK3akSmlaqd1n8O2X+xViCIbu3/53S6EwJCqHRwtyINNP0KnURDov+dbVjuhi0iMiBQ3gF0PzDfGePCaUEopv5V7BH78px1zpbSrELlDeAM75O3az+wZqJWxfT7kpvt1cwtUrtvidGAJ0ElEUkTkOhG5SURucop0AdaLSBIwErjTc+EqpfzawhfsVYhGPVv6VYjcpec4ey3QLT9XrvzGGRBcD9oN8VxMNaAyvVwur2D6EqCD2yJSStVOh3bA4lehx1jbZ9yTTjrbtoev/sgO3FWeokJ7hmmHYRDs312J9UxRpVTNmPO47XUytAaGmw4Kge4XV24ogJRfITPVb88OdaUJXSnleTt/tcPbnn57zQ1J22scFObChq/LL7fxW3sQtUM1R2n0AZrQlVKeZYw9EBrZBAbcVXP1tugLjU6yfdLLi23jt9B2EITVr7nYPEQTulLKs9Z/ASnL7fjioZE1V6+IPTj6x0Lbfl+afevs2ap+3rulmCZ0pZTn5OfA7MfsBdp7X1Hz9Rdfl3TNJ6VP3zgDEOjkX5eaK4smdKWU5yx9HdJ3wPAnPNtNsSwNWtszP1d/ZJtXStr4rR0qILJ2nLmuCV0p5RkZqbDgeXv2ZbtB3ouj12VwIBl2rzz29YNbIXV9rWluAU3oSilPmfsEFGTDsP96N46uF0Jg6PEHRzfOsPedNaErpVTZ9q2Hle/BKTdA7EnejSU8xg4FsO7zY4cC2PgtNO1Z/Qtr+BBN6Eop9zIGfvwXhNaHQf/wdjRWL2cogM0/2edH9tqeN7XgZCJXmtCVUu6VPBu2zoXBD0JEQ29HY510NkQ0gtXT7fOk7+y9JnSllCpDYT7M+pe9tmfCdd6O5i+BwXYogN+/h+xDtrmlYXuI6+ztyNxKE7pSdY0nr9m7Yirs32QvXOGuy8q5S/FQACvfs9cd7XKe315qriya0JWqSw7thBe6wcx/2Is6uFP2nzD3SWhzhj0I6Wua94FGHWzvm6IC6HK+tyNyO03oStUls/5txyNf/iZMPRcO73bfsuc/a5P6OU/65p6viO2TXpADUc1sgq9lNKErVVds/QU2fAVn/gMunWq7Fr55JmxfWP1lH9wKy96Ek/8GzXpWf3me0vMyQGzf84Dal/5q3ztSSh2vMB++fwBiWsGAO6DbGLjhZwiLgXfPtxeeqE7b+uxH7BC0Zz3svpg9IaYVjP8GhvzT25F4hCZ0peqCXydD2kY45ykIDrevNe5sk3rnUbZnyqfX2Gt+VtX2RbbXyMC7IaqpW8P2iLZn+k53SjfThK5UbZeRBnOfgvZnQecSowqG1Yex78Ow/8DGb+CtoZC2qfLLLiqyY53XbwH9b3Vv3KrKNKErVdv99BjkZ8KIZ0o/WCkCA+6Eq76CrAPw1pCKr/JTbM3HsGcVnP0YhES4MWh1IjShK1WbpayA3z6AfjdDXMfyy7YbBDf+Yk+2+eRqmPUwFBaUXT4vE3563PYW6X6Je+NWJ0QTulK1VVERzLzPXvrtzEqOqRIdD9fOtGd5Ln4Z3r/QNtmUZvErcGQPjHiqVvYY8Uf6KShVW6360I4BPuw/VbteZlAonPc8XPgGpPxquzbu/PXYMod3w6KX7NC0rfq5N251wjShK1UbZR+COY9By9OcvtcnoPcVcN1sOw7KOyNtT5niro0//deebXn2Y24KWLmDJnSlaqN5T9sDnCP/V72zNpv1tO3q7YfAd/fCVzfDH0tg9TTbLt+wrftiVtWmCV2p2mbfBlg+CfpeA817V3954Q3g8o9h8EP22pxTR0FELJxxb/WXrdxKE7pStYkx8P0/IDQKhj7ivuUGBNjxza/4BKKawzlPQFi0+5av3KLChC4iU0QkVUTWlTE9WkS+FZHVIrJeRK51f5jKVxQVGVbvPERhkQeHYFUnbsNXdmjYs/7tmbMhOw6He9bboWiVz6nMHvpUYEQ5028FNhhjegGDgedExMcGQlbukJVXwK3TVnLBa4t44ruN3g5HlZSXCT/+G5r0gIS/ezsa5QUVJnRjzHzgYHlFgCgRESDSKVvO2QjKH+1Jz+bSiUv4Yf1eTm3TkCmLtjFt2Q5vh6VcLXwBDqfAqP9BQKC3o1Fe4I429FeBLsBuYC1wpzGmqLSCIjJBRBJFJDEtrYyTFZTPWbXzEOe/uojt+zOZfHUC0yf0Y3CnOB75eh2LN+/3dngK4OA2WPQy9LgUWp/u7WiUl7gjoZ8DrAKaA72BV0Wk1LMYjDGTjDEJxpiEuLi4E6osv7CI3ILCEw5WVc03q3dz2ZtLCA0K4ItbBjC0SxMCA4RXLj+ZdnH1uOmDFWxNy/B2mOrHf0JAkD2JSNVZQW5YxrXA08YYA2wWkW1AZ2C5G5Z9nHmrN/PMp3MJCgwkIjSEiLBgwkNDqBcaTL3QECLD7X29sBAiw0OIDHNu4SFEhYcSFRZMvfAQggICCAgIIECEABECAwMQKf59E9t31xevulJDiooML/6UzMs/JXNKmwZMvLIvjSJDj06PCgvm7fGncMFri7ju3US+vOV0YiL00IlXJM+B32fak3zqN/d2NMqL3JHQdwBDgQUi0gToBGx1w3JL1SMnkTmhzrgUhUCmc/OwIgSDOI8DyJZwcgPrkR8USVFIFBJan4CIaEIiogmLjCE8qgEBYfUhtL497To0yj4OjbLdvULrQ6A7Vr/7ZecVcu+nq5i5di+X9o3n/43pTmjQ8W2yLRtGMOmqvlzx1jJu/mAl7113KsGB2hO2RhXkwQ8P2CvY97vF29EoL6swo4jIdGzvlVgRSQEeBYIBjDETgf8CU0VkLSDAA8YYjzWsNu12JtR/B0xR2beiQvILC8nNKyAnL5/c/AJy8/PJzbP3+QUFmCJDkX0TGFOEMcae1WwMhuLXiqcbMEV/lS8qoCg3A8k9QmDOEcKyMonkT6LIIkSyCSaLAKm4WcgER0BYNFKc4MPq/5Xsw6KdH4L69qoyRx9HQ3gM1GvskR+Evek53PBeIut2p/PPUZ254Yx2SDn/VBLaNOTpi3twzyereeTrdTw5pke55ZWbLX0dDmyGv31mx2BRdVqFGcEYc3kF03cDw90WUUWi4+2tAsHOLdLjAUFeQRH7M3JJPZJL0uEcUg/ncCD9MEcOHSTj8EFyjhwiN/MQJvcwkSabKMkiiizqF2QRnZtNbGYODQOziQnYSSSbiCjKILTwCIFF+WVXKoH2QrfR8RDdwrlvaS80ULyOwhtUqdlo9c5D3PBeIpm5BUy+OoGhXZpUar6L+sSzJS2D1+ZuoX1cJNef0a7SdapqOLwH5v8fdBwJHYZ5OxrlA3zzP7+fCQkKoHlMOM1jwsstl19oE/++w7nsTc9h3+Ecth3OYWl6DnsP29u+9Bwy8+zefSh5RGF/AJqF5tIqIp8W4fnEh+XQNTKTtsEHCcnYDbtW2kuAFeYdW2FwPZdkHw/14//6AYhqZodVDYsGEWas2c29n6wmNjKUz285nc5NqzA6H3DvsE5sTcvkiZkbaRdXj7M6V+7HQFXD7EfstUJHPOntSJSPEFOdC8NWQ0JCgklMTPRK3b4uI7fgaMLf6yT74sf7Duew/UAW6dn5BAj0adWAIZ0bM6RjLF3q5yKHUyA9BdJ3Ofc77f3hXZCx77i6TFAYhwMbkpwdSX54HL27dia8QXN7bcjIphDVxN5HNKpwzOusvALGvrmEbWmZJ/SjoKrgjyXwzgg44z4Y6uMXZlZuJSIrjDEJpU7ThO5/CosMa1IOMff3NOYmpbJ2VzoATeuHMaRzHEM6NWbASbHUCy3xB6wg1yb29BQ4so/89N38snIdGftT6BaVTfvwDAIyUiE3/fhKA4Jsu31xgo+Mg3px9rV6sc7jOFKLorhgygYCAoP56tYBxEVpu67bFRXCpEGQ9SfcthxC6nk7IlWDNKHXcqmHc5i3ySb3Bcn7ycgtICQwgNPaNWRIp8YM6dyYtrHHfun3pucw4f1E1u5K58ERnZlwpsvBz/xsOLLX7tGXdZ+5HzLTwBx/8NcgHDRRZAY1oEV8KwIj4yDSJfFHNoVWp9k2flV1v062Q9le8g50v8jb0YZh28EAABo4SURBVKgapgm9DskrKCLxj4PMTUpl7u9pbE61J/20ja3H4E5xnNW5MREhgdzy4Uoycgp4cdzJDOt6gu3dRUWQc8gm9qM3m+h37NjOhs1b6RiZTdvwbCRz/7F7/hJoL77Q8Rx7i+tcd/r9FxVBXgbkHrG3vAzIPfzX89wjkFvitaPlD8P+zRCfAOO/rTvrTB2lCb0O23kwi7m/p/JzUipLthwgt8COytAiJpzJ4xPo0sxz7dyv/pzMs7M2ce+wjtw+tINt8sncD4d2wJafYNOPsHeNLRzTCjqcAx1HQJuBEBzmsbi8xhh7weYf/1V6s1ZJQWHO+QtREBL513kMEY1g0D+gQWvPx6x8jiZ0BdgThpZs3c/vezO4NCGe2EjPtm8bY7jnk9V8+dsuXruiD+f2bHZ8ofRdkDzL3rbOg/wsCI6AtoP+2nuvDWc/Zh6Ab++ApBnQeiB0GuGSrKP+euyawIP0zFt1PE3oymty8gv52+RlrN+dzic39qdnfEzZhfNzYPtC2PQDJP9o9+QBmvb4a++9RR//G0kweQ58fQtkHbQXneh/W4U9hpQqiyZ05VX7M3K58LVF5BUU8fVtA2gWXX5/fcA2T6Ql2WaZTT/CzmX2AGxEI2h/lr0EWkCg7X1TfAsMOvZ5QJBLmeC/ngeFQdszIdTDp53lZcGcR+3l4OK6wMVv2R8npapBE7ryut/3HuHiNxbTulEEn97Un4iQKp7Tlv0nbHba3bcvsMmyKN9eeb6owA75UBXhDeDUG+G0Gz1zZZ/dq+CLG2D/JjvGytBHa+dxAVXjNKErnzA3KZXr3v2Vs7s04Y0r+xIY4MYeGkVFdg++OMEXFdj+2sWPC/P/ep6xD5ZNtCMUBtezF1Puf6s9g7bacRTCohdh7pO2i+aFb0D7IdVfrlIOTejKZ7y9cBv/nbGBkxpHcs+wjozo1pQAdyb2qkjdCAtfhLWfggRAr8tgwF0Q2+HElvfnH/DljbBjCXS9EM57wTN7/6pO04SufIYxhh/X7+XZWZvYnJpB9xb1uW94JwZ1jPPeKI1//gFLXoWV79mulV1Gwxn3QPOTKze/MbD6I5h5v31+7rPQ8zLtI648QhO68jmFRYavftvFC3M2kfJnNqe0acD953Tm1LZe3KPNSINlb8DyybafeLshNrG3OaPs5Jx1EGbcDRu+glanw5iJ2j9ceZQmdOWz8gqK+DhxJ6/8lEzqkVzO7BjH/cM70SM+2ntB5aRD4hRY8jpkpkKLvjDwHug06tjuhlvmwlc32zNkh/wLBtzpf10qld/RhK58XnZeIe8v3c7r87ZwKCufEd2acu/wjnRoEuW9oPJzYNWHsOglOPQHxHaCgXfZJpm5T9qLS8R2hIvegua9vRenqlM0oSu/cSQnn7cXbmPygm1k5hUwpncL7jq7I60aRXgvqMIC26Sy8AXYt872aS/Kh1MnwNmPQ4gXY1N1jiZ05Xf+zMxj4i9bmLp4O4VFhstOacntZ3WgabQX+3IbY4coWP8VdL8YOpztvVhUnaUJXfmtfYdzePXnzUxfvoPAAOHq/q25efBJNKyn45youkkTuvJ7Ow9m8eKcZL78LYXw4EDuPLsD1w5oS3Cgjomi6pbyErp+G5RfaNkwgufG9mLW3WfSr10jnpyZxOhXFpK4/aC3Q1PKZ2hCV37lpMZRTB6fwJtX9eVwdj6XTFzCA5+t4c/MvIpnVqqW04Su/I6IcE63psy+ZxA3ntmOz1amcNZz8/gkcSdFRd5pQlTKF2hCV36rXmgQD43qwnd3DKR9XCT/+GwNl01awu97j3g7NKW8QhO68nudm9bnkxv787+Le7I5NYNzX17AUzM3kpVX4O3QlKpRmtBVrRAQIIw9pSU/3TuYi/q04M35Wxn2/Hxmrd/r7dCUqjEVJnQRmSIiqSKyrozp94vIKue2TkQKRUTHDFVe0bBeCP+7pBef3tSfyNAgJry/guvf/ZWUP7O8HZpSHldhP3QRORPIAN4zxnSvoOxo4G5jzFkVVaz90JWn5RcWMWXhNl6ck4zBcOfQjlw3sC0hQfrHVPmvavVDN8bMByrb2fdyYHoVYlPKY4IDA7hxUHvm3DuIMzvE8cwPSZz78gKWbT3g7dCU8gi37aqISAQwAvi8nDITRCRRRBLT0tLcVbVS5WoRE86kqxOYfHUCWXmFXDZpKU98t4H8wipeh1QpH+fO/56jgUXGmDL35o0xk4wxCcaYhLi4ODdWrVTFzu7ahNn3nMlV/Vrz1oJtXPHWUvYdzvF2WEq5jTsT+ji0uUX5uIiQIP57YXdeGteb9bsPc+7LC1i8eb+3w1LKLdyS0EUkGhgEfO2O5SnlaRf0bsE3tw0gJiKEK99exmtzN+tZpsrvVabb4nRgCdBJRFJE5DoRuUlEbnIpNgaYZYzJ9FSgSrnbSY2j+PrWAYzu1Zz/+/F3rnv3Vw5l6Zgwyn/p8LmqzjPG8MGyHfz32w3ERYXy+t/60KtljLfDUqpUOnyuUuUQEa7q15pPb+oPwCUTF/P+ku14a2dHqROlCV0pR6+WMXx3x0DO6BDHw1+v586PVpGZq+PBKP+hCV0pFzERIUy+OoH7z+nEjDW7ueC1RSTv09EblX/QhK5UCQEBwq1DTuKD607jUFYe57+6iK9X7fJ2WEpVSBO6UmU4/aRYvrvjDLq3qM+dH63i4a/WkVtQ6O2wlCqTJnSlytGkfhjTbujHjWe24/2lf3DpxCXsPKgjNyrfpAldqQoEBwbw0KguvHlVX7alZXLeKwtZvEXPLlW+RxO6UpV0TremzLhjIE3qh3LNlF+ZuXaPt0NS6hia0JWqgtaN6vHpjafTMz6aW6et5P2lf3g7JKWO0oSuVBVFRwTz/nWnMbRzYx7+ah0vzN6kJyEpn6AJXakTEB4SyMQr+3Jp33he+imZf3+1jkId3Et5WZC3A1DKXwUFBvC/S3oSGxXKG/O2cCAjjxfH9SYsONDboak6SvfQlaoGEeGBEZ15+Lyu/LB+L9e8s5zDOfneDkvVUZrQlXKD6wa25aVxvUnc/ieXvbmUVL0SkvICTehKuckFvVvw9jWn8MeBTC6euJjt+/XyAKpmaUJXyo0GdYxj2g39yMgp4OI3FrM2Jd3bIak6RBO6Um7Wu2UMn918OmHBgYybtIRFes1SVUM0oSvlAe3jIvn85tOJbxDBte/8yow1u70dkqoDNKEr5SFNo8P45Mb+9GoZze3Tf+O9Jdu9HZKq5TShK+VBf51V2oRHvl7Pc7N+17NKlcdoQlfKw8KCA5l4ZR/GJsTzys+b+eeXaykoLPJ2WKoW0jNFlaoBQYEBPHNxT+KiQnlt7hZy84t49tJeBASIt0NTtYgmdKVqiIhw/zmdCQsK5LnZmwgNDuDJMT0Q0aSu3EMTulI17PahHcgpKOS1uVsIDQrk0dFdNakrt9CErpQX3De8Ezn5Rby9cBuhwQE8OKKzJnVVbZrQlfICEeHf53Yht6CQN3/ZSlhQIHcP6+jtsJSf04SulJeICP85vzu5+UW89FMyYcGB3Dy4vbfDUn6swoQuIlOA84BUY0z3MsoMBl4EgoH9xphB7gxSqdoqIEB4+uKe5BYU8cwPSYQFB3DtgLbeDkv5qcrsoU8FXgXeK22iiMQArwMjjDE7RKSx+8JTqvYLDBCeG9uL3IJCHv92A6FBgVxxWitvh6X8UIUnFhlj5gMHyylyBfCFMWaHUz7VTbEpVWcEBwbw8uUnM7hTHP/6ai1frEzxdkjKD7njTNGOQAMRmSciK0Tk6rIKisgEEUkUkcS0tDQ3VK1U7REaZK9T2r9dI+77dLUO6KWqzB0JPQjoC5wLnAM8LCKlHq43xkwyxiQYYxLi4uLcULVStUtYcCCTxyfQt3UD7vpoFbPW7/V2SMqPuCOhpwA/GGMyjTH7gflALzcsV6k6KSIkiCnXnEK3FtHcNu03ftmk/2ZV5bgjoX8NnCEiQSISAZwGbHTDcpWqs6LCgnnv2lM5qXEkE95LZMmWA94OSfmBChO6iEwHlgCdRCRFRK4TkZtE5CYAY8xG4AdgDbAcmGyMWefJoJWqC+zQu6fSqmEE1737Kyv+KK9vglIg3hqbOSEhwSQmJnqlbqX8SeqRHC57cyn7j+Ty4Q2n0TM+xtshKS8SkRXGmITSpul46Er5uMZRYXx4/WlERwRz1dvL2bjnsLdDUj5KE7pSfqB5TDjTb+hHREggV05exhcrUziSk+/tsJSP0bFclPITLRtG8OH1p/H3qb9yzyerCQkKYEinOEb3as5ZnRsTEaJf57pOtwCl/Ei7uEh+vncwK3f8yYw1e/hu7R5+XL+P8OBAzu7ahPN6NmNQxzjCggO9HaryAj0oqpQfKywyLNt2gBlr9vDDur0czMwjKjSIYd2aMLpncwacFEtIkLas1iblHRTVhK5ULZFfWMSSLQf4dvVufly/l8M5BcREBDOiW1PO69mcfu0aEhSoyd3faUJXqo7JLShkwab9zFizm9kb9pGZV0hsZAgjuzdjdK/mnNKmgV4hyU9pQleqDsvJL2RuUioz1uzhp6R95OQXcd/wjtx2Vgdvh6ZOQHkJXQ+KKlXLhQUHMrJHM0b2aEZmbgH/+GwNL85J5qzOTejavL63w1NupA1qStUh9UKD+H8XdicmIoT7Pl1NfmGRt0NSbqQJXak6pkG9EJ4c050New7z2tzN3g5HuZEmdKXqoOHdmnJB7+a8+vNm1u9O93Y4yk00oStVRz02upvT9LKGvAJteqkNNKErVUcVN71s3HOY1+dp00ttoAldqTpseLemXKhNL7WGJnSl6rjHzu9Gg3ra9FIbaEJXqo6LiQjhyTE92Ki9XvyeJnSlFMO6NmHMyS14ba42vfgzTehKKQAeHd2VBvVCuPeT1dr04qc0oSulANv08tSYHiTtPcKr2vTilzShK6WOOrtrEy46uQWvz93Mul3a9OJvNKErpY7x6OhuNKxnx3rRphf/ogldKXWM6Ihgnixuevk52dvhqCrQhK6UOk5x08tr87Zo04sf0YSulCrVo6O70UibXvyKJnSlVKmiI4J56iJtevEnFSZ0EZkiIqkisq6M6YNFJF1EVjm3R9wfplLKG4Z2acJFfbTpxV9UZg99KjCigjILjDG9ndt/qh+WUspXPHpeN2IjtenFH1SY0I0x84GDNRCLUsoHuTa9vFKNppeCwiJ2HMhi+baDZOcVujFCVcxdF4nuLyKrgd3AfcaY9aUVEpEJwASAVq1aualqpZSnndW5CRf3ief1eVsY3rUpPeKjSy2Xnp3PzoNZ7HBufxzIOvp816FsCosMAK0bRfDcpb1IaNOwJt9GrSfGmIoLibQBZhhjupcyrT5QZIzJEJFRwEvGmA4VLTMhIcEkJiZWPWKllFekZ+cz/IVfiAkP4ZHRXdl5MIs/nGRdnLQPZeUfM0/DeiG0bBhBq4YRtGoYTuuG9QgNDuDZWb+T8mc21w9sy73DOxEWHOild+V/RGSFMSah1GnVTeillN0OJBhj9pdXThO6Uv7n56R9/H3qX9/b4EAhvkGEk7TDncQdcTSJR4UFl7qcjNwCnpy5kWnLdnBS40ieu7QXvVrG1NTb8GseTegi0hTYZ4wxInIq8BnQ2lSwYE3oSvmnxVvsvlqrhhE0iw4nMEBOeFnzN6XxwOdrSD2Sy82D2nPH0A6EBGlv6vJUK6GLyHRgMBAL7AMeBYIBjDETReQ24GagAMgG7jHGLK4oKE3oSimwTTn/+XYDn69MoXPTKJ4f25uuzet7OyyfVe09dE/QhK6UcjV7wz4e+mIt6dl53HFWB24e3J6gQN1bL6m8hK5rSynlE4Z1bcLsu89kRPdmPDd7Exe9sZjkfUe8HZZf0YSulPIZDeqF8MrlJ/PaFX3YeTCLc19ZyKT5W452d1Tl04SulPI55/Zsxqy7BzG4YxxPzkzisjeXsH1/prfD8nma0JVSPikuKpQ3r+rL82N78fu+I4x8aQHvLt5Oke6tl8ldZ4oqpZTbiQgX9Ynn9PaxPPD5Gh79Zj0/rt/LsK5NCAwQexMhIEAIcp4HiH1c/FqAU8b1tQb1QmgeHU54SO06oUkTulLK5zWNDmPqtafw0a87eeK7jSzecsAty42JCKZZdDjNo8NoFhNmHxffR4fTJDqU0CD/Sfqa0JVSfkFEuPzUVlzUpwVZuYUUFBmKjKGwyOVW8rnzWlGRseWLDPlFhgMZuexJz2H3oWz2pOew61A2iX/8SXp2/nH1xkaG0Cw6nGbRYTSPsQn//F4taBod5oW1UD5N6EopvxIaFOixveasvAL2pOew51AOu9Oz2XMohz3p2exOz2Hb/kwWbzlARm4Bk+ZvY/L4BHr72HAFmtCVUsoRERJE+7hI2sdFlllm074jXP9uIpe9uYTnx/bm3J7NajDC8mkvF6WUqoKOTaL46tYB9IyP5tZpK3n152S8dcZ9SZrQlVKqihrWC+GD609jzMkteHbWJu79dDW5Bd6/aIc2uSil1AkIDQrk+bG9aBdbj+dmbyLlYDYTr+pLw3ohXotJ99CVUuoEiQi3D+3AK5efzKqUQ4x5fRGbUzO8Fo8mdKWUqqbRvZrz0YR+ZOYWcNHri1i0udzr+3iMJnSllHKDPq0a8OUtA2gWHc74KcuZvnxHjcegCV0ppdykZcMIPru5PwNOiuWhL9byxHcbanSkSE3oSinlRlFhwbw9PoHx/Vvz1oJt3Pj+CjJzC2qkbk3oSinlZkGBATx+QXceP78bPyft49KJS9iTnu3xejWhK6WUh4w/vQ1vjz+FHQezuPC1RaxNSfdofZrQlVLKg4Z0bsxnN/cnKCCAsW8u4Yd1ez1WlyZ0pZTysM5N6/PlrafTqWkUN3+4gikLt3mkHk3oSilVAxpHhfHRhH6c36s57eLqeaQOPfVfKaVqSFhwIC+NO9ljy9c9dKWUqiU0oSulVC2hCV0ppWoJTehKKVVLVJjQRWSKiKSKyLoKyp0iIoUicon7wlNKKVVZldlDnwqMKK+AiAQCzwA/uiEmpZRSJ6DChG6MmQ8crKDY7cDnQKo7glJKKVV11W5DF5EWwBhgYiXKThCRRBFJTEtLq27VSimlXLjjxKIXgQeMMYUiUm5BY8wkYBKAiKSJyB8nWGcs4J1LglSOr8cHvh+jxlc9Gl/1+HJ8rcua4I6EngB85CTzWGCUiBQYY74qbyZjTNyJVigiicaYhBOd39N8PT7w/Rg1vurR+KrH1+MrS7UTujGmbfFjEZkKzKgomSullHK/ChO6iEwHBgOxIpICPAoEAxhjKmw3V0opVTMqTOjGmMsruzBjzDXViqbyJtVQPSfK1+MD349R46seja96fD2+UokxNXcBU6WUUp6jp/4rpVQtoQldKaVqCZ9O6CIyQkR+F5HNIvJgKdNDReRjZ/oyEWlTg7G1FJG5IrJRRNaLyJ2llBksIukissq5PVJT8Tn1bxeRtU7diaVMFxF52Vl/a0SkTw3G1sllvawSkcMicleJMjW+/kobu0hEGorIbBFJdu4blDHveKdMsoiMr8H4/k9EkpzP8EsRiSlj3nK3Bw/G95iI7HL5HEeVMW+533cPxvexS2zbRWRVGfN6fP1VmzHGJ29AILAFaAeEAKuBriXK3AJMdB6PAz6uwfiaAX2cx1HAplLiG4ztxumtdbgdiC1n+ijge0CAfsAyL37We4HW3l5/wJlAH2Cdy2v/Ax50Hj8IPFPKfA2Brc59A+dxgxqKbzgQ5Dx+prT4KrM9eDC+x4D7KrENlPt991R8JaY/BzzirfVX3Zsv76GfCmw2xmw1xuQBHwEXlChzAfCu8/gzYKhUdLqqmxhj9hhjVjqPjwAbgRY1UbcbXQC8Z6ylQIyINPNCHEOBLcaYEz1z2G1M6WMXuW5n7wIXljLrOcBsY8xBY8yfwGwqGNTOXfEZY2YZYwqcp0uBeHfXW1llrL/KqMz3vdrKi8/JHWOB6e6ut6b4ckJvAex0eZ7C8QnzaBlng04HGtVIdC6cpp6TgWWlTO4vIqtF5HsR6VajgYEBZonIChGZUMr0yqzjmjCOsr9E3lx/xZoYY/aA/SEHGpdSxlfW5d+x/7pKU9H24Em3OU1CU8posvKF9XcGsM8Yk1zGdG+uv0rx5YRe2p52yT6WlSnjUSISiR1p8i5jzOESk1dimxF6Aa8ANX0G7QBjTB9gJHCriJxZYrovrL8Q4Hzg01Ime3v9VYUvrMt/AQXAh2UUqWh78JQ3gPZAb2APtlmjJK+vP+Byyt8799b6qzRfTugpQEuX5/HA7rLKiEgQEM2J/d07ISISjE3mHxpjvig53Rhz2BiT4TyeCQSLSGxNxWeM2e3cpwJfYv/WuqrMOva0kcBKY8y+khO8vf5c7CtuinLuSxsm2qvr0jkIex7wN+M0+JZUie3BI4wx+4wxhcaYIuCtMur19voLAi4CPi6rjLfWX1X4ckL/FeggIm2dvbhxwDclynwDFPcmuAT4uayN2d2c9ra3gY3GmOfLKNO0uE1fRE7Fru8DNRRfPRGJKn6MPXBW8qpT3wBXO71d+gHpxU0LNajMvSJvrr8SXLez8cDXpZT5ERguIg2cJoXh1NAFX0RkBPAAcL4xJquMMpXZHjwVn+txmTFl1FuZ77snnQ0kGWNSSpvozfVXJd4+KlveDdsLYxP26Pe/nNf+g91wAcKwf9U3A8uBdjUY20DsX8I1wCrnNgq4CbjJKXMbsB57xH4pcHoNxtfOqXe1E0Px+nONT4DXnPW7Fkio4c83Apugo11e8+r6w/647AHysXuN12GPy/wEJDv3DZ2yCcBkl3n/7myLm4FrazC+zdj25+LtsLjnV3NgZnnbQw3F976zfa3BJulmJeNznh/3fa+J+JzXpxZvdy5la3z9Vfemp/4rpVQt4ctNLkoppapAE7pSStUSmtCVUqqW0ISulFK1hCZ0pZSqJTShK6VULaEJXSmlaon/D5O3Ta5aMH+XAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# evaluate the model\n",
    "_, train_acc = model.evaluate(X_train, y_train, verbose=0)\n",
    "_, test_acc = model.evaluate(X_test, y_test, verbose=0)\n",
    "print('Train: %.3f, Test: %.3f' % (train_acc, test_acc))\n",
    "# plot training history\n",
    "plt.plot(history.history['loss'], label='train')\n",
    "plt.plot(history.history['val_loss'], label='test')\n",
    "plt.title('Training and test loss at each epoch', fontsize=14)\n",
    "plt.legend()\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The Neural Network model with simple LSTM layer performed not much better than previous SGD or RF Classification models. We will perform some hyperparameter tuning in the following section."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Tuning the Network Model\n",
    "\n",
    "- Use Word2Vec/GloVe embedding:https://blog.keras.io/using-pre-trained-word-embeddings-in-a-keras-model.html\n",
    "\n",
    "- Optimizers\n",
    "- Learning Rate\n",
    "- Batch Size\n",
    "- Regularization: prevent overfitting\n",
    "- Dropout\n",
    "- Early Stopping"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Hyperparameter tuning on a regular embedding layer "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X_t, y, test_size=0.1, random_state=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Add Convolutional Layer and MaxPooling Layer\n",
    "model = Sequential()\n",
    "\n",
    "embedding_size = 128  #each word onto a 128 length real valued vector\n",
    "model.add(Embedding(input_dim = 1000, output_dim = embedding_size, input_length = 400)) #try 100, 200, single LSTM and single dense 100\n",
    "model.add(Conv1D(filters=32, kernel_size = 3, padding='same', activation='relu'))\n",
    "model.add(MaxPooling1D(pool_size=2))\n",
    "model.add(LSTM(64))\n",
    "#model.add(LSTM(64))\n",
    "#model.add(GlobalMaxPool1D())\n",
    "#model.add(Dropout(0.5))\n",
    "model.add(Dense(32, activation='relu'))\n",
    "model.add(Dropout(0.5))\n",
    "#model.add(Dense(32, activation='relu'))\n",
    "#model.add(Flatten())   #add Flatten layer \n",
    "model.add(Dense(6, activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_18\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_7 (Embedding)      (None, 400, 128)          128000    \n",
      "_________________________________________________________________\n",
      "conv1d_4 (Conv1D)            (None, 400, 32)           12320     \n",
      "_________________________________________________________________\n",
      "max_pooling1d_4 (MaxPooling1 (None, 200, 32)           0         \n",
      "_________________________________________________________________\n",
      "lstm_10 (LSTM)               (None, 64)                24832     \n",
      "_________________________________________________________________\n",
      "dense_38 (Dense)             (None, 32)                2080      \n",
      "_________________________________________________________________\n",
      "dropout_8 (Dropout)          (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dense_39 (Dense)             (None, 6)                 198       \n",
      "=================================================================\n",
      "Total params: 167,430\n",
      "Trainable params: 167,430\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 866 samples, validate on 97 samples\n",
      "Epoch 1/20\n",
      "866/866 [==============================] - 4s 5ms/step - loss: 1.7882 - accuracy: 0.1975 - val_loss: 1.7881 - val_accuracy: 0.2062\n",
      "Epoch 2/20\n",
      "866/866 [==============================] - 3s 4ms/step - loss: 1.7819 - accuracy: 0.1894 - val_loss: 1.7747 - val_accuracy: 0.2062\n",
      "Epoch 3/20\n",
      "866/866 [==============================] - 4s 4ms/step - loss: 1.7720 - accuracy: 0.2021 - val_loss: 1.7616 - val_accuracy: 0.2165\n",
      "Epoch 4/20\n",
      "866/866 [==============================] - 3s 4ms/step - loss: 1.7441 - accuracy: 0.2136 - val_loss: 1.7527 - val_accuracy: 0.1959\n",
      "Epoch 5/20\n",
      "866/866 [==============================] - 3s 4ms/step - loss: 1.7198 - accuracy: 0.2483 - val_loss: 1.7462 - val_accuracy: 0.2371\n",
      "Epoch 6/20\n",
      "866/866 [==============================] - 4s 4ms/step - loss: 1.6851 - accuracy: 0.2714 - val_loss: 1.8915 - val_accuracy: 0.2371\n",
      "Epoch 7/20\n",
      "866/866 [==============================] - 4s 4ms/step - loss: 1.6586 - accuracy: 0.2714 - val_loss: 1.8232 - val_accuracy: 0.2577\n",
      "Epoch 8/20\n",
      "866/866 [==============================] - 4s 4ms/step - loss: 1.6229 - accuracy: 0.2887 - val_loss: 1.7345 - val_accuracy: 0.2887\n",
      "Epoch 9/20\n",
      "866/866 [==============================] - 4s 4ms/step - loss: 1.5677 - accuracy: 0.3245 - val_loss: 1.7882 - val_accuracy: 0.2784\n",
      "Epoch 10/20\n",
      "866/866 [==============================] - 4s 5ms/step - loss: 1.4841 - accuracy: 0.3441 - val_loss: 1.8641 - val_accuracy: 0.2990\n",
      "Epoch 11/20\n",
      "866/866 [==============================] - 4s 4ms/step - loss: 1.4589 - accuracy: 0.3614 - val_loss: 1.7423 - val_accuracy: 0.2887\n",
      "Epoch 12/20\n",
      "866/866 [==============================] - 4s 4ms/step - loss: 1.4443 - accuracy: 0.3661 - val_loss: 1.7797 - val_accuracy: 0.2784\n",
      "Epoch 13/20\n",
      "866/866 [==============================] - 4s 4ms/step - loss: 1.3865 - accuracy: 0.3891 - val_loss: 1.8236 - val_accuracy: 0.2887\n",
      "Epoch 14/20\n",
      "866/866 [==============================] - 4s 5ms/step - loss: 1.4357 - accuracy: 0.3753 - val_loss: 1.8015 - val_accuracy: 0.2990\n",
      "Epoch 15/20\n",
      "866/866 [==============================] - 3s 4ms/step - loss: 1.4018 - accuracy: 0.3845 - val_loss: 1.8166 - val_accuracy: 0.2784\n",
      "Epoch 16/20\n",
      "866/866 [==============================] - 3s 4ms/step - loss: 1.3633 - accuracy: 0.3811 - val_loss: 1.9341 - val_accuracy: 0.2784\n",
      "Epoch 17/20\n",
      "866/866 [==============================] - 3s 4ms/step - loss: 1.3126 - accuracy: 0.4226 - val_loss: 2.1875 - val_accuracy: 0.2577\n",
      "Epoch 18/20\n",
      "866/866 [==============================] - 3s 4ms/step - loss: 1.2627 - accuracy: 0.4492 - val_loss: 2.1013 - val_accuracy: 0.2577\n",
      "Epoch 19/20\n",
      "866/866 [==============================] - 4s 5ms/step - loss: 1.2620 - accuracy: 0.4538 - val_loss: 2.2811 - val_accuracy: 0.2577\n",
      "Epoch 20/20\n",
      "866/866 [==============================] - 4s 5ms/step - loss: 1.3267 - accuracy: 0.4238 - val_loss: 2.1342 - val_accuracy: 0.3299\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(X_train, y_train, epochs=20, batch_size=32, validation_data=[X_test, y_test])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Accuracy: 1.000, Test Accuracy: 0.515\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAEICAYAAAB25L6yAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAb8UlEQVR4nO3de3Sb1Znv8e8j24njOBdwHEhiUgdIISHQAIbCwOkpFCgBhmmHNRR60plz6Up7pjOFGUoLvTBD23PartXh0K4ZoIEy01MYWoZLKRQ6gdWkwIFCHUiH3CBAAzEB7ITmHiex/Jw/3i1LliVbdixrx/591tLS+75769Wj19JPW1uyZO6OiIjEK1XpAkREpH8KahGRyCmoRUQip6AWEYmcglpEJHIKahGRyCmoD0FmVmVmu8xsdqVrkcoxs3ozczM7stK1FGNmrWZ2RaXrONQpqEdACNXMqdvM9uas/5fB7s/d0+5e7+5vlqPegZjZyzn1p82sM2f9iwex3/vM7Lp+2qMMpljrktGjutIFjAXuXp9ZNrONwKfd/Yli/c2s2t27RqK2oXD34zLLZvY0cIe7/0vlKhIZ3TSijoCZfdPMfmpm95jZTmCxmZ1pZr8xs21m9raZfd/MakL/6jCCaw7rd4X2x8xsp5k9a2ZzilzXE2b22bxta8zsUjNLhf20m9l2M/sPM5s/xNv0l2Hk/Z6ZPWJmM8P2KjO71cw6wnWsMrNjzexvgY8BN4aR+T2DvL6qcBw3mdk7ZnaHmdWHtnozuzfU8odwXKeEts+a2RvhuL1mZn9aZP8fMrPnQ82bzewfzKwqND8Zzl8LtV8ymGMS2paa2VtmtsPMnjOz03PaaszsRjP7fWh/3swac3Z9iZm9Hvb73X6OUbWZ/X3YzxYz+7GZTQ5tC8Iro8+F4/eWmf1lzmXrwt/tHTNrM7PvZO6Pof0TZvZSOI6vmNmHc656bqh5R7jdU4rVKEW4u04jeAI2AuflbfsmsB/4Y5InzwnAacAHSV71HA28AvxV6F8NONAc1u8CtgAtQA3wU+CuItf/34Ff56x/AHgPGAdcDDwPTAl1zAeOHOD2PA3817xti4E1wLGhnm8Bj4e2y4CngMnhOk4EGkPbfcB1/VxXfbjdfWoCPh+uc3ao/zHg1tB2TTgmteHYnR6OcWO47UeHfrOA44tc9xnh+FYBc4HXSV4Z9VtXKccktP85cFho+7twP6kObTcCreF+kAJODbcxc73/BkwCjgF2AGcXqeGrwArgyHD7fwzcHtoWAN3AnaHtVGAbcEZovylctgGYAbwAfCm0nQNsBf5zqO99wNzQ1gqsA+aEep8Dvlrpx+Ghdqp4AWPtRPGg/tUAl/sC8G9huVBQ35bT91JgdZH9TAH2AE1h/TvA0rB8AbCe5AkiVeLtKRTUTwGfyFkfD6TDg/xS4CWSJyLLu9zBBPVzwJ/nrJ8K7AzLnwd+BczPu0wD8AeSJ8jxg/w7fhX48UB1lXJMCvStAjqBY8L6W8BH+jkeC3O2PUp4Qi/QfxPwwZz1uTnHaEHYV1NO+y3A98Lyu8CHctouy9zHgLuBbxS5zlbg6pz1LwL3levxNVpPmvqIx6bcFTM73sx+EV5q7gC+Dkzr5/Lv5CzvIXkQ9+Hu24FfAp8wMwOuIHmg4e7LgNuAW4F3zew2M5s0hNvyPuCOMG2zjeRBvh9oAh4GfgTcDrxjZv9oZnVDuI58M4E3ctbfAOrDy+ylwDPAg2Fq5JtmlnL3rcBfAH9DcnsfMrNjCu08TA380szeDX+PL9P/3yNff8cEM/tKmBbZTjI6HQdMC9MrM4DX+tn3gH/7sJ9ZwLKcGn4L1JjZ1NCtm+RJIeMNYKaZVQPT6Xt8Z4Xlow62Pumfgjoe+V9j+ANgNXCsu08GbgBsmK7rHuBK4GyS+0BmjhV3v9ndTyEZYc0H/nYI+98ELHb3qTmnCe7+O098190XAgtJphP+OnP1B3GbNpOEYcZsYJe7b3f3Tnf/qidvgp4DfBL4MwB3/7m7n0sSOpuBfyyy/x+STAsdHf4e/5vs36OUuoseEzNbBHwW+BNgKslIfz/JK4408DbJtMaQ5eznQ3k11Lr7ttAtRTZ8ITmGmz15Y7udvsc3E+qbDrY+6Z+COl6TgO3AbjObB3xmGPf9MMnL3huAn3h4TWpmp4dTNbCbJCzSQ9j/bcANZvb+sN/DMm/SWfIm6anhOnblXce7JPOwAxlvZrU5pxTJk8+1ZtYU3iD7BuGVgpmdb2bzQr8dQBeQNrOjzOwiM5tAMtWwu5/bOwnY5u67zexE4NOZBnffHW5Lf7UXPSZh310k7zOMA/4XyVx1xh3At8ys2ZI3fE8Z4htytwHfMbNZoYYj8t74dODvwzE9heQJ7aeh7Z7QdrglH0P8MsmUW6a+/2lmZ1titpnNHUJ9UoSCOl7XkLws30kyuv5p/91L5+6dwM+A84B/zWmaSjJy3EYyl/428H+GsP8fk4TCz8I0wSrgI6H5cOD/hut4neQl8z+FttuAP7Lkkxl393MVG4G9OafLSUbCD5PMVW8gCf1rQ/+jQttO4HfAQ8ADJHP9Xwl9t5CM8K8qcp1Xk4TRLuB7wE/y2m8gmVrZZmYXD/KY/Ax4Fvh9OCZvkcydZ3wTWAb8muS43UIS6IP1LZJXT78ONTwNnJzTvh94keT4PgLc4O7PhLavkBzXdcBKYDnwD+G2LSd5VfQDkifCx0mmomSYWBhMicgYZmYLgFZ3r610LdKXRtQiIpFTUIuIRE5THyIikdOIWkQkcmX5UqZp06Z5c3NzOXYtIjIqrVy5cou7NxZqK0tQNzc309raWo5di4iMSmb2RrE2TX2IiEROQS0iEjkFtYhI5BTUIiKRU1CLiEROQS0iEjkFtYhI5PQr5CJy6HKH7jR0d2VP3h2W0yVuy9le6rbuNHjOcmZ7zQQ4++phv5kKapHYde2H916HrRtgywbY+mqynt4PVgWpqnCeglT1ELflLfech75Ft6WSgEofyDk/AOmucJ6//UAItWJtpewjd/uBSv91eqs/QkEtMmq5w+6OJIi3vJKE8ZYNSTj/4Y1k9JZRfyQ0HAsTDus96uvaD743bySYGfkNcttB/SpahkFVDaRqoKo6OU9Vh23Vfdsy22sm9F4v1q9ne014oqnOnnqeTKqzbaVu63lSGsy2zPpw/VpebyUFtZltJPl1jDTQ5e4tZalGZLQ70Nl7dJwJ4y2vwr7t2X7VtUkYH3kSLLgMGubCtLnJttrJ5a8zM6XQK9DT0N3dN+QLhm8ITxkWgxlRn+PuW8pWicho4Q473+k9VZEZKW97k16j1UkzkwA+6c9CGB+bnE85KpmiqBSzZASrF91R0F9BZKgO7IWtr/WdqtjyKuzfme1XUwcNx8CsU+EDV2ZHxg3Hwvj6ytUvh4xSg9qBZWbmwA/cfWl+BzNbAiwBmD179vBVKFJpe96DjpehY31yvuXlJIy3b6LX6HjKUUn4LrwyO1UxbW4yaq7k6FgOeaUG9VnuvtnMpgOPm9l6d38yt0MI76UALS0t+tkYOfTs3hLCeD20r88G8+72bJ+aiUn4zv4gNCzOTlU0HAPjJlaudhnVSgpqd98cztvN7EHgdJKfnRc5tLjDrvZsIGfCuGM97Nma7Td+MjQeB++/ABqPD6fjYHKTRscy4gYMajObCKTcfWdYvgD4etkrEzkY7rDzbWhf13vaomM9dG7L9qudAo3z4PhLsmHceDxMnlm2j1qJDFYpI+ojgActudNWA//q7r8sSzW/fxKmNMHUZo1apDTuyVxxTxhnAvll2Lcj22/C4TB9Hiz405wR8vFQP12BLNEbMKjd/XXgA2WvpDsNd18OXXuTecDpxycPrOknwBHzYfr85EElY1O6Ky+Qw/mWV2D/rmy/idOTUfFJn0juQ5lAnjitcrWLHKS4Pp73Fw9D+9rk9O4aePkxePGubHvdtBDaJyQhfsQJyYNQH3E6tLhD5/ZkTnjPe+F8K+zZkrMctu8O23KnKwAmzUj+9id/Kjtd0Xgc1B1emdskUkbxBHWqCo46LTllZP6t9t01OQG+Fl74ERzYk+13WHMy4p4+PxvkDcck/x0l5Xdgb+9QLRi+udu2hn9TLqBqXPKEXNcAExtg5sJkua4hCefp82Da+2HC1JG9jSIVFE9QF2KWTHfUT4djzslu7+6GbRuT0M6MvtvXwSv/nv1OhKpxyQN6+vzs6Hv6/GQOXHOSfXWnYd/OZF53387sqXN7GP3mBe2eLdltuU+auSyVzA1ngvbwo6HptGQaIrOtriEZBWfCedxE/X1E8sQd1MWkUsmD/vCjYd4l2e0HOpM5y/Z10L4mCfI3/h+8dG+2z/jJYe57fja8p887dF8yd3cn/wW3byd05obsjt6h29OWfx7aDuwe+LrGTw6h2pB8MdD0E7LrdQ19A7h2ir7vQWQYHJpBXUxNLcw4KTnl2rstG97t65IAX/MArPznbJ9JM7JTJ43HJ1+KA8n0C558+UzPcn/bwvZel81sK9TPi+zDk1cH+3YlX9bTa5SbE7K5/6pclMH4STmnyVA7NflPutrJyXpuW2a5dkp2W10DVI8b+t9GRIZsdAV1MROmwvvOTE4Z7rBjc+/Rd/taeG4ppPdVrtZ84+rzwnNy8hnfnlDNayu0fVy9Pu4ocggbG0FdiBlMmZWc5p6X3Z7ugu1vJl9KjiXzrJk5U7OwLWzPLPe3ractf1uqSL+8fShgRca8sRvUxVRVJ3PfIiKR0HBNRCRyCmoRkcgpqEVEIqegFhGJnIJaRCRyCmoRkcgpqEVEIqegFhGJnIJaRCRyCmoRkcgpqEVEIqegFhGJnIJaRCRyCmoRkcgpqEVEIqegFhGJnIJaRCRyCmoRkcgpqEVEIqegFhGJnIJaRCRyJQe1mVWZ2Ytm9kg5CxIRkd4GM6K+ClhXrkJERKSwkoLazJqAi4E7yluOiIjkK3VEfTPwRaC7jLWIiEgBAwa1mV0CtLv7ygH6LTGzVjNr7ejoGLYCRUTGulJG1GcBl5rZRuAnwLlmdld+J3df6u4t7t7S2Ng4zGWKiIxdAwa1u1/v7k3u3gxcAfzK3ReXvTIREQH0OWoRkehVD6azu68AVpSlEhERKUgjahGRyCmoRUQip6AWEYmcglpEJHIKahGRyCmoRUQip6AWEYmcglpEJHIKahGRyCmoRUQip6AWEYmcglpEJHIKahGRyCmoRUQip6AWEYmcglpEJHIKahGRyCmoRUQip6AWEYmcglpEJHIKahGRyCmoRUQip6AWEYmcglpEJHIKahGRyCmoRUQip6AWEYmcglpEJHIKahGRyCmoRUQiN2BQm1mtmT1vZr8zszVmduNIFCYiIonqEvrsA851911mVgM8bWaPuftvylybiIhQQlC7uwO7wmpNOHk5ixIRkayS5qjNrMrMVgHtwOPu/lyBPkvMrNXMWjs6Ooa7ThGRMauUqQ/cPQ0sNLOpwINmtsDdV+f1WQosBWhpadGIW0QG5cCBA7S1tdHZ2VnpUsqqtraWpqYmampqSr5MSUGd4e7bzGwFcCGweoDuIiIla2trY9KkSTQ3N2NmlS6nLNydrVu30tbWxpw5c0q+XCmf+mgMI2nMbAJwHrB+yJWKiBTQ2dlJQ0PDqA1pADOjoaFh0K8aShlRzwB+ZGZVJMF+r7s/MoQaRUT6NZpDOmMot3HAEbW7/4e7n+zuJ7n7Anf/+pCqExGJ2LZt27jlllsGfbmLLrqIbdu2laGiLP1noogIxYM6nU73e7lHH32UqVOnlqssYJBvJoqIjFbXXXcdr732GgsXLqSmpob6+npmzJjBqlWrWLt2LR/72MfYtGkTnZ2dXHXVVSxZsgSA5uZmWltb2bVrF4sWLeLss8/mmWeeYdasWTz00ENMmDDhoGtTUItIdG58eA1rN+8Y1n3OnzmZv/vjE4q2f/vb32b16tWsWrWKFStWcPHFF7N69eqeT2fceeedHH744ezdu5fTTjuNyy67jIaGhl772LBhA/fccw+33347l19+Offffz+LFy8+6NoV1CIiBZx++um9PkL3/e9/nwcffBCATZs2sWHDhj5BPWfOHBYuXAjAqaeeysaNG4elFgW1iESnv5HvSJk4cWLP8ooVK3jiiSd49tlnqaur48Mf/nDBj9iNHz++Z7mqqoq9e/cOSy16M1FEBJg0aRI7d+4s2LZ9+3YOO+ww6urqWL9+Pb/5zch+J51G1CIiQENDA2eddRYLFixgwoQJHHHEET1tF154IbfddhsnnXQSxx13HGecccaI1mbJl+MNr5aWFm9tbR32/YrI6LVu3TrmzZtX6TJGRKHbamYr3b2lUH9NfYiIRE5BLSISOQW1iEjkFNQiIpFTUIuIRE5BLSISOQW1iAhD/5pTgJtvvpk9e/YMc0VZCmoREeIOav1noogIvb/m9Pzzz2f69Once++97Nu3j49//OPceOON7N69m8svv5y2tjbS6TRf+9rXePfdd9m8eTPnnHMO06ZNY/ny5cNem4JaROLz2HXwzkvDu88jT4RF3y7anPs1p8uWLeO+++7j+eefx9259NJLefLJJ+no6GDmzJn84he/AJLvAJkyZQo33XQTy5cvZ9q0acNbc6CpDxGRPMuWLWPZsmWcfPLJnHLKKaxfv54NGzZw4okn8sQTT/ClL32Jp556iilTpoxIPRpRi0h8+hn5jgR35/rrr+czn/lMn7aVK1fy6KOPcv3113PBBRdwww03lL0ejahFROj9Nacf/ehHufPOO9m1axcAb731Fu3t7WzevJm6ujoWL17MF77wBV544YU+ly0HjahFROj9NaeLFi3ik5/8JGeeeSYA9fX13HXXXbz66qtce+21pFIpampquPXWWwFYsmQJixYtYsaMGWV5M1FfcyoiUdDXnOprTkVEDlkKahGRyCmoRUQip6AWkWiU4z2z2AzlNiqoRSQKtbW1bN26dVSHtbuzdetWamtrB3U5fTxPRKLQ1NREW1sbHR0dlS6lrGpra2lqahrUZRTUIhKFmpoa5syZU+kyojTg1IeZHWVmy81snZmtMbOrRqIwERFJlDKi7gKucfcXzGwSsNLMHnf3tWWuTUREKGFE7e5vu/sLYXknsA6YVe7CREQkMahPfZhZM3Ay8FyBtiVm1mpmraP9zQARkZFUclCbWT1wP3C1u+/Ib3f3pe7e4u4tjY2Nw1mjiMiYVlJQm1kNSUjf7e4PlLckERHJVcqnPgz4IbDO3W8qf0kiIpKrlBH1WcCngHPNbFU4XVTmukREJBjw43nu/jRgI1CLiIgUoO/6EBGJnIJaRCRyCmoRkcgpqEVEIqegFhGJnIJaRCRyCmoRkcgpqEVEIqegFhGJnIJaRCRyCmoRkcgpqEVEIqegFhGJnIJaRCRyCmoRkcgpqEVEIqegFhGJnIJaRCRyCmoRkcgpqEVEIqegFhGJnIJaRCRyCmoRkcgpqEVEIqegFhGJnIJaRCRyCmoRkcgpqEVEIqegFhGJ3IBBbWZ3mlm7ma0eiYJERKS3UkbU/wJcWOY6RESkiAGD2t2fBN4bgVpERKSAYZujNrMlZtZqZq0dHR3DtVsRkTFv2ILa3Ze6e4u7tzQ2Ng7XbkVExjx96kNEJHIKahGRyJXy8bx7gGeB48yszcz+R/nLEhGRjOqBOrj7lSNRiIiIFKapDxGRyCmoRUQip6AWEYmcglpEJHIKahGRyCmoRUQip6AWEYmcglpEJHIKahGRyCmoRUQip6AWEYmcglpEJHIKahGRyCmoRUQip6AWEYmcglpEJHIKahGRyCmoRUQip6AWEYmcglpEJHIKahGRyCmoRUQip6AWEYmcglpEJHIKahGRyCmoRUQip6AWEYmcglpEJHIKahGRyCmoRUQiV1JQm9mFZvaymb1qZteVq5g9+7vY15Um3e24e7muRkTkkFI9UAczqwL+CTgfaAN+a2Y/d/e1w13Mqd94gr0H0j3rNVVGVcqoSaWoqjKqU0Z1KpVsC23VqRTVmbaqVNiWLFf3LBtVqRQ1qXCZ0JbdTypnf2E/ZphBVcpImZFKGVVmpIzscgpSZtk+Pcs5fUK/qrCPXn1yLlsV9tWzLfQxLPwdCEv0LOS2ZTZbWMn0tby+WO/+5F0mv7/1XGlvudsNK9qWez3ZduunrXhfkbFqwKAGTgdedffXAczsJ8CfAMMe1Ndc8H72dXXTlXbS3d10dXtySjtdYT2ddg50d5PO2Z7udg6kPZx3s7+rmz3708llwvZkX93hMtm+ufvp1iD+kNHfk03v9t4dc2O/2JNSsX3m76OnvcRawAr0La2GXm35ty0yAz1RJ32s/z5D2Eeu/FfkBR/a3u9qwVf1ffv0Xj9s4jge+txZResaqlKCehawKWe9DfhgficzWwIsAZg9e/aQivn0fzp6SJcbLt05gd7tkO52urudbnfS7nR3kyyHbZk+HtqT5WRb2sP27hL79Czn9An3AndwssuQc4fJ9Mlty9mWf5k+d+CeNu+z79y2QpcppNj+e9b7bSvxskVvX+FjVPB29OlT2mX71DSIWrKXzWsroX7PKzjWMUUpAdnffaLUffQNWe/7xDUMTxiFngv6e4U4qbaUSB28UvZa6Pb1OXbuvhRYCtDS0hLr/ahfqZQxLmWM03usIhKRUhKpDTgqZ70J2FyeckREJF8pQf1bYK6ZzTGzccAVwM/LW5aIiGQMOPXh7l1m9lfAvwNVwJ3uvqbslYmICFDaHDXu/ijwaJlrERGRAvSumYhI5BTUIiKRU1CLiEROQS0iEjkrx5cfmVkH8MYQLz4N2DKM5RzKdCx60/HoTccjazQci/e5e2OhhrIE9cEws1Z3b6l0HTHQsehNx6M3HY+s0X4sNPUhIhI5BbWISORiDOqllS4gIjoWvel49KbjkTWqj0V0c9QiItJbjCNqERHJoaAWEYlcNEE9Uj+geygws6PMbLmZrTOzNWZ2VaVrqjQzqzKzF83skUrXUmlmNtXM7jOz9eE+cmala6okM/ub8DhZbWb3mFltpWsablEEdc4P6C4C5gNXmtn8ylZVUV3ANe4+DzgD+NwYPx4AVwHrKl1EJL4H/NLdjwc+wBg+LmY2C/g80OLuC0i+ivmKylY1/KIIanJ+QNfd9wOZH9Adk9z9bXd/ISzvJHkgzqpsVZVjZk3AxcAdla6l0sxsMvAh4IcA7r7f3bdVtqqKqwYmmFk1UMco/AWqWIK60A/ojtlgymVmzcDJwHOVraSibga+CHRXupAIHA10AP8cpoLuMLOJlS6qUtz9LeC7wJvA28B2d19W2aqGXyxBXdIP6I41ZlYP3A9c7e47Kl1PJZjZJUC7u6+sdC2RqAZOAW5195OB3cCYfU/HzA4jefU9B5gJTDSzxZWtavjFEtT6Ad08ZlZDEtJ3u/sDla6ngs4CLjWzjSRTYuea2V2VLami2oA2d8+8wrqPJLjHqvOA37t7h7sfAB4A/qjCNQ27WIJaP6Cbw8yMZA5ynbvfVOl6Ksndr3f3JndvJrlf/MrdR92IqVTu/g6wycyOC5s+AqytYEmV9iZwhpnVhcfNRxiFb66W9JuJ5aYf0O3jLOBTwEtmtips+3L47UqRvwbuDoOa14H/VuF6KsbdnzOz+4AXSD4t9SKj8N/J9S/kIiKRi2XqQ0REilBQi4hETkEtIhI5BbWISOQU1CIikVNQi4hETkEtIhK5/w9UVbnv6u5gvwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# evaluate the model\n",
    "_, train_acc = model.evaluate(X_train, y_train, verbose=0)\n",
    "_, test_acc = model.evaluate(X_test, y_test, verbose=0)\n",
    "print('Train Accuracy: %.3f, Test Accuracy: %.3f' % (train_acc, test_acc))\n",
    "\n",
    "plt.plot(history.history['loss'], label='train')\n",
    "plt.plot(history.history['val_loss'], label='test')\n",
    "plt.title(\"Train vs Test Loss at each epoch\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Try fewer input dimension and input words, add Early Stop layer. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Add Early Stopping layer, L1 and L2 regularization\n",
    "from keras.callbacks import EarlyStopping\n",
    "from keras import regularizers\n",
    "\n",
    "\n",
    "model = Sequential()\n",
    "embedding_size = 64  \n",
    "model.add(Embedding(input_dim = 1000, output_dim = embedding_size, input_length = 200)) #try 100, 200, single LSTM and single dense 100\n",
    "#Take out conv1d and maxpooling\n",
    "model.add(Conv1D(filters=32, kernel_size = 3, padding='same', activation='relu'))\n",
    "model.add(MaxPooling1D(pool_size=2))\n",
    "model.add(LSTM(128, return_sequences=True)) #the last LSTM doesn't need  to return sequence\n",
    "#add LSTM\n",
    "\n",
    "#add dropout here\n",
    "model.add(Dense(100, activation='relu'))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Flatten())   #add Flatten layer \n",
    "model.add(Dense(6, activation='softmax'))\n",
    "\n",
    "model.compile(loss='categorical_crossentropy', optimizer='Adam', metrics=['accuracy'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 907 samples, validate on 101 samples\n",
      "Epoch 1/30\n",
      "907/907 [==============================] - 8s 9ms/step - loss: 1.9354 - accuracy: 0.2194 - val_loss: 1.9799 - val_accuracy: 0.1386\n",
      "Epoch 2/30\n",
      "907/907 [==============================] - 7s 7ms/step - loss: 1.8568 - accuracy: 0.2404 - val_loss: 1.9903 - val_accuracy: 0.1782\n",
      "Epoch 3/30\n",
      "907/907 [==============================] - 7s 7ms/step - loss: 1.6685 - accuracy: 0.3197 - val_loss: 2.2110 - val_accuracy: 0.3168\n",
      "Epoch 4/30\n",
      "907/907 [==============================] - 7s 7ms/step - loss: 1.4865 - accuracy: 0.3936 - val_loss: 1.7135 - val_accuracy: 0.2871\n",
      "Epoch 5/30\n",
      "907/907 [==============================] - 7s 7ms/step - loss: 1.2415 - accuracy: 0.5281 - val_loss: 1.6493 - val_accuracy: 0.2970\n",
      "Epoch 6/30\n",
      "907/907 [==============================] - 7s 8ms/step - loss: 1.0580 - accuracy: 0.5910 - val_loss: 1.7719 - val_accuracy: 0.3168\n",
      "Epoch 7/30\n",
      "907/907 [==============================] - 7s 7ms/step - loss: 0.8364 - accuracy: 0.6858 - val_loss: 2.0349 - val_accuracy: 0.3762\n",
      "Epoch 8/30\n",
      "907/907 [==============================] - 7s 7ms/step - loss: 0.6496 - accuracy: 0.7685 - val_loss: 2.0146 - val_accuracy: 0.4356\n",
      "Epoch 9/30\n",
      "907/907 [==============================] - 7s 7ms/step - loss: 0.4700 - accuracy: 0.8390 - val_loss: 2.2638 - val_accuracy: 0.4455\n",
      "Epoch 10/30\n",
      "907/907 [==============================] - 7s 7ms/step - loss: 0.3477 - accuracy: 0.8743 - val_loss: 2.7191 - val_accuracy: 0.4554\n",
      "Epoch 00010: early stopping\n",
      "Train Accuracy: 0.931, Test Accuracy: 0.455\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEICAYAAABPgw/pAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd3wUdf7H8dcnPSEkQBJCSIAUepMuLXQVLHj2hh1RT+/snt5Zzqv6s+t5Is2K2M+KgiAQmmBAlColoSSBJARIgfR8f3/MIAGTEEiZ3c3n+XjsI7s7szOfnSTvnf3Od74jxhiUUkq5Py+nC1BKKVU/NNCVUspDaKArpZSH0EBXSikPoYGulFIeQgNdKaU8hAa6BxMRbxEpEJH2TteinCMiwSJiRKSN07VUR0SSReRKp+twdxroLsQO36O3ChEprPT4mlNdnjGm3BgTbIzZ3RD1noyI/FKp/nIRKar0+ME6LPcjEXmohukuGWCuWpfyHD5OF6COMcYEH70vIjuBycaYBdXNLyI+xpiyxqjtdBhjuhy9LyLLgBnGmDecq0gpz6Z76G5ERP4hIu+LyBwRyQcmicgQEfleRA6JyF4ReUlEfO35few9wlj78Tv29K9FJF9EVopIXDXrWiAit53w3EYRmSgiXvZyskQkV0R+FpHup/mefm/vyR8QkS9FpK39vLeIvCoi2fY61olIRxG5F/gd8IS9pz/nFNfnbW/HPSKyT0RmiEiwPS1YRD6wazlob9dQe9ptIrLL3m47ROTiapY/QkRW2zVniMizIuJtT06yf+6waz//VLaJPW2aiKSLSJ6IrBKRQZWm+YrIEyKSak9fLSIRlRZ9voik2Mt9poZt5CMif7WXs19E3haREHtaT/ub1h329ksXkd9Xem2Q/XvbJyJpIvLU0b9He/oVIrLe3o5bRWRUpVV3smvOs993aHU1qmoYY/TmgjdgJzDuhOf+AZQAF2B9GAcCA4Ezsb5txQNbgTvt+X0AA8Taj98B9gMDAF/gfeCdatZ/E7Ck0uMzgAOAH3AesBoItevoDrQ5yftZBtxwwnOTgI1AR7uefwPf2tMuAZYCIfY6egER9rSPgIdqWFew/b5/UxPwR3ud7e36vwZetafdZ2+TAHvbDbK3cYT93uPt+aKBrtWse7C9fb2BTkAK1jetGuuqzTaxp18HtLSnPW7/nfjY054Aku2/Ay+gv/0ej673Q6A5kADkAcOrqeERYDHQxn7/bwPT7Wk9gQpglj2tP3AIGGxPf85+bRgQBawF/mRPGw3kACPt+joAnexpycBmIM6udxXwiNP/h+52c7wAvVXzi6k+0L87yevuBz6071cV6FMrzTsR2FDNckKBI0CM/fgpYJp9/2xgC9YHiVct309Vgb4UuKLSY3+g3A6DicB6rA8sOeF1dQn0VcB1lR73B/Lt+38EvgO6n/CaMOAg1gep/yn+Hh8B3j5ZXbXZJlXM6w0UAQn243RgbA3bo0+l5+Zif/BXMf8e4MxKjztV2kY97WXFVJr+X+BF+34mMKLStEuO/o0Bs4G/V7POZODuSo8fBD5qqP8vT71pk4v72VP5gYh0FZGv7K+4ecDfgPAaXr+v0v0jWP/sv2GMyQW+Aa4QEQGuxPqHxBgzH5gKvApkishUEWl+Gu+lAzDDbi46hBUGJUAM8AXwJjAd2Cci/xGRoNNYx4naArsqPd4FBNtf76cBK4D/2U0y/xARL2NMDnA9cA/W+/1MRBKqWrjdJPGNiGTav48/U/Pv40Q1bRNE5C92c0wu1t6uHxBuN+tEATtqWPZJf/f2cqKB+ZVq+AHwFZEW9mwVWB8eR+0C2oqID9Ca327faPt+u7rWp2qmge5+Thwe8zVgA9DRGBMCPAZIPa1rDnAVMBzrb+VoGzDGmBeMMf2w9ti6A/eexvL3AJOMMS0q3QKNMT8ZyzPGmD5AH6xmjD8cXX0d3lMGVmge1R4oMMbkGmOKjDGPGOtg7mjgauAyAGPM58aYMVjhlAH8p5rlz8Rqjoq3fx//4tjvozZ1V7tNRGQCcBtwIdAC65tDCdY3mHJgL1ZzymmrtJwRJ9QQYIw5ZM/mxbGQBmsbZhjrAH0Wv92+R8N/T13rUzXTQHd/zYFc4LCIdANurcdlf4H1dfsx4D1jfxcWkUH2zQc4jBUq5aex/KnAYyLS2V5uy6MHG8U62NvfXkfBCevIxGonPhl/EQmodPPC+pB6QERi7AN9f8f+5iEiZ4lIN3u+PKAMKBeRdiJyrogEYjVxHK7h/TYHDhljDotIL2Dy0QnGmMP2e6mp9mq3ib3sMqzjIH7AP7Ha0o+aAfxbRGLFOnDd7zQPLE4FnhKRaLuGyBMO4Brgr/Y27Yf1wfe+PW2OPa2VWN0z/4zV1He0vttFZLhY2otIp9OoT1VDA9393YfVHJCPtbf+fs2z154xpgj4FBgHvFtpUgusPdFDWG39e4HnT2P5b2OFx6d288Q6YKw9uRXwlr2OFKyv6q/Y06YCQ8XqiTK7hlXsBAor3S7H2rP+AqstfRvWh8MD9vzt7Gn5wE/AZ8AnWMci/mLPux/rG8Nd1azzbqzQKgBeBN47YfpjWE06h0TkvFPcJp8CK4FUe5ukY7XtH/UPYD6wBGu7/Rcr+E/Vv7G+jS2xa1gG9K00vQT4EWv7fgk8ZoxZYU/7C9Z23QysARYBz9rvbRHWt6zXsD4wv8VqAlP1ROydLqWUOikR6QkkG2MCnK5F/ZbuoSullIfQQFdKKQ+hTS5KKeUhdA9dKaU8hGODc4WHh5vY2FinVq+UUm5pzZo1+40xEVVNcyzQY2NjSU5Odmr1SinllkRkV3XTtMlFKaU8hAa6Ukp5CA10pZTyEC51xaLS0lLS0tIoKipyupQGFxAQQExMDL6+viefWSmlasGlAj0tLY3mzZsTGxuLNWKrZzLGkJOTQ1paGnFxVV4wSCmlTplLNbkUFRURFhbm0WEOICKEhYU1iW8iSqnG41KBDnh8mB/VVN6nUqrxuFygK6WUxzIGFj8F+9Y3yOI10Cs5dOgQ//3vf0/5deeeey6HDh06+YxKqaZtxcuw+F+w4ZMGWbwGeiXVBXp5ec0X45k7dy4tWrSocR6lVBP384fw7aPQ4yIY82iDrMKlerk47aGHHmLHjh306dMHX19fgoODiYqKYt26dWzatInf/e537Nmzh6KiIu666y6mTJkCHBvGoKCggAkTJjB8+HBWrFhBdHQ0n332GYGBgQ6/M6WUo1IWw6e3Q2wiXPQaeDXMvrTLBvoTX2xkU0ZevS6ze9sQHr+gR7XTn3zySTZs2MC6detYvHgx5513Hhs2bPi1a+GsWbNo1aoVhYWFDBw4kEsuuYSwsLDjlrFt2zbmzJnD9OnTufzyy/n444+ZNGlSvb4PpZQb2bce3psE4Z3ginfAx7/BVuWyge4KBg0adFw/8Zdeeon//e9/AOzZs4dt27b9JtDj4uLo06cPAP3792fnzp2NVq9SysUc2g3vXAoBIXDNRxDYsE2zLhvoNe1JN5ZmzZr9en/x4sUsWLCAlStXEhQUxKhRo6rsR+7vf+zT19vbm8LCwkapVSnlYo4cgHcugbJCuGkehEY3+CpdNtCd0Lx5c/Lz86uclpubS8uWLQkKCmLLli18//33jVydUsptlBbCnKvg4E649lNo3a1RVquBXklYWBjDhg2jZ8+eBAYGEhkZ+eu08ePHM3XqVHr37k2XLl0YPHiwg5UqpVxWRTl8PBn2rILLXofYYY22aseuKTpgwABz4gUuNm/eTLdujfNJ5gqa2vtVyuMZA3Pvhx9mwPinYPBt9b4KEVljjBlQ1TTth66UUvVl2XNWmA/9Y4OE+clooCulVH1Y9y4s/Bv0uhzGPeFICRroSilVV9sXwOd/gLiRcOErDXbi0MlooCulVF1k/AjvXwcR3ewTh/wcK0UDXSmlTteBVJh9GQSFwTUfWicQOUgDXSmlTsfhHOvEoYoymPQxhEQ5XdHJA11E2onIIhHZLCIbReSuKuYZJSK5IrLOvj3WMOU2rNMdPhfghRde4MiRI/VckVLKJZUcgXcvh7x0uOo9iOjsdEVA7fbQy4D7jDHdgMHAHSLSvYr5lhpj+ti3v9VrlY1EA10pdVLlZfDRjZCxFi6ZCe1d5yTDk54paozZC+y17+eLyGYgGtjUwLU1usrD55511lm0bt2aDz74gOLiYi666CKeeOIJDh8+zOWXX05aWhrl5eU8+uijZGZmkpGRwejRowkPD2fRokVOvxWlVEMwBr66F7Z+A+c9C93Od7qi45zSqf8iEgv0BVZVMXmIiPwEZAD3G2M21qmyrx+q/8s0tekFE56sdnLl4XPnz5/PRx99xOrVqzHGMHHiRJKSksjOzqZt27Z89dVXgDXGS2hoKM899xyLFi0iPDy8fmtWSrmOJf8Ha9+ExPtg4GSnq/mNWh8UFZFg4GPgbmPMiQOVrwU6GGPOAF4GPq1mGVNEJFlEkrOzs0+35kYxf/585s+fT9++fenXrx9btmxh27Zt9OrViwULFvCnP/2JpUuXEhoa6nSpSqnGsPYt6/JxZ1zVYFccqqta7aGLiC9WmM82xvzmYniVA94YM1dE/isi4caY/SfMNw2YBtZYLjWutIY96cZgjOHhhx/m1ltv/c20NWvWMHfuXB5++GHOPvtsHnvMLY8BK6Vqa+s8+OJuSBgLE18GEacrqlJterkIMBPYbIx5rpp52tjzISKD7OXm1GehjaHy8LnnnHMOs2bNoqCgAID09HSysrLIyMggKCiISZMmcf/997N27drfvFYp5UHS1sCHN1hNtpe/Cd6+TldUrdrsoQ8DrgXWi8g6+7k/A+0BjDFTgUuB20WkDCgErjRODeNYB5WHz50wYQJXX301Q4YMASA4OJh33nmH7du388ADD+Dl5YWvry+vvvoqAFOmTGHChAlERUXpQVGlPEXODnj3MmgWYZ045N/c6YpqpMPnOqipvV+l3EpBFsw8C4rz4eZvISzB6YqAmofP1QtcKKXUiYoLrBOH8jPhhi9dJsxPRgNdKaUqKy+FD6+HvT/BlXMgpsqdYZfkcmO5uGHT+2lpKu9TKbdiDHxxlzUc7vnPQ5fxTld0Slwq0AMCAsjJyfH4sDPGkJOTQ0BAgNOlKKUqW/RPWDcbRj4E/W9wuppT5lJNLjExMaSlpeHqJx3Vh4CAAGJiYpwuQyl11A8zIelp6HstjHrI6WpOi0sFuq+vL3FxcU6XoZRqarZ8ZV3cudPZcP4LLnvi0Mm4VJOLUko1uj2r4aObIKoPXPYGeLvUfu4p0UBXSjVd+7dZ3RND2sLVH4BfM6crqhMNdKVU05S/D965GLx8rCsOBUc4XVGdue93C6WUOl1FeTD7Uusycjd8Ca3ina6oXmigK6WalrIS+OBayNxkNbNE93O6onqjga6UajoqKuDzOyFlMVz4X+g0zumK6pW2oSulmo6FT8DP78OYR6DvNU5XU+90D10p5fkqKmD587D8BRhwEyTe73RFDUIDXSnl2fb+DF/eA+nJ0G0inPuM2544dDIa6Eopz1ScD4v+BaumQmAruOg16H2Fx4Y5aKArpTyNMbDpU/jmYauvef8bYNzjENjS6coanAa6uystAlMBfkFOV6KU8w6kwFf3w46F1jVAr3jHrcYzrysNdHdVXgpr3oAlT4FfMNzyHQS1croqpZxRVgzLX4SkZ8DbD8Y/CQNvcetxWU5H03q3nsAY2Pw5LHgCDuyAmEHWlVU+vB4mfeLSVyRXqkGkLIav7oOc7dDjIjjnX9bYLE2Q9kN3J7tWWBet/eA6K7iveg9ung8XvAipSTDvL05XqFTjyc+EjyfDWxdCRbk1HstlbzTZMAfdQ3cPWVtgwV9h69fQPAomvgxnXH3s62Sfq2Dfevj+FWjTE/pd52i5SjWoinJIngUL/w5lhTDyTzD8HvANdLoyx2mgu7K8vbD4X/DjO+DbDMY8CoN/X/UB0LP+Blmb4Mt7IbwztB/c+PUq1dDS18JX90LGjxA/Cs59FsI7Ol2Vy9BAd0VFedYBnpWvQEUZDJoCIx6AZuHVv8bbBy57HaaPgfevhSmLIFQvcac8RFGutUf+wwwIbg2XzISel3h0n/LToYHuSspKrK+SSf8HR3KsP9gxj0KrWl6WL7Cl1a4+fSy8dw3c9I1+DVXuzRjY8DHM+zMUZMGgW6xxWAJCna7MJWmgu4KKCtj0P1j4Nzi4E2ITrSaU0xnWM6ILXDId5lwFn90Jl8zQvRjlnvZvh7n3Wb1YovpYOyseNNRtQ9BAd1pqEnz7mNUm2LoHXPMRdBxXtxDuMsHai/nu79ZB0uH31F+9SjW00iJY9hwsex58AqyxVwbcBF7eTlfm8jTQnZK5Eb59HLZ/CyEx8LtXrXEm6uuPNvE+ax0LnrA+KDqfXT/LVaohbV9gnel5MBV6XQZn/xOaRzpdldvQQG9suWnWgEHr3gX/EKtpZdCU+m/rFoELX7FOtvj4Zpi8ECI61+86lKoveRnW2CubPoWwjnDdZ1YvFnVKNNAbS+Eh62vkqtessVeG3GHtRTfk6fp+QXDluzB9NLx3lRXqgS0abn1KnaryMvhhOnz3TygvgdF/gWF3gY+/05W5JQ30hlZaZP3BJj1jdb3qfQWM+Qu0aN8462/RDi5/G968wNpTv/oDbYtUriEtGb682zopruM4OPdpj7lYs1M00BtKRQWs/xC++wfk7oaEMTDuCYjq3fi1dBhi/bN8eTcseBzO/kfj16DUUYUHrR5dya9D8zZw2ZvQ/ULtjVUPNNAbwo7vrJ4r+9ZDm94w8UUr0J004EbI3AArXobIXnDGFc7WoxrewV2w5Utr9EG/YPAPtn82t376NbOfa944oxIaY13Pc/4j1nkWg2+HUQ9DQEjDr7uJOOlvUUTaAW8BbYAKYJox5sUT5hHgReBc4AhwgzFmbf2X6+L2/mT1XElZZDWpXDwdel4KXi4yBtr4JyH7F/j8D9bp0tH9na5INYSKclg9zdoLLj1Su9f4BFgBX13gH/0w8Gt2bLq//bjy9KM/Txz1M/sXa0TEnUsheoA1MqgT31Y9XG0+lsuA+4wxa0WkObBGRL41xmyqNM8EoJN9OxN41f7ZNBzcZTWtrP/AOlvznH/BwMmud2DH29f6ejttlHUm6ZTF1lde5TmytsDnd0LaD9DxLJjwlBXAxflQUgAlh6G4AEry7Z+Hreerml540OqVVVJw7DlTUbs6vP0rfQgEw/6t1kH681+Afte7zk6OhzlpoBtj9gJ77fv5IrIZiAYqB/qFwFvGGAN8LyItRCTKfq3nOnIAlj5r7Q2JFwy72zqJx5V7kjQLg6vehZlnw/uT4PovwTfA6apUXZWVWFe0T3raCtCLp1v9uI+2Swe3rvs6jIGyotp/IFSe3mGoNSpicETd61DVOqWGMxGJBfoCq06YFA3sqfQ4zX7uuEAXkSnAFID27euxl4cx1hV8youtn2XFVheoo7ey4krTS6w//mrnPXq/8vTK9ystJ+MnKM6DPtfA6IfdZzCsNr3goqnWuOpf3Wv1V9cDUu4rfQ189gfI2miN/zP+qYYJThHrfAnfQECD2RXVOtBFJBj4GLjbGJN34uQqXmJ+84Qx04BpAAMGDPjN9Fr55Wv48p7fhmt98/a3mky8fa373r72Y79jt07jrL7kkT3qf/0NrfuF1h7TkqesgB98u9MVqVNVcsQaXnnlKxAcCVfOga7nOl2VclCtAl1EfLHCfLYx5pMqZkkD2lV6HANk1L28KgRHQqez7FCtHLRHg9cPfPyqmG4/V+28laZ7+TSNPdaRD1nDA8z7C0R0hYTRTlekais1CT7/o3WKfP8brDOOdQTCJq82vVwEmAlsNsY8V81snwN3ish7WAdDcxus/Ty6n464Vl+8vKyml5lnw4c3WBeaDktwuipVk6Jcq0vsmjegZRxc/wXEjXC6KuUianOoeRhwLTBGRNbZt3NF5DYRuc2eZy6QAmwHpgO/b5hyIbewlE0ZeZSW1/Jou6qZf3NreAAReO9q6+IayjVtmQuvnAlr34Khf4DbV2iYq+PUppfLMqpuI688jwHuqK+iarJ0WzZ3vvsjft5edGnTnJ7RIXRvG0rPtiF0iwohwFdPaz9lreKs7oxvXwSfTLECXruVuY6CbPj6Qdj4iTVy5pWz9RwCVSWxsrjxDRgwwCQnJ5/y67Lyi/g+5QAb03PZkJHLxow8Dh0pBcDbS0iIaEbPtqH0iA6lR9sQurcNISTA9yRLVYA1cNjXD1qXuxvziNPVKGPg5w/gmz9ZXQBHPmh1jfXxc7oy5SARWWOMGVDlNHcL9BMZY0g/VMiG9Dw2ZeSyISOPjRm5ZOYV/zpPbFgQPdqG0iM6xAr7tiGEBbvYST+uwBjrLNIf34bL3oAeFzldUdN1aI/Vm2v7txAzECb+B1p3dboq5QJqCnS3H8tFRIhpGURMyyDG9zx21mNWfhEbM/LYlJHHhvRc1qfn8tX6Y8dpo0ID6NE2hB5tQ+lp781HhQYgTaF3S3VE4LxnrbP6Pv09tErQ07MbW0UFJM+EBX+1zsoc/5R1HU0dIVPVgtvvoZ+K3COlbNyb+2vIb8jIIyW7gAp7E7Rq5lcp5K2fHVoF4eXVxEI+P9MaQ1284JZFenZfY9m/zfqGtHslxI+GC16AlrFOV6VcjEc3udTVkZIyNu/Nt5pr0vPYkJHL1sx8Ssut7RLs70P3tiH0aGs11/SMDiUhohk+3h5+0DDjR5g1Htr2s64eo+22Dae81BoFc/GT1jAM5/wb+lzdNM6FUKdMA/0UlZRVsDUzn432QdcN6bls3ptPYWk5AP4+XnSNCmFAh5ZcPySW9mFBDlfcQNZ/ZF0UY8BNcP7zTlfjmfb+BJ/dYQ213G2idUFkvYamqoFHt6E3BD8fL3pGW3vjR5VXGFL3F7Ah3TrouiE9j7dX7uL15amc2yuK20YmHDe/R+h1qRU0y1+AyJ4w8GanK/IcpUWw5ElY/hIEhcHlb1nDMShVB7qHXgeZeUXMWp7Ku9/vJr+4jMRO4dw2MoGhCWGec3C1ohzmXGldtOO6zyB2uNMVub9dK60hbnO2Q59JcM4/rGGXlaoFbXJpYHlFpcz+fjezlqeSnV9Mr+hQbh0Zz4SeUXh7wgHVolyYPhYKD1gHSVt2cLoi91ScDwuesK4x26I9XOACV7JSbkcDvZEUl5Xzv7XpTEtKIWX/YTqEBXFLYjyX9o9x/zNY92+H6WOsILp5nnWlGlV7276FL+6GvHRrZMvRf7EuAKHUKdJAb2TlFYZvN+3j1SUp/LTnEOHBftw4LI5JZ3YgNMiNz1rdtgDevcw6eHfZG9oLozYO58C8h61raUZ0hYkvQ7tBTlel3JgGukOMMXyfcoCpS3awZGs2zfy8uWpQe25OjCMqNNDp8k7P8pfg20dh9CMw8gGnq3Fdxlhjr8x9EIoOWePmJ97nepclVG5He7k4REQYkhDGkIQwNmXkMS1pB6+v2MmbK3dyYZ9obh0RT6fI5k6XeWqG/gEyN8Cif0Bkd+h6ntMVuZbSIti5DH6YAVu/hrZ9YeJn0Kan05WpJkD30BvZngNHmLkslfd+2E1RaQXjurXmtpEJDIht5XRptVdaCK9PsM5svPlbK9ibssP7Yes8+GUu7FgEpYet63qOegjOvB28db9J1R9tcnFBBw6X8Ka9t37oSCkDOrTktpEJjOna2j2GGsjLgGmjrOtL3rIIgtzoA6mujIHsLdblEH/5GtJ+AAyEREPn8dBlAsQm6sW3VYPQQHdhR0rKeP+HPcxYmkr6oUI6RwYzZUQCE89oi5+Piw8vsOcHeONc64ru13zs2Xui5aWwazn88o3VlHJwp/V8VB/oci50GQ9teuuBYtXgNNDdQGl5BV/9vJepS3awZV8+UaEB3Dw8jqsGtaeZvwsH5Y/vWKeun3k7THjS6Wrq15EDsH2BtRe+fSEU54JPAMSNtPbCO4+HkCinq1RNjAa6GzHGsHhrNlMX72BV6gFCA325dnAHbhgWS7irjuH+9UOw6lXofYW1xxreGSI6Q0iM+135KGeH1Rb+yzfWqIemHJq1hs7nWHvi8SO1D75ylAa6m/px90GmLtnB/E2Z+Hl7cdmAGKYkJrjeYGDlZdawr1u/hsKDx573DYKwjhDRxQr5o7ewBNfpvldeBmmrj4V4zjbr+ciex9rD2/Zzvw8m5bE00N3cjuwCpiel8MnadMoqKlx3MDBjrB4f+7fC/l+sXjDZ9s/c3cfmEy9rnO/wLhDe6fjAD2zR8HUW5cGOhVZTyrb51oeQly/EJULnCVZ7eIv2DV+HUqdBA91DnDgY2PCO1mBgwzq6wWBgJYetwaiytx4f+Dnbobzk2HzNWtsB38kK/Ag76EOi63bA8eAu2PqNFeI7l0FFKQS2gk5nW3vhCWMgIKTu71OpBqaB7mFOHAysZ3QIk4fHc17vKHzd7cIb5WVwaJcd8luPD/yi3GPz+TartDdvh314Z2gVX/XFNyoqIH2N1Qz0yzeQtdF6Pryz3ZRyrnUKvl7aTbkZDXQPdXQwsOlLU9iRfZio0ABuGBrLlYPaExroxmPGgN18k2032Ww9PvDz0o7NJ97QKu5Y803LDnaQz4fDWdb0DkOPtYeHJTj3npSqBxroHq6iwrB4axbTk1JZmZJDMz9vrhjYnhuHxdKulYsdQK0PxQXWwctf2+jtsM/ZYTWl+IdCp3FWe3incTrWuPIoGuhNyIb0XGYsTeHLn/dSYQwTekYxOTGOvu2bQKiVl1l77yHR4O3m31CUqoYGehO0N7eQN1bs5N1Vu8kvKmNAh5ZMToznrO6RnnHRDaWaKA30JqyguIwPftjDrOWppB0spENYEDcNi+OyATEE+bnwGahKqSppoCvKyiuYvymT6UtT+HH3IUIDfbnmzPZcPzSWyBAdREopd6GBro6zZtcBpielMm/TPny8hIlnRDM5MY5uUdoPWylXpxe4UMfp36EV/a9txa6cw7y+fCcfJO/h47VpDO8YzuTEOEZ2jnD9E5WUUr+he+iK3COlzF69izdX7CQzr5jOkcFMHh7PhX3b4u+jJ94o5Uq0yUXVSklZBRNixmQAABQESURBVF/8lMH0pSls2ZdPeLA/1w3pwKTBHWjVrIqzMZVSjU4DXZ0SYwzLt+cwY1kKi3/JJsDXi0v6xXDz8DjiI4KdLk+pJq1ObegiMgs4H8gyxvzmSrciMgr4DEi1n/rEGPO30y9XOU1EGN4pnOGdwtmamc/Mpal8mJzGu6t3M7ZrayYnxnNmXCttZ1fKxZx0D11ERgAFwFs1BPr9xpjzT2XFuofuXrLzi3l75U7e/n4XB4+U0is6lMmJcZzbyw0HBFPKjdW0h37S/0RjTBJwoN6rUm4lork/957dhRUPjeWfF/XkcHEZd723jpH/t4hpSTvIKyp1ukSlmrxataGLSCzwZQ176B8DaUAG1t76xmqWMwWYAtC+ffv+u3btOt26lcMqKgzfbcli+tIUVqUeINjfh5uGxXL7qI4E+mnPGKUaSp0Pip4k0EOACmNMgYicC7xojOl0smVqk4vnWJ+Wy9QlO/hq/V5iWgby1wt6MK57pNNlKeWR6tTkcjLGmDxjTIF9fy7gKyLhdV2uch+9YkJ55Zp+zLllMIG+3kx+K5mb3/iBPQeOOF2aUk1KnQNdRNqI3d1BRAbZy8yp63KV+xmSEMbcuxL587ldWZmSw7jnlvDigm0UlZY7XZpSTcJJA11E5gArgS4ikiYiN4vIbSJymz3LpcAGEfkJeAm40jjVuV05ztfbiykjElh430jGdY/k+QVbOeeFJBb9kuV0aUp5PD2xSDWopduyefyzjaTsP8w5PSJ57IIeRLcIdLospdxWg7ahK1WTxE4RfH13Ig+c04UlW7MZ++xiXlm0nZKyCqdLU8rjaKCrBufv480dozuy4N6RjOwcwdPzfmH8i0ks377f6dKU8iga6KrRxLQM4rVrB/D6jQMprzBcM2MVd7y7ln25RU6XppRH0EBXjW50l9bMu3sE94zrzIJNmYx9djHTk1IoLddmGKXqQgNdOSLA15u7xnXi23tGcmZ8GP+cu5nzXlrK9yna41Wp06WBrhzVPiyIWTcMZPp1AzhcXM6V077n7vd+JCtfm2GUOlUa6MolnNU9kgX3juQPYzoyd/0+xj6zhFnLUinTZhilak0DXbmMQD9v7ju7C/PuGUHfDi3525ebOP/lZSTv1ME+laoNDXTlcuLCm/HmjQN59Zp+5BaWcunUldz/4U/sLyh2ujSlXJoGunJJIsKEXlEsvG8kt41M4NMf0xnzzGLeXrmT8godWUKpqmigK5cW5OfDQxO68s3difSMDuXRzzZy4SvL+HH3QadLU8rlaKArt9CxdXNmTz6Tl67qS1ZeMRe/uoKHP/mZg4dLnC5NKZehga7chogw8Yy2LLxvJDcPi+OD5DTGPLuY91bvpkKbYZTSQFfup3mAL4+c352v/jicTq2b89An67n41RVsSM91ujSlHKWBrtxW1zYhvH/rYJ67/AzSDh7hgv8s49FPN5CjvWFUE+XjdAFK1YWIcHG/GMZ2i+T5b7fy1sqdfJC8h0v6x3Dz8DgSIoKdLlGpRqMXuFAeZUd2ATOWpvLx2jRKyysY2zWSKSPiGRjbEvtKiUq5tZoucKGBrjzS/oJi3lq5i7dX7uTgkVLOaNeCKYnxnNMjEh9vbWlU7ksDXTVZhSXlfLQ2jZlLU9iZc4R2rQK5aVgclw9oRzN/bXFU7kcDXTV55RWGBZszmZ6UQvKug4QE+DBpcAduGBpL65AAp8tTqtY00JWqZO3ug8xYmsI3G/bh7SX8rk80t4yIp3Nkc6dLU+qkNNCVqsKunMPMWpbKB8lpFJaWM6pLBLckxjM0IUwPoCqXpYGuVA0OHi5h9qpdvLFiF/sLiukeFcKUEfGc1zsKXz2AqlyMBrpStVBUWs5n69KZvjSV7VkFRIUGcNOwOK4Y1I6QAF+ny1MK0EBX6pRUVBiWbM1mWlIKK1NyCPb34apB7bhxWBxtWwQ6XZ5q4jTQlTpN69Nymb40ha/W70WA83tHMTkxnp7RoU6XppooDXSl6ij9UCGvL0tlzurdHC4pZ2hCGLeMiGdU5wg9gKoalQa6UvUkt7CU91bv5vXlO9mXV0TnyGAmJ8ZzYZ+2+Pt4O12eagI00JWqZyVlFXz5cwbTklLYsi+fiOb+3DA0lmvObE+LID+ny1MeTANdqQZijGH59hymLU0haWs2gb7eXDGwHTcNi6N9WJDT5SkPpIGuVCPYvDePGUtT+fyndMorDJcPaMc9Z3UmUocWUPVIA12pRpSZV8TUJTt45/tdeHsJtyTGM2VEPM21L7uqBzUF+klPgxORWSKSJSIbqpkuIvKSiGwXkZ9FpF9dC1bKnUWGBPD4BT1YeO8ozurehpe/286opxfz5oqdlJRVOF2e8mC1Oa/5DWB8DdMnAJ3s2xTg1bqXpZT7ax8WxMtX9eXzO4fRKTKYxz/fyNnPL2Hu+r049c1YebaTBroxJgk4UMMsFwJvGcv3QAsRiaqvApVyd71jWjDnlsG8fsNA/Hy8+P3stVz86gpWp9b0b6XUqauPkYeigT2VHqfZzymlbCLC6K6t+fquEfzfJb3JOFTI5a+t5Ja3ktmele90ecpD1EegV3WaXJXfJ0Vkiogki0hydnZ2PaxaKffi7SVcPrAdi+8fzQPndGHljhzOfj6Jhz9ZT1ZekdPlKTdXH4GeBrSr9DgGyKhqRmPMNGPMAGPMgIiIiHpYtVLuKdDPmztGd2TJA6O4bkgsHybvYeTTi3nu260UFJc5XZ5yU/UR6J8D19m9XQYDucaYvfWwXKU8XliwP3+d2IMF945kTLfWvLRwG6OeXsTbK3dSWq49YtSpOWk/dBGZA4wCwoFM4HHAF8AYM1WskYn+g9UT5ghwozHmpB3MtR+6Ur+1bs8h/jV3M6tTDxAf3owHx3fhnB5tdAAw9Ss9sUgpN2KM4bstWfz76y1szyqgX/sW/PncbgyIbeV0acoF1OnEIqVU4xIRxnaL5Ju7Enny4l6kHSzk0qkrmfJWMtuzCpwuT7kw3UNXysUdKSlj5tJUXktKobC0nCsHtuOucZ1o3VzHiGmKtMlFKQ+wv6CYlxduY/aq3fj5eDFlRDy3JMbTzN/H6dJUI9JAV8qDpO4/zNPztjB3/T7Cg/25e1wnrhjYDl9vbUFtCrQNXSkPEhfejP9e059Pfj+UuPAgHvl0A+e8kMS8jft0jJgmTgNdKTfVr31LPrh1CNOvG4AAt769hkunrmTNLh0jpqnSQFfKjYkIZ3WPZN7dI/jXRb3YfeAIl7y6ktveXkNKtvaIaWq0DV0pD3KkpIwZS1N5bckOisoquHJgO24bmUC7Vno5PE+hB0WVamKy84t5ceFW3lu9BwOc1yuKKSPi6Rkd6nRpqo400JVqovbmFjJrWSpzVu+hoLiMxE7h3DoigWEdw3Q4ATelga5UE5dbWMq7q3Yza3kq2fnF9GgbwpQR8ZzXKwof7e7oVjTQlVIAFJeV8+mP6byWlEJK9mFiWgZyS2I8lw2IIchPT1ByBxroSqnjVFQYFmzO5LWkFNbsOkjLIF+uHRLL9UM6EBbs73R5qgYa6EqpaiXvPMDUJSks2JyJv48Xlw9ox+TEODqENXO6NFUFDXSl1Eltz8pnWlIK//sxnfIKw4ReUdw6Ip7eMS2cLk1VooGulKq1zLwiXl++k9nf7yK/uIyhCWHcOjKBEZ3CtWeMC9BAV0qdsvyiUuas3s3MZalk5hXTtU1zbhuZwHm9o3QgMAdpoCulTltJWQWfrUtnWlIK27IKiG4RyE3D47hyYDsdutcBGuhKqTqrqDAs+iWL15JSWJ16gNBAX64d3IHrh8YS0Vx7xjQWDXSlVL1au/sg05akMG/TPny9vbi0fwy3JMYTF649YxqaBrpSqkGkZBcwfWkqH69No7S8gvE92jBlRDx927d0ujSPpYGulGpQWflFvLliJ2+v3EVeURmD4lpx28h4RnVujZeX9oypTxroSqlGUVBcxnt2z5i9uUV0jgxmyogEJp7RFj8f7RlTHzTQlVKNqrS8gi9+ymBaUgpb9uXTJiSA64Z24OpB7WkR5Od0eW5NA10p5QhjDIu3ZjNzaSrLtu8n0NebS/vHcOOwWOIjgp0uzy1poCulHLd5bx6zlqXy2boMSsorGNu1NTcnxjEkXsdmPxUa6Eopl5GdX8zb3+9i9ve7yDlcQreoEG4eHscFZ0Th7+PtdHkuTwNdKeVyikrL+WxdOjOXpbI1s4CI5v5cN7gD1wzuQKtm2s5eHQ10pZTLMsawdNt+Zi5LZcnWbPx9vLi4XzQ3DYujU2Rzp8tzOTUFug7EoJRylIgwonMEIzpHsC0zn1nLU/lkbTpzVu9hZOcIJifGMbyjjvRYG7qHrpRyOTkFxcxetZu3Vu5if0ExXSKbc9PwWC7sE02Ab9NuZ9cmF6WUWyouK+eLn/Yyc1kqm/fmEdbMj0mDOzBpcIcmOyCYBrpSyq0ZY1i5I4eZy1JZuCULP28vLuzTlpsT4+jaJsTp8hpVndvQRWQ88CLgDcwwxjx5wvQbgKeBdPup/xhjZpx2xUopVYmIMLRjOEM7hrMju4DXl6fy0Zo0PlyTxvCO4dw8PI6RnSOa/LgxJ91DFxFvYCtwFpAG/ABcZYzZVGmeG4ABxpg7a7ti3UNXStXFoSMlvLt6N2+u2ElmXjEJEc24aXgcF/eNIdDPc9vZa9pDr81oOYOA7caYFGNMCfAecGF9FqiUUqeqRZAfvx/VkaUPjuGFK/oQ6OfNX/63gaFPLuSZeb+QlVfkdImNrjaBHg3sqfQ4zX7uRJeIyM8i8pGItKtqQSIyRUSSRSQ5Ozv7NMpVSqnj+fl48bu+0Xxx53DenzKYgbGteGXxdoY99R33vr+ODem5TpfYaGrThl5Vo9SJ7TRfAHOMMcUichvwJjDmNy8yZhowDawml1OsVSmlqiUinBkfxpnxYezKOczry3fyQfIePvkxncHxrbh5eDxju3r2+Oy1aUMfAvzVGHOO/fhhAGPMv6uZ3xs4YIwJrWm52oaulGpouYWlvP/Dbt5YvpOM3CJaN/dnbLfWjOsWybCO4W7Zp71O3RZFxAfroOhYrF4sPwBXG2M2Vponyhiz175/EfAnY8zgmparga6Uaiyl5RXM27iPuev3krR1PwXFZQT4ejG8Yzhju0UytmtrWocEOF1mrdSp26IxpkxE7gTmYXVbnGWM2SgifwOSjTGfA38UkYlAGXAAuKHeqldKqTry9fbi/N5tOb93W4rLylmVcoCFmzNZsDmLBZuzAOgdE8q4bpGM7daa7lEhbjnUgJ5YpJRqsowx/JKZz8LNWSzYnMm6PYcwBtqGBjCmW2vGdotkSHyYSzXN6JmiSilVC9n5xSzaYoX70m37KSwtJ8jPm8ROVtPMmK6tCQ92dsgBDXSllDpFRaXlrEzJYeHmTBZuzmJvbhEi0Kddi1+bZrpENm/0phkNdKWUqgNjDBsz8li4OYuFWzL5Oc3q2x7TMvDXcD8zLgw/n9qc2lM3GuhKKVWPMvOK+G5LFgvtppnisgqC/X0Y0TmcsV0jGd21dYNddUkDXSmlGkhhSTnLt+9n4RaraSYrvxgvgf4dWjK2WyTjurUmISK43ppmNNCVUqoRVFQYNmTkWt0hN2WyaW8eAB3Cghjb1Qr3gXGt8PU+/aYZDXSllHJAxqFCFtpNMyt25FBSVkHzAB/+OKYTt4yIP61l6jVFlVLKAW1bBHLt4A5cO7gDh4vLWLZ9Pws3Z9ImtGHOStVAV0qpRtDM34dzerThnB5tGmwdDd/HRimlVKPQQFdKKQ+hga6UUh5CA10ppTyEBrpSSnkIDXSllPIQGuhKKeUhNNCVUspDOHbqv4hkA7tO8+XhwP56LMfd6fY4nm6PY3RbHM8TtkcHY0xEVRMcC/S6EJHk6sYyaIp0exxPt8cxui2O5+nbQ5tclFLKQ2igK6WUh3DXQJ/mdAEuRrfH8XR7HKPb4ngevT3csg1dKaXUb7nrHrpSSqkTaKArpZSHcLtAF5HxIvKLiGwXkYecrsdJItJORBaJyGYR2Sgidzldk9NExFtEfhSRL52uxWki0kJEPhKRLfbfyBCna3KKiNxj/49sEJE5ItIwlwxymFsFuoh4A68AE4DuwFUi0t3ZqhxVBtxnjOkGDAbuaOLbA+AuYLPTRbiIF4FvjDFdgTNoottFRKKBPwIDjDE9AW/gSmerahhuFejAIGC7MSbFGFMCvAdc6HBNjjHG7DXGrLXv52P9w0Y7W5VzRCQGOA+Y4XQtThOREGAEMBPAGFNijDnkbFWO8gECRcQHCAIyHK6nQbhboEcDeyo9TqMJB1hlIhIL9AVWOVuJo14AHgQqnC7EBcQD2cDrdhPUDBFp5nRRTjDGpAPPALuBvUCuMWa+s1U1DHcLdKniuSbf71JEgoGPgbuNMXlO1+MEETkfyDLGrHG6FhfhA/QDXjXG9AUOA03ymJOItMT6Jh8HtAWaicgkZ6tqGO4W6GlAu0qPY/DQr061JSK+WGE+2xjzidP1OGgYMFFEdmI1xY0RkXecLclRaUCaMeboN7aPsAK+KRoHpBpjso0xpcAnwFCHa2oQ7hboPwCdRCRORPywDmx87nBNjhERwWoj3WyMec7pepxkjHnYGBNjjInF+rv4zhjjkXthtWGM2QfsEZEu9lNjgU0OluSk3cBgEQmy/2fG4qEHiH2cLuBUGGPKROROYB7WkepZxpiNDpflpGHAtcB6EVlnP/dnY8xcB2tSruMPwGx75ycFuNHhehxhjFklIh8Ba7F6hv2Ihw4BoKf+K6WUh3C3JhellFLV0EBXSikPoYGulFIeQgNdKaU8hAa6Ukp5CA10pZTyEBroSinlIf4fxUXz8D1zZ4cAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "es = EarlyStopping(monitor='val_loss', patience = 5, mode='min', verbose=1)\n",
    "\n",
    "history = model.fit(X_train, y_train, epochs=30, batch_size=32, validation_data=[X_test, y_test], callbacks=[es])\n",
    "# evaluate the model\n",
    "_, train_acc = model.evaluate(X_train, y_train, verbose=0)\n",
    "_, test_acc = model.evaluate(X_test, y_test, verbose=0)\n",
    "print('Train Accuracy: %.3f, Test Accuracy: %.3f' % (train_acc, test_acc))\n",
    "# plot training history\n",
    "plt.plot(history.history['loss'], label='train')\n",
    "plt.plot(history.history['val_loss'], label='test')\n",
    "plt.title(\"Train vs Test Loss at each epoch\")\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 5: Use pre-trained Word2Vec as the embedding layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "nw7rYGqdde2e"
   },
   "outputs": [],
   "source": [
    "w2v_model2 = Word2Vec(data,\n",
    "                size=100, #size is the number of dimensions of the N-dimensional space and Word2Vec maps the words into\n",
    "                window=5,\n",
    "                min_count=3, #min frequency appeared in the corpus\n",
    "                workers=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "UZ0XpKJkde2h",
    "outputId": "06eb2c99-3434-4668-d890-e61ea841fbb3"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3186140, 3460040)"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "w2v_model2.train(data,total_examples=w2v_model2.corpus_count, epochs=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "60ZVS0yRde2o"
   },
   "outputs": [],
   "source": [
    "wv = w2v_model2.wv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "#save model to be used in deep learning step\n",
    "filename = 'jd_embedding_word2vec2.txt'\n",
    "w2v_model2.wv.save_word2vec_format(filename, binary=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [],
   "source": [
    "#load the word embedding as a directory of words to vectors\n",
    "\n",
    "import os\n",
    "\n",
    "embeddings_index = {}\n",
    "f = open(os.path.join('','jd_embedding_word2vec2.txt'), encoding = \"utf-8\")\n",
    "for line in f:\n",
    "    values = line.split()\n",
    "    word = values[0]\n",
    "    coefs = np.asarray(values[1:])\n",
    "    embeddings_index[word] = coefs\n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 13272 unique tokens in the dataset.\n"
     ]
    }
   ],
   "source": [
    "#Get the total_vocabulary in the corpus\n",
    "total_vocabulary = set(word for jd in data for word in jd)\n",
    "\n",
    "max_length=len(total_vocabulary)\n",
    "print('There are {} unique tokens in the dataset.'.format(len(total_vocabulary)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 26519 unique tokens.\n",
      "Shape of the jd tensor: (963, 500)\n"
     ]
    }
   ],
   "source": [
    "#Convert the word embedding into tokenized vector. \n",
    "new_tokenizer = text.Tokenizer() \n",
    "new_tokenizer.num_words = 1000  #limit the number of words to keep most common 1000 words\n",
    "new_tokenizer.fit_on_texts(list(salary_df['combined_text']))\n",
    "list_tokenized_jd = new_tokenizer.texts_to_sequences(salary_df['combined_text'])\n",
    "\n",
    "#pad sequences\n",
    "word_index = new_tokenizer.word_index\n",
    "print('Found %s unique tokens.' % len(word_index))\n",
    "\n",
    "jd_pad = sequence.pad_sequences(list_tokenized_jd, maxlen=500, padding = 'post') \n",
    "print('Shape of the jd tensor:', jd_pad.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "#map embeddings from word2vec model for each word to new_tokenzier.word_index and create a matrix with word vectors\n",
    "\n",
    "num_words = len(word_index)+1  #1000  + 1\n",
    "embedding_matrix = np.zeros((num_words, 100)) #embedding size = 100\n",
    "\n",
    "for word, i in word_index.items():\n",
    "    if i > num_words:\n",
    "        continue\n",
    "    embedding_vector = embeddings_index.get(word)\n",
    "    if embedding_vector is not None:\n",
    "        #words not found in embedding index will be all-zeros.\n",
    "        embedding_matrix[i] = embedding_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "26520\n"
     ]
    }
   ],
   "source": [
    "print(num_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Embedding, LSTM, GRU\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.initializers import Constant"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "shall we limit the input_dim since it's too big? "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "#define model\n",
    "model = Sequential()\n",
    "\n",
    "embedding_size = 100  #word2vec has a 100 dimension space\n",
    "embedding_layer = Embedding(input_dim = num_words, #input_dim: size of the vocabulary\n",
    "                    output_dim = embedding_size,  #dimension of the dense embedding\n",
    "                    embeddings_initializer = Constant(embedding_matrix),\n",
    "                    input_length = 500, #length of input sequences\n",
    "                    trainable = False)\n",
    "model.add(embedding_layer)\n",
    "#model.add(LSTM(128))\n",
    "model.add(Dense(128, activation='relu'))\n",
    "model.add(Dense(32, activation='relu'))\n",
    "model.add(Flatten())   #add Flatten layer \n",
    "model.add(Dense(6, activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_22\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_11 (Embedding)     (None, 500, 100)          2652000   \n",
      "_________________________________________________________________\n",
      "dense_46 (Dense)             (None, 500, 64)           6464      \n",
      "_________________________________________________________________\n",
      "dense_47 (Dense)             (None, 500, 32)           2080      \n",
      "_________________________________________________________________\n",
      "flatten_5 (Flatten)          (None, 16000)             0         \n",
      "_________________________________________________________________\n",
      "dense_48 (Dense)             (None, 6)                 96006     \n",
      "=================================================================\n",
      "Total params: 2,756,550\n",
      "Trainable params: 104,550\n",
      "Non-trainable params: 2,652,000\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(jd_pad, y, test_size=0.1, random_state=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 866 samples, validate on 97 samples\n",
      "Epoch 1/10\n",
      "866/866 [==============================] - 2s 2ms/step - loss: 1.9188 - accuracy: 0.2460 - val_loss: 1.6356 - val_accuracy: 0.3402\n",
      "Epoch 2/10\n",
      "866/866 [==============================] - 1s 1ms/step - loss: 0.7641 - accuracy: 0.8545 - val_loss: 1.6006 - val_accuracy: 0.3918\n",
      "Epoch 3/10\n",
      "866/866 [==============================] - 1s 996us/step - loss: 0.2480 - accuracy: 0.9654 - val_loss: 1.8980 - val_accuracy: 0.3814\n",
      "Epoch 4/10\n",
      "866/866 [==============================] - 1s 993us/step - loss: 0.0780 - accuracy: 0.9954 - val_loss: 2.0158 - val_accuracy: 0.3814\n",
      "Epoch 5/10\n",
      "866/866 [==============================] - 1s 1ms/step - loss: 0.0313 - accuracy: 0.9977 - val_loss: 2.0759 - val_accuracy: 0.3814\n",
      "Epoch 6/10\n",
      "866/866 [==============================] - 1s 968us/step - loss: 0.0210 - accuracy: 0.9988 - val_loss: 2.2589 - val_accuracy: 0.3711\n",
      "Epoch 7/10\n",
      "866/866 [==============================] - 1s 956us/step - loss: 0.0123 - accuracy: 0.9988 - val_loss: 2.3176 - val_accuracy: 0.3505\n",
      "Epoch 8/10\n",
      "866/866 [==============================] - 1s 862us/step - loss: 0.0087 - accuracy: 0.9988 - val_loss: 2.3525 - val_accuracy: 0.3505\n",
      "Epoch 9/10\n",
      "866/866 [==============================] - 1s 826us/step - loss: 0.0067 - accuracy: 1.0000 - val_loss: 2.4039 - val_accuracy: 0.3505\n",
      "Epoch 10/10\n",
      "866/866 [==============================] - 1s 852us/step - loss: 0.0053 - accuracy: 1.0000 - val_loss: 2.4480 - val_accuracy: 0.3505\n"
     ]
    }
   ],
   "source": [
    "#lower the learning rate since it is training too fast. Limit the input length\n",
    "\n",
    "history = model.fit(X_train, y_train, epochs=10, batch_size=32, validation_data=[X_test, y_test])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Tuning the model\n",
    "\n",
    "https://blog.keras.io/using-pre-trained-word-embeddings-in-a-keras-model.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_size = 100  #word2vec has a 100 dimension space\n",
    "embedding_layer = Embedding(input_dim = num_words, #input_dim: size of the vocabulary\n",
    "                    output_dim = embedding_size,  #dimension of the dense embedding\n",
    "                    embeddings_initializer = Constant(embedding_matrix),\n",
    "                    input_length = 500, #length of input sequences\n",
    "                    trainable = False)\n",
    "\n",
    "sequence_input  = Input(shape=(500,), dtype='int32')\n",
    "embedded_sequences = embedding_layer(sequence_input)\n",
    "x = Conv1D(128, 5, activation='relu')(embedded_sequences)\n",
    "x = MaxPooling1D(3)(x)\n",
    "x = Flatten()(x)\n",
    "x = Dense(64, activation='relu')(x)\n",
    "x = Dense(16, activation='relu')(x)\n",
    "preds = Dense(6, activation='softmax')(x)\n",
    "\n",
    "opt = Adam(learning_rate=0.001)\n",
    "\n",
    "model = Model(sequence_input, preds)\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer=opt,\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 866 samples, validate on 97 samples\n",
      "Epoch 1/100\n",
      "866/866 [==============================] - 3s 3ms/step - loss: 0.3415 - accuracy: 0.8303 - val_loss: 3.0663 - val_accuracy: 0.4124\n",
      "Epoch 2/100\n",
      "866/866 [==============================] - 3s 3ms/step - loss: 0.3274 - accuracy: 0.8406 - val_loss: 3.0057 - val_accuracy: 0.3814\n",
      "Epoch 3/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.3206 - accuracy: 0.8522 - val_loss: 3.1373 - val_accuracy: 0.3711\n",
      "Epoch 4/100\n",
      "866/866 [==============================] - 3s 3ms/step - loss: 0.3153 - accuracy: 0.8580 - val_loss: 3.0982 - val_accuracy: 0.3918\n",
      "Epoch 5/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.2925 - accuracy: 0.8545 - val_loss: 3.2127 - val_accuracy: 0.3711\n",
      "Epoch 6/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.2875 - accuracy: 0.8580 - val_loss: 3.1557 - val_accuracy: 0.3711\n",
      "Epoch 7/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.2922 - accuracy: 0.8533 - val_loss: 3.2416 - val_accuracy: 0.3918\n",
      "Epoch 8/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.2581 - accuracy: 0.8707 - val_loss: 3.2466 - val_accuracy: 0.3711\n",
      "Epoch 9/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.2373 - accuracy: 0.9018 - val_loss: 3.2141 - val_accuracy: 0.3814\n",
      "Epoch 10/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.2268 - accuracy: 0.9157 - val_loss: 3.4325 - val_accuracy: 0.3608\n",
      "Epoch 11/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.2167 - accuracy: 0.9319 - val_loss: 3.4474 - val_accuracy: 0.4330\n",
      "Epoch 12/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.2077 - accuracy: 0.9249 - val_loss: 3.3549 - val_accuracy: 0.3814\n",
      "Epoch 13/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.1933 - accuracy: 0.9388 - val_loss: 3.4593 - val_accuracy: 0.4227\n",
      "Epoch 14/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.1731 - accuracy: 0.9353 - val_loss: 3.3237 - val_accuracy: 0.4021\n",
      "Epoch 15/100\n",
      "866/866 [==============================] - 3s 3ms/step - loss: 0.1506 - accuracy: 0.9457 - val_loss: 3.4951 - val_accuracy: 0.4330\n",
      "Epoch 16/100\n",
      "866/866 [==============================] - 3s 3ms/step - loss: 0.1288 - accuracy: 0.9573 - val_loss: 3.6544 - val_accuracy: 0.3814\n",
      "Epoch 17/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.1290 - accuracy: 0.9665 - val_loss: 3.5717 - val_accuracy: 0.3711\n",
      "Epoch 18/100\n",
      "866/866 [==============================] - 3s 3ms/step - loss: 0.1146 - accuracy: 0.9734 - val_loss: 3.5994 - val_accuracy: 0.3814\n",
      "Epoch 19/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.1090 - accuracy: 0.9654 - val_loss: 3.7549 - val_accuracy: 0.3711\n",
      "Epoch 20/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.1063 - accuracy: 0.9711 - val_loss: 3.6280 - val_accuracy: 0.3814\n",
      "Epoch 21/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.1005 - accuracy: 0.9781 - val_loss: 3.8866 - val_accuracy: 0.3711\n",
      "Epoch 22/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0969 - accuracy: 0.9711 - val_loss: 3.7503 - val_accuracy: 0.3505\n",
      "Epoch 23/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0952 - accuracy: 0.9769 - val_loss: 3.7747 - val_accuracy: 0.3918\n",
      "Epoch 24/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0928 - accuracy: 0.9711 - val_loss: 3.8336 - val_accuracy: 0.3918\n",
      "Epoch 25/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0920 - accuracy: 0.9781 - val_loss: 3.7937 - val_accuracy: 0.3918\n",
      "Epoch 26/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0910 - accuracy: 0.9781 - val_loss: 3.9000 - val_accuracy: 0.4021\n",
      "Epoch 27/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0900 - accuracy: 0.9769 - val_loss: 3.8597 - val_accuracy: 0.3918\n",
      "Epoch 28/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0901 - accuracy: 0.9792 - val_loss: 3.9559 - val_accuracy: 0.4021\n",
      "Epoch 29/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0885 - accuracy: 0.9792 - val_loss: 4.0877 - val_accuracy: 0.4021\n",
      "Epoch 30/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0884 - accuracy: 0.9700 - val_loss: 4.1416 - val_accuracy: 0.4021\n",
      "Epoch 31/100\n",
      "866/866 [==============================] - 3s 3ms/step - loss: 0.0867 - accuracy: 0.9700 - val_loss: 4.1623 - val_accuracy: 0.3814\n",
      "Epoch 32/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0840 - accuracy: 0.9711 - val_loss: 4.2185 - val_accuracy: 0.4021\n",
      "Epoch 33/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0833 - accuracy: 0.9677 - val_loss: 4.1273 - val_accuracy: 0.3402\n",
      "Epoch 34/100\n",
      "866/866 [==============================] - 3s 3ms/step - loss: 0.0877 - accuracy: 0.9746 - val_loss: 4.1669 - val_accuracy: 0.4124\n",
      "Epoch 35/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0870 - accuracy: 0.9746 - val_loss: 4.0943 - val_accuracy: 0.3814\n",
      "Epoch 36/100\n",
      "866/866 [==============================] - 3s 3ms/step - loss: 0.0858 - accuracy: 0.9688 - val_loss: 4.3453 - val_accuracy: 0.3608\n",
      "Epoch 37/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0841 - accuracy: 0.9781 - val_loss: 4.2996 - val_accuracy: 0.3608\n",
      "Epoch 38/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0803 - accuracy: 0.9792 - val_loss: 4.3553 - val_accuracy: 0.3608\n",
      "Epoch 39/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0778 - accuracy: 0.9758 - val_loss: 4.4959 - val_accuracy: 0.3711\n",
      "Epoch 40/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0764 - accuracy: 0.9804 - val_loss: 4.4006 - val_accuracy: 0.3814\n",
      "Epoch 41/100\n",
      "866/866 [==============================] - 3s 3ms/step - loss: 0.0749 - accuracy: 0.9804 - val_loss: 4.4806 - val_accuracy: 0.4021\n",
      "Epoch 42/100\n",
      "866/866 [==============================] - 3s 3ms/step - loss: 0.0745 - accuracy: 0.9804 - val_loss: 4.4791 - val_accuracy: 0.3918\n",
      "Epoch 43/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0742 - accuracy: 0.9804 - val_loss: 4.5361 - val_accuracy: 0.3711\n",
      "Epoch 44/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0740 - accuracy: 0.9804 - val_loss: 4.4316 - val_accuracy: 0.3918\n",
      "Epoch 45/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0740 - accuracy: 0.9804 - val_loss: 4.4920 - val_accuracy: 0.3814\n",
      "Epoch 46/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0747 - accuracy: 0.9804 - val_loss: 4.5391 - val_accuracy: 0.4124\n",
      "Epoch 47/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0734 - accuracy: 0.9804 - val_loss: 4.5199 - val_accuracy: 0.4021\n",
      "Epoch 48/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0732 - accuracy: 0.9804 - val_loss: 4.5527 - val_accuracy: 0.4021\n",
      "Epoch 49/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0729 - accuracy: 0.9815 - val_loss: 4.7345 - val_accuracy: 0.4433\n",
      "Epoch 50/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0888 - accuracy: 0.9734 - val_loss: 4.4874 - val_accuracy: 0.4021\n",
      "Epoch 51/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0860 - accuracy: 0.9781 - val_loss: 4.3252 - val_accuracy: 0.4330\n",
      "Epoch 52/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0993 - accuracy: 0.9723 - val_loss: 4.4659 - val_accuracy: 0.4124\n",
      "Epoch 53/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.1057 - accuracy: 0.9677 - val_loss: 4.3114 - val_accuracy: 0.3608\n",
      "Epoch 54/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.1436 - accuracy: 0.9584 - val_loss: 4.2939 - val_accuracy: 0.4124\n",
      "Epoch 55/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.1575 - accuracy: 0.9550 - val_loss: 3.8770 - val_accuracy: 0.3918\n",
      "Epoch 56/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.1435 - accuracy: 0.9550 - val_loss: 3.8216 - val_accuracy: 0.4124\n",
      "Epoch 57/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.1393 - accuracy: 0.9527 - val_loss: 4.0638 - val_accuracy: 0.3918\n",
      "Epoch 58/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.1069 - accuracy: 0.9688 - val_loss: 3.9049 - val_accuracy: 0.3608\n",
      "Epoch 59/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0927 - accuracy: 0.9758 - val_loss: 4.2697 - val_accuracy: 0.4433\n",
      "Epoch 60/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0878 - accuracy: 0.9746 - val_loss: 3.8504 - val_accuracy: 0.4124\n",
      "Epoch 61/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.1021 - accuracy: 0.9746 - val_loss: 4.5131 - val_accuracy: 0.4330\n",
      "Epoch 62/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0777 - accuracy: 0.9792 - val_loss: 3.9556 - val_accuracy: 0.4124\n",
      "Epoch 63/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0680 - accuracy: 0.9827 - val_loss: 4.1687 - val_accuracy: 0.3918\n",
      "Epoch 64/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0642 - accuracy: 0.9861 - val_loss: 4.2859 - val_accuracy: 0.3711\n",
      "Epoch 65/100\n",
      "866/866 [==============================] - 3s 3ms/step - loss: 0.0615 - accuracy: 0.9861 - val_loss: 4.0800 - val_accuracy: 0.4021\n",
      "Epoch 66/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0619 - accuracy: 0.9850 - val_loss: 4.1279 - val_accuracy: 0.3918\n",
      "Epoch 67/100\n",
      "866/866 [==============================] - 3s 3ms/step - loss: 0.0581 - accuracy: 0.9861 - val_loss: 4.2555 - val_accuracy: 0.3608\n",
      "Epoch 68/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0571 - accuracy: 0.9861 - val_loss: 4.2951 - val_accuracy: 0.3608\n",
      "Epoch 69/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0565 - accuracy: 0.9861 - val_loss: 4.3392 - val_accuracy: 0.3711\n",
      "Epoch 70/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0560 - accuracy: 0.9873 - val_loss: 4.3669 - val_accuracy: 0.3711\n",
      "Epoch 71/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0561 - accuracy: 0.9861 - val_loss: 4.3145 - val_accuracy: 0.3711\n",
      "Epoch 72/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0556 - accuracy: 0.9873 - val_loss: 4.3193 - val_accuracy: 0.3608\n",
      "Epoch 73/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0556 - accuracy: 0.9861 - val_loss: 4.3578 - val_accuracy: 0.3711\n",
      "Epoch 74/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0552 - accuracy: 0.9873 - val_loss: 4.3508 - val_accuracy: 0.3608\n",
      "Epoch 75/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0550 - accuracy: 0.9873 - val_loss: 4.4173 - val_accuracy: 0.3711\n",
      "Epoch 76/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0551 - accuracy: 0.9861 - val_loss: 4.4362 - val_accuracy: 0.3608\n",
      "Epoch 77/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0548 - accuracy: 0.9861 - val_loss: 4.3094 - val_accuracy: 0.3814\n",
      "Epoch 78/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0547 - accuracy: 0.9873 - val_loss: 4.3101 - val_accuracy: 0.3814\n",
      "Epoch 79/100\n",
      "866/866 [==============================] - 3s 3ms/step - loss: 0.0546 - accuracy: 0.9873 - val_loss: 4.3483 - val_accuracy: 0.3918\n",
      "Epoch 80/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0544 - accuracy: 0.9873 - val_loss: 4.3970 - val_accuracy: 0.3711\n",
      "Epoch 81/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0543 - accuracy: 0.9873 - val_loss: 4.4718 - val_accuracy: 0.3608\n",
      "Epoch 82/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0542 - accuracy: 0.9873 - val_loss: 4.4892 - val_accuracy: 0.3711\n",
      "Epoch 83/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0541 - accuracy: 0.9873 - val_loss: 4.5244 - val_accuracy: 0.3608\n",
      "Epoch 84/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0539 - accuracy: 0.9873 - val_loss: 4.5377 - val_accuracy: 0.3608\n",
      "Epoch 85/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0538 - accuracy: 0.9873 - val_loss: 4.5426 - val_accuracy: 0.3608\n",
      "Epoch 86/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0536 - accuracy: 0.9873 - val_loss: 4.5711 - val_accuracy: 0.3608\n",
      "Epoch 87/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0537 - accuracy: 0.9873 - val_loss: 4.5498 - val_accuracy: 0.3918\n",
      "Epoch 88/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0537 - accuracy: 0.9861 - val_loss: 4.5817 - val_accuracy: 0.3608\n",
      "Epoch 89/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0533 - accuracy: 0.9861 - val_loss: 4.5513 - val_accuracy: 0.3608\n",
      "Epoch 90/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0532 - accuracy: 0.9873 - val_loss: 4.5025 - val_accuracy: 0.3814\n",
      "Epoch 91/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0531 - accuracy: 0.9873 - val_loss: 4.5048 - val_accuracy: 0.3918\n",
      "Epoch 92/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0530 - accuracy: 0.9873 - val_loss: 4.5247 - val_accuracy: 0.3918\n",
      "Epoch 93/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0528 - accuracy: 0.9873 - val_loss: 4.5475 - val_accuracy: 0.3814\n",
      "Epoch 94/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0527 - accuracy: 0.9873 - val_loss: 4.5800 - val_accuracy: 0.3814\n",
      "Epoch 95/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0526 - accuracy: 0.9873 - val_loss: 4.5980 - val_accuracy: 0.3711\n",
      "Epoch 96/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0525 - accuracy: 0.9873 - val_loss: 4.6100 - val_accuracy: 0.3711\n",
      "Epoch 97/100\n",
      "866/866 [==============================] - 3s 3ms/step - loss: 0.0524 - accuracy: 0.9873 - val_loss: 4.6299 - val_accuracy: 0.3711\n",
      "Epoch 98/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0523 - accuracy: 0.9873 - val_loss: 4.6311 - val_accuracy: 0.3814\n",
      "Epoch 99/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0522 - accuracy: 0.9873 - val_loss: 4.6422 - val_accuracy: 0.3814\n",
      "Epoch 100/100\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 0.0521 - accuracy: 0.9873 - val_loss: 4.6497 - val_accuracy: 0.3814\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.callbacks.History at 0x159369be0>"
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=100, batch_size=128)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Word2Vec + MLP got a validation accuracy of 0.5258 - still not very good. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Grid Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "\n",
    "def create_model(learn_rate=0.1):\n",
    "    # create model\n",
    "    embedding_size = 100  #word2vec has a 100 dimension space\n",
    "    embedding_layer = Embedding(input_dim = num_words, #input_dim: size of the vocabulary\n",
    "                    output_dim = embedding_size,  #dimension of the dense embedding\n",
    "                    embeddings_initializer = Constant(embedding_matrix),\n",
    "                    input_length = 500, #length of input sequences\n",
    "                    trainable = False)\n",
    "\n",
    "    sequence_input  = Input(shape=(500,), dtype='int32')\n",
    "    embedded_sequences = embedding_layer(sequence_input)\n",
    "    x = Conv1D(128, 10, activation='relu')(embedded_sequences)\n",
    "    x = MaxPooling1D(5)(x)\n",
    "    x = Flatten()(x)\n",
    "    x = Dense(128, activation='relu')(x)\n",
    "    preds = Dense(6, activation='softmax')(x)\n",
    "    \n",
    "    opt=Adam(learning_rate=learn_rate)\n",
    "\n",
    "    model = Model(sequence_input, preds)\n",
    "    \n",
    "    \n",
    "    #Compile model\n",
    "    model.compile(loss='categorical_crossentropy', optimizer=opt,\n",
    "              metrics=['accuracy'])\n",
    "    return model\n",
    "\n",
    "model = KerasClassifier(build_fn=create_model, verbose=0, epochs=20, batch_size=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: 0.419186 using {'learn_rate': 0.001}\n",
      "0.419186 (0.010495) with: {'learn_rate': 0.001}\n",
      "0.297982 (0.091300) with: {'learn_rate': 0.005}\n",
      "0.168589 (0.005812) with: {'learn_rate': 0.01}\n",
      "0.179018 (0.020777) with: {'learn_rate': 0.05}\n"
     ]
    }
   ],
   "source": [
    "param_grid = {\n",
    "    'learn_rate':[0.001,0.005,0.01,0.05]\n",
    "}\n",
    "\n",
    "grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1, cv=3)\n",
    "grid_result = grid.fit(X_train, y_train)\n",
    "\n",
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print(\"%f (%f) with: %r\" % (mean, stdev, param))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Try Count Vectorizer and TFIDF in Deep Learning Models with Grid Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = salary_df['combined_text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "\n",
    "count_vectorizer = CountVectorizer(max_features = 300, max_df = 0.5, min_df = 0.03, ngram_range=(1,3), stop_words='english')\n",
    "word_count_vector = count_vectorizer.fit_transform(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TfidfTransformer(norm='l2', smooth_idf=True, sublinear_tf=False, use_idf=True)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfidf_transformer=TfidfTransformer(smooth_idf=True, use_idf=True)\n",
    "tfidf_transformer.fit(word_count_vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_idf = pd.DataFrame(tfidf_transformer.idf_, index=count_vectorizer.get_feature_names(), columns=['idf_weights'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>idf_weights</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>behavior</th>\n",
       "      <td>3.580632</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>clinical</th>\n",
       "      <td>3.514382</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>center</th>\n",
       "      <td>3.286124</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>social media</th>\n",
       "      <td>3.236362</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>intelligence</th>\n",
       "      <td>3.170611</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>friday</th>\n",
       "      <td>1.733364</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>support</th>\n",
       "      <td>1.733364</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>000</th>\n",
       "      <td>1.718359</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>analysis</th>\n",
       "      <td>1.718359</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>new</th>\n",
       "      <td>1.707776</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>300 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              idf_weights\n",
       "behavior         3.580632\n",
       "clinical         3.514382\n",
       "center           3.286124\n",
       "social media     3.236362\n",
       "intelligence     3.170611\n",
       "...                   ...\n",
       "friday           1.733364\n",
       "support          1.733364\n",
       "000              1.718359\n",
       "analysis         1.718359\n",
       "new              1.707776\n",
       "\n",
       "[300 rows x 1 columns]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_idf.sort_values(by=['idf_weights'], ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfidf_mat = tfidf_transformer.transform(word_count_vector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "scipy.sparse.csr.csr_matrix"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(tfidf_mat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<963x300 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 73819 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Each job description has 300 features, each feature is one word\n",
    "tfidf_mat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Encoding to categorical classes\n",
    "y = to_categorical(target_class.values, num_classes=6, dtype=\"float32\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Train - Test split\n",
    "X_train, X_test, y_train, y_test = train_test_split(tfidf_mat, y, test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /Applications/anaconda3/lib/python3.7/site-packages/tensorflow/python/ops/resource_variable_ops.py:435: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "Model: \"sequential_1\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_1 (Dense)              (None, 100)               30100     \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 64)                6464      \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 64)                0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 6)                 390       \n",
      "=================================================================\n",
      "Total params: 36,954\n",
      "Trainable params: 36,954\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(Dense(100, input_shape=(300,), activation='relu'))\n",
    "model.add(Dense(64, activation='relu'))\n",
    "model.add(Dropout(0.5))\n",
    "#model.add(Flatten())   #add Flatten layer \n",
    "model.add(Dense(6, activation='softmax'))\n",
    "\n",
    "opt = optimizers.Adam(learning_rate=0.001)\n",
    "#Compile model\n",
    "model.compile(loss='categorical_crossentropy', optimizer=opt,\n",
    "          metrics=['accuracy'])\n",
    "\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /Applications/anaconda3/lib/python3.7/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "Train on 770 samples, validate on 193 samples\n",
      "Epoch 1/40\n",
      "770/770 [==============================] - 0s 322us/step - loss: 1.7866 - accuracy: 0.1935 - val_loss: 1.7682 - val_accuracy: 0.3264\n",
      "Epoch 2/40\n",
      "770/770 [==============================] - 0s 41us/step - loss: 1.7556 - accuracy: 0.2818 - val_loss: 1.7458 - val_accuracy: 0.3575\n",
      "Epoch 3/40\n",
      "770/770 [==============================] - 0s 55us/step - loss: 1.7263 - accuracy: 0.3312 - val_loss: 1.7155 - val_accuracy: 0.4041\n",
      "Epoch 4/40\n",
      "770/770 [==============================] - 0s 41us/step - loss: 1.6879 - accuracy: 0.3649 - val_loss: 1.6736 - val_accuracy: 0.4041\n",
      "Epoch 5/40\n",
      "770/770 [==============================] - 0s 36us/step - loss: 1.6424 - accuracy: 0.3740 - val_loss: 1.6212 - val_accuracy: 0.3679\n",
      "Epoch 6/40\n",
      "770/770 [==============================] - 0s 45us/step - loss: 1.5879 - accuracy: 0.4091 - val_loss: 1.5618 - val_accuracy: 0.3938\n",
      "Epoch 7/40\n",
      "770/770 [==============================] - 0s 48us/step - loss: 1.5297 - accuracy: 0.4117 - val_loss: 1.5070 - val_accuracy: 0.4041\n",
      "Epoch 8/40\n",
      "770/770 [==============================] - 0s 37us/step - loss: 1.4679 - accuracy: 0.4519 - val_loss: 1.4465 - val_accuracy: 0.4249\n",
      "Epoch 9/40\n",
      "770/770 [==============================] - 0s 37us/step - loss: 1.4274 - accuracy: 0.4429 - val_loss: 1.4090 - val_accuracy: 0.4145\n",
      "Epoch 10/40\n",
      "770/770 [==============================] - 0s 53us/step - loss: 1.3507 - accuracy: 0.5091 - val_loss: 1.3799 - val_accuracy: 0.4352\n",
      "Epoch 11/40\n",
      "770/770 [==============================] - 0s 53us/step - loss: 1.3218 - accuracy: 0.5078 - val_loss: 1.3594 - val_accuracy: 0.4249\n",
      "Epoch 12/40\n",
      "770/770 [==============================] - 0s 43us/step - loss: 1.2743 - accuracy: 0.5351 - val_loss: 1.3344 - val_accuracy: 0.4352\n",
      "Epoch 13/40\n",
      "770/770 [==============================] - 0s 43us/step - loss: 1.2425 - accuracy: 0.5481 - val_loss: 1.3202 - val_accuracy: 0.4301\n",
      "Epoch 14/40\n",
      "770/770 [==============================] - 0s 77us/step - loss: 1.2164 - accuracy: 0.5831 - val_loss: 1.2851 - val_accuracy: 0.4456\n",
      "Epoch 15/40\n",
      "770/770 [==============================] - 0s 62us/step - loss: 1.1844 - accuracy: 0.5740 - val_loss: 1.2791 - val_accuracy: 0.4352\n",
      "Epoch 16/40\n",
      "770/770 [==============================] - 0s 59us/step - loss: 1.1405 - accuracy: 0.5844 - val_loss: 1.2607 - val_accuracy: 0.4715\n",
      "Epoch 17/40\n",
      "770/770 [==============================] - 0s 61us/step - loss: 1.0767 - accuracy: 0.6351 - val_loss: 1.2392 - val_accuracy: 0.4715\n",
      "Epoch 18/40\n",
      "770/770 [==============================] - 0s 59us/step - loss: 1.0332 - accuracy: 0.6377 - val_loss: 1.2451 - val_accuracy: 0.4715\n",
      "Epoch 19/40\n",
      "770/770 [==============================] - 0s 44us/step - loss: 1.0261 - accuracy: 0.6364 - val_loss: 1.2361 - val_accuracy: 0.4922\n",
      "Epoch 20/40\n",
      "770/770 [==============================] - 0s 43us/step - loss: 1.0100 - accuracy: 0.6338 - val_loss: 1.2308 - val_accuracy: 0.4715\n",
      "Epoch 21/40\n",
      "770/770 [==============================] - 0s 57us/step - loss: 0.9730 - accuracy: 0.6688 - val_loss: 1.2233 - val_accuracy: 0.4870\n",
      "Epoch 22/40\n",
      "770/770 [==============================] - 0s 57us/step - loss: 0.9142 - accuracy: 0.7130 - val_loss: 1.2185 - val_accuracy: 0.4819\n",
      "Epoch 23/40\n",
      "770/770 [==============================] - 0s 52us/step - loss: 0.9026 - accuracy: 0.6987 - val_loss: 1.2245 - val_accuracy: 0.5181\n",
      "Epoch 24/40\n",
      "770/770 [==============================] - 0s 56us/step - loss: 0.8732 - accuracy: 0.7156 - val_loss: 1.2184 - val_accuracy: 0.4974\n",
      "Epoch 25/40\n",
      "770/770 [==============================] - 0s 48us/step - loss: 0.8727 - accuracy: 0.7091 - val_loss: 1.2277 - val_accuracy: 0.4870\n",
      "Epoch 26/40\n",
      "770/770 [==============================] - 0s 46us/step - loss: 0.8309 - accuracy: 0.7156 - val_loss: 1.2313 - val_accuracy: 0.4922\n",
      "Epoch 27/40\n",
      "770/770 [==============================] - 0s 41us/step - loss: 0.8144 - accuracy: 0.7532 - val_loss: 1.2187 - val_accuracy: 0.4974\n",
      "Epoch 28/40\n",
      "770/770 [==============================] - 0s 49us/step - loss: 0.7848 - accuracy: 0.7325 - val_loss: 1.2174 - val_accuracy: 0.5078\n",
      "Epoch 29/40\n",
      "770/770 [==============================] - 0s 49us/step - loss: 0.7600 - accuracy: 0.7740 - val_loss: 1.2249 - val_accuracy: 0.5337\n",
      "Epoch 30/40\n",
      "770/770 [==============================] - 0s 40us/step - loss: 0.7007 - accuracy: 0.7987 - val_loss: 1.2416 - val_accuracy: 0.5130\n",
      "Epoch 31/40\n",
      "770/770 [==============================] - 0s 41us/step - loss: 0.7181 - accuracy: 0.7597 - val_loss: 1.2425 - val_accuracy: 0.5181\n",
      "Epoch 32/40\n",
      "770/770 [==============================] - 0s 41us/step - loss: 0.6792 - accuracy: 0.7922 - val_loss: 1.2563 - val_accuracy: 0.5181\n",
      "Epoch 33/40\n",
      "770/770 [==============================] - 0s 46us/step - loss: 0.6609 - accuracy: 0.8039 - val_loss: 1.2553 - val_accuracy: 0.5130\n",
      "Epoch 34/40\n",
      "770/770 [==============================] - 0s 54us/step - loss: 0.6592 - accuracy: 0.7857 - val_loss: 1.2703 - val_accuracy: 0.5078\n",
      "Epoch 35/40\n",
      "770/770 [==============================] - 0s 43us/step - loss: 0.6144 - accuracy: 0.8299 - val_loss: 1.2655 - val_accuracy: 0.5233\n",
      "Epoch 36/40\n",
      "770/770 [==============================] - 0s 43us/step - loss: 0.5927 - accuracy: 0.8221 - val_loss: 1.2717 - val_accuracy: 0.5285\n",
      "Epoch 37/40\n",
      "770/770 [==============================] - 0s 45us/step - loss: 0.5731 - accuracy: 0.8338 - val_loss: 1.2700 - val_accuracy: 0.5181\n",
      "Epoch 38/40\n",
      "770/770 [==============================] - 0s 46us/step - loss: 0.5560 - accuracy: 0.8351 - val_loss: 1.2801 - val_accuracy: 0.5181\n",
      "Epoch 00038: early stopping\n"
     ]
    }
   ],
   "source": [
    "es = EarlyStopping(monitor='val_loss', patience = 10, mode='min', verbose=1)\n",
    "\n",
    "history = model.fit(X_train, y_train,  validation_data=(X_test, y_test), epochs=40, batch_size=64, callbacks=[es])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train: 0.904, Test: 0.518\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEJCAYAAACE39xMAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3dd3hUVfrA8e+bTkINIbRQIr33omBBQEARxK7YC/pTLGtZce267trbKiqWtYGIsCgKCoKAWOi9E3ogJKEFSE/m/P44ExjCJJmEhCl5P88zTzJzz733nTsz75w599xzxBiDUkop/xfk7QCUUkqVD03oSikVIDShK6VUgNCErpRSAUITulJKBQhN6EopFSA0oZeCiEwUkcmlXGehiLxWUTFVNBGJExEjIr29HYsKPCLS2vn+au/tWErLF2MPqITuPLjF3T47zV3cBdxRynUuBp47zf36tIr40hKRwc7XrGop1pklIvkicm55xhJoRCTCeWyHejsWVb5CvB1AOavv8v9Q4KNCj2W6W0lEQo0xuSVt3BiTVtqAjDEHS7uOKj0RaQKcDfwH+6W7wMvxBAFijMn3ZhyqkjHGBOQNuNI+vVMebw0Y4CpgPpCFTQB1gW+APUAGsBYYWWjdicBkl/sLgTeBV4GDwD7gX9gPsmuZ11zu7wMeAz4FjgK7gfsL7act8IcztvXAQCAPuLaY53sOMBs4AKQBvwE9XJZHOJ/3rcBU53PcClxdaDtnAyud+14KDHOu17uI/U50Lne91XMu6wD8DBwDkoGvgDou63YB5jmPw1FgBdDX5TVyvX1Qwuv9HDAeaAGkA9XdlLkDWAdkO1+HcS7LagHjnHFmOctd7lx2N7C/0LYGO+Oq6loGGO58zfKA5iW9LsXtG6jhfJ2GFip/qbNcrTK+F/YVOrYbizmu0cAnQCpwBPgV6Oyy3JPPTRAwBkhwHvtdwLOFPo/Dgbku27ighNc7CHgC2I6tqK3G5b3sst1rgL9cjmu/QtvpDyxxxpUEvAKEVmTsFZr3vLXjCn9iJSf0rcBlQDzQAGgK/A3oDJwF3Ov8UPZ1WdddQk8DnsQmkpFAPjCiUJnCCT0VmwCaA4844+nqXB4CbMEmwo7YBLcEcFB8Qr8IuN75/NoAHzr3U8O5vCCh7wKude77Decbvb6zTA3sF9N4oB22uWgLxSf0GsAyYCxQz3kLAhoBh4AXgFbO4/ozsMBl3S3Af53LmwNXAD2AYOA6537Pcm7zlARd6EO3CxjicszvLlTmAecH7n6gpXM/DzuXBTuP8Wrsl+dZwCXApc7lnib0bOB37JdiKyDKg9elpH1/Dvyv0L6nAl+fxnuhkTP2G5zHNqaI7QQDi4HvgO7Y9/grzvdIHWeZppT8uXkT++VyE9AM6APcVejzuA77fmsJfI39nEQU8xxfd65zEfYzfDM2sQ8otN2d2C/HguNwDIh1iT0L+6uuNTYfpAIvVmTsFZr3vLHTM/LESk7o93qwje+Ad13uu0vocwuts6DQOu4S+n8LrbMbeMT5/3Agp+BN53zsQmfMRSZ0N7EHOT94VzrvFyT0Z1zKhAO5LmXud76hI1zK3EExCd3dc3Q+9gowvdBj9Zzb6giI88N0TRHbPClhlvBchziPa7Dz/j3AkkLHIhVnzcrN+pdik1CzIpZ7mtAN0K6Ur0tJ++7rfD8UJNBY5/0B5fBeGFrCehc71wsr9PhGCv2qLOpzg63h5wK3FFG24PN4s8tjzZyPdS9inZrOY1D4l84HOL/8XLb7sMvyYGAH8KTzfsGXgusv6ruxX/yhFRF7Rd8C6qRoKS11vSMiISLyjIisEZGDInIMW1NqXMJ2Vhe6vxf7oSvrOq2BHcaYFJfli0rYHiJSX0Q+FpEtIpKG/Xlck1PjP75vY0w29gNbsO82wApjTJZL+b9K2ncRugEDReRYwQ37sxVs8jLY2s9XIvKLiDwuIi3KuK/bsTXWgvbqiUBHEenkvB8HxABzili/C7DTGLO1jPsvkGmMWef6gAevS7H7Nsb8jj1uNzofuhHbvFHUcynNe6Ek3XD+aiv0OjbHJi5PPjftsb86i4zXyfUzsdf5t6jPUQdswp1bKK5bC+Jycfz963x/LME2aYJ9v//pfC8W+B2ogq31V0TsFSrQToqWRnqh+09gfy4+iP3WTsd+g4eXsJ3CJ1MNtiZQ2nUKvlzFeb+0JgCR2Fr2Lk78/A8r5b7LSxC2pvYPN8v2ARhjHnf2PLoY+9P5ORG51Rgz3tOdiEgdbDt/sIjc57IoGPvr4j5Kfl4lLXe4KRPqplyGm8dKel08OeafALdhm8huxf7CK+494ul7oSRBQCK2nbmwgg4CJX1uPH1Pub4vC55bURXOgscH43wvucjxcH9Q/GfNUDGxV6jKXEMvrC8w1RgzwRizCtiGbRM70zYATZ2JqkDP4lYQEcG27b1ljPnJWUvMovS1hPVAZxFx/RLzpP95Dqd+iS3HtsNvN8YkFLodKyhkjNlkjHnTGDME23Z/u8s2cbPdwm7CtpN2wrbjFtxGAyNFJALbpLUf94mpINYmIlK4dlcgFajp3FaBziXE5enrUtK+wbajt3B+YbUBPjvNfebhWcVjOfb8Urab1zDVWaakz80a5/6KOvZlUbDNRm7i2lWo7PH3r4gEY88FbHA+tB7o4zxmBfpi2+J3VFDsFUoT+gmbgUEicraIFJxAaeCFOKZja1Wfi0hHEekDvMSJHgmncNbWtgA3OS926IU9OeO2m2YxvsDWPD8WkbYiMgT4uwfr7QB6i0gTEYlxfkDexnYZnSAiPUTkLBG5SEQ+EZEwEakhIu+IyPnO9c7Bnkxc77JNgKEiUkdEoorY9+3At8aYta437MnWcGxPFQe299HfReQ+EWkhIl1E5EHnNmYAq4D/icgAEYkXkUEu/bT/xH7B/FtEmovINcCdJR0UD1+XkvaNMWY/9tfOa8BsN0mrVPs0xuRha94DRKSeiNQsYnMzsEl9mvO1ayoi54jIP53bhRI+N8Z22x0LvC4iN4lIMxHpLSKjSjp+xTzHg9j319su2+wiIveKyG2Fit8vIpeJSGvgXewX2zjnsv9gm2jedh6r4diT+G8aY3IrIvYK542G+zNxo+STou0LPR4DTONEF7sXsT91f3Yp4+6kaOGTgcWWwf5EHF1oncJl2mKTSDa2NnEJ9mf/8GKebzfseYEs7Af6Gmzb6xjncrcnwgrHg62hrHLuewX2zH9JJ0XbYntDZHByt8XW2ER02LlsI/aDGIxtEpjIiSaBPcD7QJTLdv/pfC0cuOm2iO2eZ3D2EHKzfBLwq8v9u4FN2OSc5LpNoDa2K+l+bPJbC1zmsvwq5/HMxH7p3oKbboulfV082bezzEXO/V3t7rmWYZ9XYHt65VJ8t8UawHvYtuEc5+s1HmhSis9NMPAU9ku6YBtPF/V5xIOTttjK6EPO91QOkALMxNkt0WW712Lfm9nYykL/Qtvp7zxWBV1ZC3dbLPfYK/ImziCUD3PWhhZi3zjrSiqvAo+I3Ixtm25o7MlsVQxnjXwD0MHYX2yVQmU+KeqzROQqbB/uBOxPwreAxZrMKx8RicT27x4DvK/JXBVH29B9Uw1sn9qN2HbtFdihDFTl8zT29U/EnktRqkja5KKUUgFCa+hKKRUgvNaGHhMTY5o2beqt3SullF9atmzZfmNMHXfLvJbQmzZtytKlS0suqJRS6jgR2VnUMm1yUUqpAKEJXSmlAoQmdKWUChA+dWFRbm4uiYmJZGVllVzYj0VERBAXF0doqLsB+5RSqmx8KqEnJiZSrVo1mjZtyskDoAUOYwwHDhwgMTGR+Ph4b4ejlAogPtXkkpWVRe3atQM2mQOICLVr1w74XyFKqTPPpxI6ENDJvEBleI5KqTPPp5pclFLK36UcySItM5f0nHwysvM4lp1HRk6+828ex7Lz6d86lk6NihqGvuw0obs4fPgwEyZM4J577inVehdffDETJkygZs3yf4GUUr4vL9/BzHXJfPrHdpbtPFRi+dhq4ZrQK9rhw4cZO3bsKQk9Pz+f4OCiZ+uaMWNGRYemlDoDxkxZzcZ9R7mgVR0ubB1L+wY1CAoquok0LSOXiUt28fmfO9iblkXj6EgeG9yahrWqUDU8mMiwEKqGhxAZFmz/hocQGRpc7DZPhyZ0F2PGjGHr1q107tyZ0NBQqlatSv369Vm5ciXr16/nsssuY/fu3WRlZfHAAw8wapSdiapgGINjx44xZMgQ+vbty59//knDhg35/vvvqVKlipefmVKqJLPXJzNxyW6a1I7k7TlbeGv2FmKqhtOvVR36tY6lb4sYqkfYrsZbU4/x2R87mLwskczcfM4+qzbPDW/Pha1jCa6gZO0Jn03oz/2wjvV7j5TrNts2qM4zl7YrcvlLL73E2rVrWblyJfPmzeOSSy5h7dq1x7sXfvrpp0RHR5OZmUmPHj244oorqF279knb2LJlC19//TUfffQRV199NVOmTOGGG24o1+ehlCpfWbn5PPfjOlrEVmXGA+dyJDOX+ZtTmbsplZnr9vHtskRCgoQeTaMJCwli/uZUwoKDGN65Abf2iadtg+refgqADyd0X9CzZ8+T+oq/8847TJ06FYDdu3ezZcuWUxJ6fHw8nTvbCeG7devGjh07zli8SqmyGTs3gd0HM/n6zt6EBgdRu2o4l3eN4/KuceTlO1i+6zBzN6Uwd2MKaZm5PDigBSN7NaFOtXBvh34Sn03oxdWkz5SoqBMTzc+bN4/Zs2fz119/ERkZyQUXXOC2L3l4+IkXODg4mMzMzFPKKKV8x/b96XwwfxvDOzfg7Ga1T1keEhxEz/hoesZH89jg1l6I0HM+1w/dm6pVq8bRo0fdLktLS6NWrVpERkayceNGFi5ceIajU0qVN2MMz0xbR3hIEE9c3Mbb4Zw2n62he0Pt2rXp06cP7du3p0qVKtStW/f4ssGDB/PBBx/QsWNHWrVqRe/evb0YqVKqPMxct4/fNqfy9NC2xFaP8HY4p82jOUVFZDDwNhAMfGyMeanQ8sbA50BNZ5kxxphi+/J1797dFJ7gYsOGDbRp4//fkp6oTM9VKV+UkZPHgNfnU71KKD/e15eQYP9osBCRZcaY7u6WlfgMRCQYeA8YArQFrhORtoWKPQlMMsZ0Aa4Fxp5eyEop5bmktExe+XkjQ/+zgPGLduJJRfWdOQnsTcvin5e195tkXhJPmlx6AgnGmG0AIjIRGA6sdyljgIJ+OzWAveUZpFLK/23fn07tqmHH+3KXh5W7D/Pp79uZsSYJhzE0jYniialrmbUumZev6Ei9Gu6bURJSjvLxgm1c2S2O7k2jyy0eb/MkoTcEdrvcTwR6FSrzLDBLRO4DooAB7jYkIqOAUQCNGzcubaxKKT/1w6q9PDRpJW0b1GDK3WefVo04L9/Bz+v28env21m+6zDVwkO45Zym3HxOUxrWrML4RTv514yNXPTmfJ4f3p7hnRucNCCeMYanvltHZFgwY4b4dq+V0vLkqLq77Knw75nrgM+MMXHAxcCXInLKto0x44wx3Y0x3evUcTtptVIqwHy5cCf3T1xBXK1IVu0+zAfzt5ZpO5k5+XwwfyvnvTKX0RNWcCA9h2cvbctf/+jPk0Pb0ig6kqAg4cazmzLjgXNpUbcaD36zkv/7ajkHjmUf384Pq5P4a9sBHh3cmpiqvtWP/HR5ktATgUYu9+M4tUnldmASgDHmLyACiCmPAJVS/skYw3/mbOGp79bSv3UsPz1wLkM71uet2VtYuyetVNvKdxjuGb+Ml37aSNOYKD6+qTu/PnwBt/SJp2r4qQ0N8TFRTLrrbB4f0ppfN6Zw0Zu/MXPdPo5m5fLPH9fToWENru8ZeK0EniT0JUALEYkXkTDsSc9phcrsAvoDiEgbbEJPLc9AlVL+w+EwvPDjBl7/ZTOXd2nI+zd0IyI0mBeGt6dWVBgPT1pFdl6+x9t7ZeZG5m5K5YXL2jPhzt4MaFu3xDFTgoOEu85vxg/39aVejQju+nIZw979g9Rj2bxwWXuvjrlSUUpM6MaYPGA0MBPYgO3Nsk5EnheRYc5iDwN3isgq4GvgFuPJaWYfUzDaYlm89dZbZGRklHNESvmf3HwHj0xexad/bOe2PvG8dlUnQp1t5rWiwnjlio5sSj7KG79s9mh7/1ueyIfzt3Fj7ybc2LtJqeNpVa8aU+/pw/39W7DrYAbX92xM5woYutYXeNQPvSL4Yj/0HTt2MHToUNauXVvqdQtGXIyJ8aylydvPVamKkJWbz+gJy5m9IYWHB7Zk9IXN3c7Q9fj/VjNxyW4m3XU2PYrpZbJi1yGuGbeQbo1r8cXtPY9/MZRVytEsakeF+3XtvLh+6HqlqAvX4XMHDhxIbGwskyZNIjs7mxEjRvDcc8+Rnp7O1VdfTWJiIvn5+Tz11FMkJyezd+9e+vXrR0xMDHPnzvX2U1HqjDuSlcsdny9lyY6DvHBZ+2Jr009c0pbfE/bz8KRV/PTAuUS5aQffl5bFqC+XUbd6OGNHdj3tZA4QW83/rwYtju8m9J/GwL415bvNeh1gyEtFLnYdPnfWrFlMnjyZxYsXY4xh2LBh/Pbbb6SmptKgQQOmT58O2DFeatSowRtvvMHcuXM9rqErFUjSs/O4btxCNu07ylvXdGZ454bFlq8aHsLrV3XmmnF/8a8ZG3hxRIeTlmfl5jPqy6VkZOcx/o4+1IoKq8jwA0ZgXB5VAWbNmsWsWbPo0qULXbt2ZePGjWzZsoUOHTowe/ZsHnvsMRYsWECNGjW8HapSXvf6rM2s23uED2/sVmIyL9AzPpo7zz2L8Yt2MW9TyvHHjTE8Onk1a/ak8fa1XWhZt1pFhR1wfLeGXkxN+kwwxvD4449z1113nbJs2bJlzJgxg8cff5yLLrqIp59+2gsRKuUbVuw6xH//3M6NvZvQv03dkldw8dDAlszblMJjU1Yz68HzqREZyth5W/lh1V4eHdSKAW1Lt73KTmvoLlyHzx00aBCffvopx44dA2DPnj2kpKSwd+9eIiMjueGGG3jkkUdYvnz5KesqVVnk5DkYM2UN9apH8PfBrUq9fkRoMG9c3ZkDx3J4etpaflmfzGuzNjGsUwPuuaBZBUQc2Hy3hu4FrsPnDhkyhOuvv56zzz4bgKpVq/LVV1+RkJDAo48+SlBQEKGhobz//vsAjBo1iiFDhlC/fn09KaoqjQ/mb2VT8lE+ubk71co4Rkv7hjW4v38L3vhlMz+t2UeHhjV45cqObnvHqOJpt0UvqUzPVQWmhJSjXPz271zUri7vXt/1tLaVl+/gqg//Ys+hTKaN7lvkoFpKuy0qpUpwNCuXlKPZNKtT1aPyDodhzJQ1RIYH8+yw058uMiQ4iK/v7E1uvqPMNX2lbehKVXp5+Q5u+e8SBr4xn7HzEjwaS3z8op0s3XmIJy9pW24DXEWEBmsyP00+l9D9cMSAUqsMz1H5j/fmbmXZzkN0jKvJKz9vYtSXyziSlVtk+b2HM3npp42c2yKGK7p61kVRnRk+ldAjIiI4cOBAQCc8YwwHDhwgIkLbCJX3Ld91iHd+3cKILg2Zes85PHlJG37dmMKw//zOxn1HTilvxxJfi8PAv0Z00BOXPsan2tDj4uJITEwkNTWwB2qMiIggLi7O22GoSu5Ydh4PTlxJveoRPDe8HSLCHeeeRce4mtw7YTkj3vuTf1/egcu6nKiF/7g6iTkbU3jykjY0io70YvTKHZ9K6KGhocTHx3s7DKUqhWenrSPxUAbf3HX2SdPC9YyPZvp9fRk9YQUPfrOSFbsO8cQlbUnPzuPZaevoFFeDW/vo59QX+VRCV0qdGdNXJzF5WSL3Xdjc7WiHsdUjGH9nL17+aSMf/76dNXvSiK0WQVpmLl/d0cuvRysMZJrQlapkktIy+cfUNXRqVJP7+7coslxocBBPDm1Ll8a1+PvkVSzfdZjR/ZrTpn71ItdR3qUJXalKxOEwPPTNKnLzHbx9TWePhqS9pGN9WtWrxk9rkrjzvLPOQJSqrDShK1WJfLRgG39tO8ArV3SkaUyUx+s1j63KfcXU5pVv8Klui0qpirN2TxqvzdrE4Hb1uKq79rIKRJrQlaoEMnPyefCblURHhfHvy7X/eKDSJhelfNzyXYeYsGgX8TFR9IqPpmNcTcJCPKuLJR7KYNG2g3y3cg8JKcf46vZeOvtPANOErlQF+/dPG9iQdJQHB7Sga+NaHq93NCuX12Zu4ouFO4kMDSY9Jx+A8JAgujauRc/4aHrFR9OlcS2qhAVjjGHb/nQWbz94/LbncCYANaqEMmZIa/q20CkSA5kmdKUq0JIdB/lw/jZCg4XfNqcyqF1dHh3UiuaxxU+rNnPdPp75fh3JR7O4+eymPHxRS3LzzYlkveMA//l1C28bCA0W2tavzp7DWew/lg1ATNVwesVHM+q8s+h1VjQtY6sRpH3HA55H46GLyGDgbSAY+NgY81Kh5W8C/Zx3I4FYY0zN4rbpbjx0pQJJXr6Dof/5naNZeXx3bx8mLNrFuN+2kpmbz1XdGvHgwBbUr1HlpHX2pWXxzLS1zFyXTOt61fj35R3oUkSt/khWLst2HGLR9oOs2HWIBjWrHK+1x8dEaTt5gCpuPPQSE7qIBAObgYFAIrAEuM4Ys76I8vcBXYwxtxW3XU3oKtB98vt2XvhxPR/c0I3B7esBcOBYNu/OTWD8wl2IwC3nNOX/LmhG9YhQxi/aycs/byI338GDA1pyx7nxHvUTV5XL6U5w0RNIMMZsc25sIjAccJvQgeuAZ8oSqFKBIvlIFm/+spnzW9ZhULsTEx3XrhrOM5e247Y+8bz5y2bGLdjGhMW7aBwdybq9R+jbPIYXR7SnSW3P+4grVcCTr/+GwG6X+4nOx04hIk2AeODXIpaPEpGlIrI00EdUVJXbi9M3kJPv4Llh7dw2fTSKjuSNazoz4/5z6dk0mrTMXF6/qhNf3t5Tk7kqM09q6O4a4opqp7kWmGyMyXe30BgzDhgHtsnFowiV8jN/Juxn2qq93N+/RYlXY7apX51PbulxhiJTgc6TGnoi0Mjlfhywt4iy1wJfn25QSvmrnDwHT32/lkbRVbjngmbeDkdVMp4k9CVACxGJF5EwbNKeVriQiLQCagF/lW+ISvmPT//YztbUdJ69tB0RocHeDkdVMiUmdGNMHjAamAlsACYZY9aJyPMiMsyl6HXARBPI88cpVYy9hzN5e/YWBrSpS/82dUteQaly5tGFRcaYGcCMQo89Xej+s+UXllL+54Uf12MwPHNpW2+Hoiop7eSqVDmYvzmVn9buY3S/5jrXpvIaTehKnaas3Hye+X4t8TFROgGE8iody0Wp0/TRb9vYcSCDL27rSXiInghV3qM1dKVOw9o9afxnbgIXd6jHeS3reDscVclpQleqjNIyc7ln/HKiI8N4YXh7b4ejlDa5KFUWxhge/XYVew9n8s1dvaldNdzbISmlNXSlyuKT37cza30yY4a0pluTaG+HoxSgCV2pUlu28yAv/bSRQe3qcnvfeG+Ho9RxmtCVKoUDx7K5d/wKGtSswitXdtJJJJRP0TZ0pTzkcBge/GYlBzNy+N//nUONKqHeDkmpk2gNXSkPvTs3gQVb9vPspe1o37CGt8NR6hSa0JXywB8J+3lz9mZGdGnIdT0blbyCUl6gTS4qIGTk5JFyJJuUo9mkHM066f+s3Hwiw0KICgsmKjzE3sKCiQwPoarzVqdaOLHVwqkVGUZQ0Mnt4slHsnhg4gqa16nKiyPaa7u58lma0JXfMsbw2JTVTF+dRHrOqZNkhQYLsdUiqBIWTEZ2Huk5+aRn55HnKHqE55AgOZ7c61SLoE61cNbsOUxGTj4TR3UlMkw/Msp36btT+a3ftuxn0tJEhrSvR8e4msRWCye2ejix1SKIrRZOzcjQU2rTxhhy8h2kZ9vknp6Tx5HMPFKPZpN6NMtZq7e3xEMZrNh1iKzcfF6+oiPNY6t56Zkq5RlN6MovORyGV2dupGHNKrx1bWePB8USEcJDggkPCSY6KsyjdYwx2syi/IKeFFV+6ae1+1i75wh/G9iywkc41GSu/IUmdOV38vIdvP7LJlrEVmVEl4beDkcpn6EJXfmd/y3fw7bUdB6+qBXBQVp7VqqAJnTlV7Jy83lr9mY6NarJoHY6EbNSrjShK78yftEu9qZl8fdBrbRtW6lCNKErv3EsO4/35ibQp3lt+jSP8XY4SvkcjxK6iAwWkU0ikiAiY4ooc7WIrBeRdSIyoXzDVAo+WbCdg+k5PDqotbdDUconldgPXUSCgfeAgUAisEREphlj1ruUaQE8DvQxxhwSkdiKClhVTgfTc/howTYGtatL50Y1vR2OUj7Jkxp6TyDBGLPNGJMDTASGFypzJ/CeMeYQgDEmpXzDVJXd+/MSyMjJ45GLWnk7FKV8licJvSGw2+V+ovMxVy2BliLyh4gsFJHB7jYkIqNEZKmILE1NTS1bxKrSSUrL5PO/djKiSxwt6url90oVxZOE7q4rQeHRjUKAFsAFwHXAxyJyyu9iY8w4Y0x3Y0z3OnXqlDZWVUm9M2cLxhgeHNDC26Eo5dM8SeiJgOsA0HHAXjdlvjfG5BpjtgObsAleqdOyfX86k5YmMrJXExpFR3o7HKV8micJfQnQQkTiRSQMuBaYVqjMd0A/ABGJwTbBbCvPQFXl43AYXpu5ifCQIO7t19zb4Sjl80rs5WKMyROR0cBMIBj41BizTkSeB5YaY6Y5l10kIuuBfOBRY8yBigxc+b6Ji3cxYfEuBrSpy9CO9TmrTlWP1ktKy2Ty0kS+WbqbxEOZ3H9hc+pUC6/gaJXyf2JM0YP9V6Tu3bubpUuXemXfquKt2HWIqz/8i1qRYaQey8YYaFu/OkM71efSjg1OaT7JzXcwZ0My3yzZzfzNqTgMnNOsNtf0aMTQjg10zBalnERkmTGmu9tlmtBVeTuUnsMl7ywgKEiYft+5ZObmM31NEj+u3suKXYcB6BRXg6EdG9C1SS1mrdvHlOWJ7D+WQ93q4VzVrRFXd29E49raZq5UYZrQ1RnjcBhu+3wJfyYcYMr/nUOHuBonLd99MON4cl+750rBiv8AAB86SURBVAhgp33r3yaWa3o04rwWdQgJ1hEplCpKcQldZyxS5er9+VuZtymVFy5rf0oyB2gUHcnd5zfj7vObsX1/Oqt2H+ac5rWJrRbhhWiVCiya0FW5+XPrfl6ftYlhnRpwQ6/GJZaPj4kiPibqDESmVOWgv21VuUg5ksX9X68kPiaKf1/eQYe2VcoLtIauTltevoP7vl5BenYeE+7sRVS4vq2U8gb95KnT9sYvm1m0/SBvXN2JljrWilJeo00u6rT8ujGZsfO2cm2PRlzeNc7b4ShVqWlCV2WWeCiDv32zirb1q/PssHbeDkepSk+bXFSpZebkM3HJLj6YvxWHwzB2ZFciQoO9HZZSlZ4mdOWx9Ow8vlq4k48WbGP/sRx6xkfz+JDWNNWuh0r5BE3oqkRHsnL5/I8dfPLHdg5n5NK3eQz3XdicXmfV9nZoSikXmtBVkQ6l5/DpH9v57M8dHM3K48LWsYy+sDldG9fydmhKKTc0oSu3NiQdYeTHiziYnsPgdvUYfWFz2jc89VJ+pQDISoPfXoX2V0KDzt6OptLShK5OsTn5KCM/XkRYcBAz7j+Xtg2qezsk5csyDsKXIyBpJSwaBxe/Ct1u9nZUFcMYMA4I8s1OAJrQ1UkSUo5x/UeLCA4SJtzZy+NJKVQllb4fvhgO+7fAiHGwagL8cD8kLoaLX4PQKt6OsOwcDji4FZJWnXxz5EP7y6HrTdCwG/jQMBea0NVx2/enc/1HCwHD13f21mSuind0n03mh3bC9ROh2YXQ4UqY+y9Y8BokrYarv4DoeG9H6hljYOsc2DwL9q2GfWsg55hdFhwGsW2h7XDIz4E138Lyz+1jXW6EjtdAlPc7Ceh46AqAXQcyuGbcX2TnOfj6zt60qqeX8KtipO2Bzy+1Sf36byD+3JOXb/oZpo6y/1/+EbQcdOZjLI3UzTDzcUiYDaFRUL8j1OsI9TvZ/+u0huDQE+WzjsDaKbDiS9izzCb81pfY5H5WPwiquGs2dYILVazEQxlc8+FC0nPymHBHb20zP9PWTYVln8GQV6FOS29HU7JDO20yzzwEIydD417uyx3cDpNutDXd8/4OF4w5ue05PxdSN51oyti3Go4lF7/vkCpQt92JRFuvI1SpWfbnkpUG81+BRR9AaKSNseeok5N3SZLXwfIvYfVEe0wiYyCihM9Qvyfsr5ky0ISuipSUlsk1Hy7kcEYOE+7srT1ZzrTUTTDuAsjNgLCqMPw9aHeZt6Mq2oGttpkl+wjcONW2IRcnNxOmPwIrv7JNMq2H2sSdtAqS10N+ti0XGgX12kONRsW3SWcdsV8QR/eeeKxWU5vg63WE+p3t/1XrFB+Xw2FjmvO8PQ/Q9Ua48OmS1ytOXjZsnA5bfgFHbvFlO4+EZv3KtBtN6JVUenYeYSFBhBYxpVvykSyuHbeQ/Uez+fKOXnRudBo1HVV6uZnw8QDbbDFyEsz4O+xZCmePhgHPlq6WeCakboYvhtnEddP3tobsCWNse/OMR237c0QNZw27E9Rz/q3drHQ9R46lwj7Xk5Wr4dD2E8ur1T+xj4JkXyPOflnsWgQ//d32ymnUC4a8DA26lO5YeNFpJ3QRGQy8DQQDHxtjXiq0/BbgVWCP86F3jTEfF7dNTegV69ulu3l08moAwkKCqBoeQmRY8PG/UeEhbEtN53BGDl/c3otuTfRioTPux4dg6Se22aLFQMjLgZn/gCUfQeNz4KrPoFpdb0cJOemw7juY/QwgNpnXbVv67RxJsgm9ZuOK6RmSedjW3gt+ASStgv2bbTdDgCrR9gTtnmVQrQEMfN42e/hQLxVPnFZCF5FgYDMwEEgElgDXGWPWu5S5BehujBntaVCa0CuOMYYhby8gJ8/BZV0akp6TR3p2HhnZ+c7/7V+Af1zchh5No70ccSW0/nuYdBOccx9c9M+Tl62eBNPut+2wV30GTc458/EZA3uWw4ovYM0UyDkKMa3g2vEQ0+LMx1NWORm2jTtppU3wqRsh/nzo+zcI989eXKc7SXRPIMEYs825sYnAcGB9sWspr1mx+zAb9x3lxRHtGdmribfDUYUd2gnf32fbny98+tTlHa+2J/6+uRE+GwoXvQC97zkzNcmMg7D6G1j+BaSstych242wbcyNz/a72ixhkdCoh71VAp4k9IbAbpf7iYC709pXiMh52Nr834wxuwsXEJFRwCiAxo1LnkRYlc2ERbuICgtmeOeG3g5FFZafC1NuBwxc8QmEhLkvV7cdjJoL391jm2F2L4L+z9i25vLkyLcXBSWtgs0/w8YfbbNIgy4w9E1of4Vt81Z+wZOE7u4ruXA7zQ/A18aYbBG5G/gcuPCUlYwZB4wD2+RSyliVB9Iyc/lx9V5GdImjqs7t6XvmvgiJS+DK/5Z8wU1EDbjmK/jzHZj9nG2maXqu7evcdljpr8LMy4HUDSefSExea3vYAETUhO632e3Xa1+256e8ypNPfCLQyOV+HLDXtYAx5oDL3Y+Al08/NFUWU5cnkpXrYGQv/QVUImNg2X9h5QQY/BLEuW2WLD8Jc+D3N6HbLfbScU+IQJ8HoMNVNs4VX9kLdmY8Ch2vssnX3WBYOenOtmOXniApG050pwurZnupdLvlxAU0MS0hWCsB/syTk6Ih2GaU/theLEuA640x61zK1DfGJDn/HwE8ZozpXdx29aRo+TPGcNGbvxEZFsz3o/t6OxzflpMB0x+CVV/bdmJHHgz+N/S4o/TtxPu32J4UtZsX3fXuaDJ80MdedHLnr7ZttywcDtj5u72QZcM0yMuCeh1sv2ZH/okLdAr37ii4EKegn3at+Aq9mlFVnNM6KWqMyROR0cBMbLfFT40x60TkeWCpMWYacL+IDAPygIPALeUWvfLY0p2H2JJyjJev6ODtUHzbga22h0nyOrjgH9DzTph6N8x4xDaHDH3Ls4R7JAlmP2uvEAR7pWG9DoUuGW8DQSG2Vp19DG7+oezJHGwSjj/P3jJfgTWT7QnMn8fY5dUa2H23HX6iD3b1hv53MlOViV5YFED+9s1KZq9PZtET/YkM05/Obm2cbpN3UDBc/jG0GGAfdzhgweu2jTu2LVzzZdEnIPOy4a/34LfXbBPG2aNtc0VB7Thpte3mBxAUCjUbwcFtcOk7FTes7P4E282xamzFbF/5jNPttqj8wKH0HKavSeKa7o00mbuTnwdz/2nbsOt3tqMA1nLp0hkUBOc/Cg27wpQ77OX4l70PbYaeKGMMbPrJ9jo5tB1aXQKD/gnRZ9nlna+zfx0OuzxppU3uSaug5WA73GpFiWlecdtWfkM/+QFiyvJEcvIcXK8nQ091LBWm3Abbf7MnAQe/DKER7ss27w93zYdJN8M3I+0JyQuftuNi/zwGtv5qL7C5caodm8SdoCBbu6/dzHb7U+oM0YQeAIwxTFi8i66Na9KmfgCNlGiMPWmZlnga23DAss8h8yAMHwtdRpa8Ts3GcNvPNoH/8bYdH/vAFjuA1OCX7IlTXxtnRSk0oQeEhdsOsi01ndeu6uTtUMrX4nF2EKXTVbu5HbPb08GkAELC7YU1cT1tYu9yA1z4FETFnH48SlUQTegBYMLiXVSPCGFox/reDqX87F0Js560bc/XjD+9XhoSVPb1O18Hna7VXiLKL2hC93MHjmXz89okRvZqQkSob05cW2rZR2HybbbP9vCx3r/YRZO58hOa0P3c5GWJ5OabwLoydPojtpfIzT/4xDyNSvkLvVTMjzkchq8X76Jn02ha1A2QOUBXTrAX6pz/GDTVq12VKg1N6D5qwZZUft2YTE6eo8gyf249wI4DGYHTVXH/Fpj+MDTpC+c96u1olPI72uTig9buSePW/y4hz2GoUSWUQe3qMrRjA85pVpsQl+nkJizeSa3IUAa3r+fFaMtJbhZ8e6sdQfCKj0o3HZlSCtCE7nNy8hw8Onk1taLCeH5YO35Zn8yMNfuYtDSR6KgwBrevx9CO9WlWpyqz1iVzyzlNffdk6N4VtrdK60tKviR91pOQvAaunwTVG5yZ+JQKMJrQfczYeQlsSDrCRzd1Z2DbugzpUJ+s3Hzmb07lx9VJTF2+hwmLdhEeEkSew3CdLza3HE2GOc/ByvH2/oxHTlz63qz/qb1WNvxg59E8ezS0HHTm41UqQGhC9yHr9x7h3V8TuKxzAwa2PTE5cERoMIPa1WNQu3pk5OTx68YUpq9Oom71CJrV8aF5EfNyYNH7MP9VO6zrOffbS9/XTrFXfG780c7G3nmkvVAnOh4O74Lv77Uz5PR/xtvPQCm/pqMt+ojcfAeXvfcHyUey+eVv51ErqoipyTyVk2Fnp6nXqWz9uB0OO5RsaIQdAraoqdIKbJ4JPz9uxzxpMQgG/evkAaPyc+0UZ8u/gITZ9pL8pudC9hE4sA3u/u3EIFdKqSLpaIt+4IN5W1m39wgf3NDt9JN56maYdKOd4bxqPXu1Y5cbPZuPMm2PbSpZ8RUc3mkfCwqFum1dxvnubOe8DIu0PVN+fhwSfrGX2I+cDC0Gnrrd4FBoc6m9pe2BVRPsJA2Hd9q5NTWZK3XatIbuAzbuO8Kl//mdwe3r85/rupzextZ9Z5swQsJtX+6tc2HLTFsjbtLXzt7eZtjJkyzk5cDmn2yC3TrHlo0/DzrfYGv3BfNPJq2yg1yBvZy+dnM7zndIFbjgMeh5V8k1eVcOBxxNgho6mbVSniquhq4J3cvy8h2MGPsnew9n8stD5xNdUDt35NvRBj1tLsnPg9nPwF/vQsPucPXnUCPOLjuSdKJGfGg7hNeADldC64ttwl81ETL229luuoy0bdzuJjA2xo58uG/1iXkqqzeEC8boxApKnSGa0H3Ye3MTeHXmJt4f2ZUhHZyDa22daydZADswVNeboE6rojdyNBkm3wo7/4Aed9r2a3c1ZWNsmeVf2Bnk87Ls9GithkCXm+xY4Nr/WymfpgndR21OPsrQd35nYNu6vDeyq22C+P11+PVFqNPatnlv/tlOYBzX0yb2diMg3KVny84/4dtbIOsIDHsHOl7t2c4zD9vkHtdDa9dK+RE9KeqD8vLtBURVI0J4bng7yDwEU//PtmW3v9Im57AoOJZim0SWfwHTRtuxuduNgK43Q+JimPWUnUrtxqn2RKWnqtS0F/wopQKGJnQv+WjBdlbtPsy713ch5ugm2yslLRGGvGpnoS8YsrVqLPS5H865D3Yvsu3ga6fAii/t8tZD4bKxEFHDe09GKeUTNKFXAGMMi7cfZPehTFKPZpNyNIuUo9mkHjnxf0ZOPkPa1+OS/F/hk4ehSjTc+hM06ul+oyLQuLe9Df43rP8OgsOg4zU6XrdSCvAwoYvIYOBtIBj42BjzUhHlrgS+BXoYYyptA/mrMzcxdt7W4/erhocQWy2cOtXC6RBXk9hq4cRVFUYeHot8/4W9wObK/0LVOp7tIKJ6xc4gr5TySyUmdBEJBt4DBgKJwBIRmWaMWV+oXDXgfmBRRQRabhz59kKY1I3Fl2s3ArrfWurNz9mQzNh5W7mqWxz39mtObPVwIsNcDnNulr0E/o+3YN8a6Ps36Pek92flUUr5PU+ySE8gwRizDUBEJgLDgfWFyr0AvAI8Uq4RlrflX8DiD+3YIcHh7stkHIAf/wa1mkKzfh5vevfBDB6atIp2DarzwmXtTx4Fcd8a2/69+hvIOmxnlr92gp6YVEqVG08SekNgt8v9RKCXawER6QI0Msb8KCJFJnQRGQWMAmjc2AujBGalwa//hMbnwK0zim57zsmAj/rB/0bB//3hUbe+7Lx8Rk9YjsMYxo7sapN5VhqsmWy/RJJW2jbvNpfay/Djz4cgnV9EKVV+PEno7rLe8c7rIhIEvAncUtKGjDHjgHFg+6F7FmI5+u1VW/se/O/iTySGRdo27Y/6wdS7YOSUEpPvv6ZvYFViGh/c0I0mYcdg6sP2Mvy8TKjbHga/bPuIR0aX85NSSinLk4SeCDRyuR8H7HW5Xw1oD8wTmyTrAdNEZJhPnRg9sBUWfmAvbW/QueTyddvC4Jfgxwfhz7dtW3cRfli1l8//2skdfeMZ3CgfPhtmB6AquMqzQRftiaKUqnCeJPQlQAsRiQf2ANcC1xcsNMakATEF90VkHvCITyVzsDPihITDhU97vk63W2D7fJjzAjTp47ZL4dbUY4yZsppuTWrx2NmR8N8h9iKhm76Hxr1O3aZSSlWQEhtxjTF5wGhgJrABmGSMWSciz4vIsIoOsFxsnQubZsC5D0O1uiWXLyACl75tB7mafLtN1C4yc/K556vlhIcG8/7FNQn9Yqg94XnTd5rMlVJnXOCP5ZKfBx+eCznpcO9iO2FDaSUug08vsoNYXf3l8eaTR75dxZTliUy6vDY9frsF8rJtzbx+x/J9Dkop5VTcWC6B381i+eeQsh4ueqFsyRwgrpudHm3DD7D0EwAmLdnN5GWJPNtT6DHvBtu//ZbpmsyVUl4T2FezZB6GuS/a9u82p9k6dPZo2P4b5ud/sDi3OU/NyOT6xoe4afNTtm3+5h8gpkX5xK2UUmUQ2DX0+a9AxsGSuymWwOEwLNpxiJfCH2B/fiQxP9/NwCpb+GfaP5CwKNunXZO5UsrLAreGvj/BXhHa5QY7D2YpGWNYsfswP6zay4w1SSQfySYiNIiIxk/wwJ5HeTf3KXsl6c0/2Ks+lVLKywI3oc96ws512b8U3RSdxi/aydi5W9lzOJOw4CDOb1WHoR3rM6BNXaLCQ+CPTNgwDa76XOfDVEr5jMBM6Alz7Ew/A54t9Ww8M9ft44mpa+nRtBYPDWzJwHZ1qR4RenKhPvfbm1JK+ZDAS+j5eTDzCdsc0vueUq2akHKMhyetolNcDb68vdfJg2sppZSPC6yEboydoi11A1zzle194qFj2Xnc9eVSwkOCeP+GbprMlVJ+J7AS+qIPYMlHtothm0s9Xs0YwyOTVrHjQAZf3t6TBjWrVGCQSilVMQKn2+LGGXbiitZDYeALpVr1/flb+XndPh4f0ppzmsWUvIJSSvmgwEjoe1fAlNvtKIqXf1SqccZ/25zKazM3cWmnBtzeN74Cg1RKqYrl/wk9LREmXAuRteG6b+xY5h7afTCD+yeuoEVsNV6+ogOiQ9wqpfyYf7ehZx2B8VdDbgbcOLNUIylm5uRz15fLcDgMH97Y7eR5P5VSyg/5bxbLz4PJt9rJnkd+ayek8JAxhiemrmHDviN8enMPmsZEVWCgSil1Zvhnk4sx8NPfIWE2DH0Dmvcv1epf/LWT/63Yw4P9W9KvdekuPFJKKV/lnwl94Vg7jG2fB+ysQqWw60AG/5y+ngFtYrnvwuYVE59SSnmB/yX0DT/aK0HbDIP+z5Z69VdnbSI4SHhxRAeCgvQkqFIqcPhfQg8Jh7POhxEflqp7IsCaxDR+WLWXO/qeRd3qZZzsQimlfJT/nRRtMRCaDyj1+ObGGF76eQO1IkMZdf5ZFRScUkp5j//V0KFMk1Us2LKfPxIOcN+FLU4dPVEppQKAfyb0UnI4DC/9tJG4WlUY2Vsno1BKBaZKkdCnrdrL+qQjPDqoFeEhOoqiUioweZTQRWSwiGwSkQQRGeNm+d0iskZEVorI7yLi+VU+FSw7L5/XZm2iXYPqXNqxgbfDUUqpClNiQheRYOA9YAjQFrjOTcKeYIzpYIzpDLwCvFHukZbRVwt3kXgokzFDWms3RaVUQPOkht4TSDDGbDPG5AATgeGuBYwxR1zuRgGm/EIsuyNZubz76xb6No/h3BZ1vB2OUkpVKE+6LTYEdrvcTwR6FS4kIvcCDwFhwIXuNiQio4BRAI0bV/zJyQ/nb+VQRi5jhrSu8H0ppZS3eVJDd9dOcUoN3BjznjGmGfAY8KS7DRljxhljuhtjutepU7E15n1pWXzy+3aGdWpA+4Y1KnRfSinlCzxJ6IlAI5f7ccDeYspPBC47naDKw9tzNpPvMDw6qJW3Q1FKqTPCk4S+BGghIvEiEgZcC0xzLSAiLVzuXgJsKb8QSy8h5SjfLNnNDb2b0Cja8wkvlFLKn5XYhm6MyROR0cBMIBj41BizTkSeB5YaY6YBo0VkAJALHAJursigS/LKz5uIDAthdD8dTVEpVXl4NJaLMWYGMKPQY0+7/P9AOcdVZuv2pjFrfTIPD2xJ7arh3g5HKaXOmIC7UnTm2n0ECdzQu4m3Q1FKqTMq4BL67A0pdG8STa2oMG+HopRSZ1RAJfQ9hzNZn3SE/m10WjmlVOUTUAn91w3JAPRvU9fLkSil1JkXUAn9lw0pxMdE0axOlLdDUUqpMy5gEvqx7DwWbj1A/9axSBkmwFBKKX8XMAn99y2p5OQ7GNBWm1uUUpVTwCT0X9anUKNKKN2b1PJ2KEop5RUBkdDzHYa5m1K4oFUdQoID4ikppVSpBUT2W7n7EAfTcxigvVuUUpVYQCT0X9anEBIknN9KJ7FQSlVeAZHQ52xIptdZ0VSPCPV2KEop5TV+n9B3HkhnS8ox+rfW5halVOXm9wl99oYUAG0/V0pVen6f0OdsSKZl3ao0rq0TWSilKje/Tuhpmbks3n5Qx25RSin8PKHP35xKnsNoc4tSSuHnCX3OhmRqR4XRuVFNb4eilFJe57cJPTffwdyNKfRrHUtwkA7GpZRSfpvQl+44xJGsPG1uUUopJ79N6LM3JBMWHMS5LWK8HYpSSvkEv0zoxhjmbEjmnOa1iQoP8XY4SinlEzxK6CIyWEQ2iUiCiIxxs/whEVkvIqtFZI6INCn/UE/YmprOjgMZ2l1RKaVclJjQRSQYeA8YArQFrhORtoWKrQC6G2M6ApOBV8o7UFezC+YOba2TQSulVAFPaug9gQRjzDZjTA4wERjuWsAYM9cYk+G8uxCIK98wTzZnQzLtGlSnQc0qFbkbpZTyK54k9IbAbpf7ic7HinI78JO7BSIySkSWisjS1NRUz6N0cTA9h2U7D2lzi1JKFeJJQnfXydu4LShyA9AdeNXdcmPMOGNMd2NM9zp1yjZ2+dyNKTgMDNSErpRSJ/Gki0gi0Mjlfhywt3AhERkAPAGcb4zJLp/wTlW9SigXta1L+4bVK2oXSinllzxJ6EuAFiISD+wBrgWudy0gIl2AD4HBxpiUco/SxcC2dRnYVmvnSilVWIlNLsaYPGA0MBPYAEwyxqwTkedFZJiz2KtAVeBbEVkpItMqLGKllFJueXRVjjFmBjCj0GNPu/w/oJzjUkopVUp+eaWoUkqpU2lCV0qpAKEJXSmlAoQmdKWUChCa0JVSKkBoQldKqQAhxri9ir/idyySCuws4+oxwP5yDKci+EOM4B9xaozlQ2MsH96OsYkxxu3YKV5L6KdDRJYaY7p7O47i+EOM4B9xaozlQ2MsH74coza5KKVUgNCErpRSAcJfE/o4bwfgAX+IEfwjTo2xfGiM5cNnY/TLNnSllFKn8tcaulJKqUI0oSulVIDwu4QuIoNFZJOIJIjIGG/H446I7BCRNc6x4Zd6Ox4AEflURFJEZK3LY9Ei8ouIbHH+reWDMT4rInucx3KliFzs5RgbichcEdkgIutE5AHn4z5zLIuJ0deOZYSILBaRVc44n3M+Hi8ii5zH8hsRCfPBGD8Tke0ux7Kzt2I8iTHGb25AMLAVOAsIA1YBbb0dl5s4dwAx3o6jUEznAV2BtS6PvQKMcf4/BnjZB2N8FnjE28fPJZ76QFfn/9WAzUBbXzqWxcToa8dSgKrO/0OBRUBvYBJwrfPxD4D/88EYPwOu9PYxLHzztxp6TyDBGLPNGJMDTASGezkmv2CM+Q04WOjh4cDnzv8/By47o0EVUkSMPsUYk2SMWe78/yh2Fq+G+NCxLCZGn2KsY867oc6bAS4EJjsf9/axLCpGn+RvCb0hsNvlfiI++EbFvuCzRGSZiIzydjDFqGuMSQKbBIBYL8dTlNEistrZJOPVZiFXItIU6IKttfnksSwUI/jYsRSRYBFZCaQAv2B/gR82dupL8IHPeOEYjTEFx/JF57F8U0TCvRjicf6W0MXNY774bdnHGNMVGALcKyLneTsgP/Y+0AzoDCQBr3s3HEtEqgJTgAeNMUe8HY87bmL0uWNpjMk3xnQG4rC/wNu4K3Zmoyq080Ixikh74HGgNdADiAYe82KIx/lbQk8EGrncjwP2eimWIhlj9jr/pgBTsW9UX5QsIvUBnH9TvBzPKYwxyc4PlAP4CB84liISik2U440x/3M+7FPH0l2MvngsCxhjDgPzsO3TNUWkYL5jn/mMu8Q42NmsZYwx2cB/8ZFj6W8JfQnQwnkWPAy4Fpjm5ZhOIiJRIlKt4H/gImBt8Wt5zTTgZuf/NwPfezEWtwqSpNMIvHwsRUSAT4ANxpg3XBb5zLEsKkYfPJZ1RKSm8/8qwABse/9c4EpnMW8fS3cxbnT58hZsG79PfMb97kpRZ1ert7A9Xj41xrzo5ZBOIiJnYWvlACHABF+IUUS+Bi7ADv2ZDDwDfIftUdAY2AVcZYzx2knJImK8ANtEYLC9h+4qaKv2BhHpCywA1gAO58P/wLZR+8SxLCbG6/CtY9kRe9IzGFu5nGSMed75GZqIbcpYAdzgrAn7Uoy/AnWwzcArgbtdTp56jd8ldKWUUu75W5OLUkqpImhCV0qpAKEJXSmlAoQmdKWUChCa0JVSKkBoQldKqQChCV0ppQLE/wNnSClxi4kojwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# evaluate the model\n",
    "_, train_acc = model.evaluate(X_train, y_train, verbose=0)\n",
    "_, test_acc = model.evaluate(X_test, y_test, verbose=0)\n",
    "print('Train: %.3f, Test: %.3f' % (train_acc, test_acc))\n",
    "# plot training history\n",
    "plt.plot(history.history['accuracy'], label='train')\n",
    "plt.plot(history.history['val_accuracy'], label='test')\n",
    "plt.title('Training and test Accuracy at each epoch', fontsize=14)\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Train loss is 0.904, Test loss is 0.518"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Grid Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "\n",
    "def create_model(neurons=1):\n",
    "    # create model\n",
    "    model = Sequential()\n",
    "    model.add(Dense(neurons, input_shape=(300,), activation='relu'))\n",
    "    model.add(Dense(16, activation='relu'))\n",
    "    #model.add(Flatten())   #add Flatten layer \n",
    "    model.add(Dense(6, activation='softmax'))\n",
    "    \n",
    "    #Compile model\n",
    "    model.compile(loss='categorical_crossentropy', optimizer='adam',\n",
    "              metrics=['accuracy'])\n",
    "    return model\n",
    "\n",
    "model = KerasClassifier(build_fn=create_model, verbose=0, epochs=20, batch_size=50)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: 0.390447 using {'neurons': 64}\n",
      "0.374870 (0.033872) with: {'neurons': 32}\n",
      "0.390447 (0.030770) with: {'neurons': 64}\n",
      "0.376947 (0.019204) with: {'neurons': 128}\n",
      "0.364486 (0.022175) with: {'neurons': 256}\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "#Define the grid search parameters\n",
    "#learning_rate = [0.001,0.0001]\n",
    "\n",
    "param_grid = {\n",
    "    'neurons':[32,64,128,256]\n",
    "}\n",
    "\n",
    "grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1, cv=3)\n",
    "grid_result = grid.fit(tfidf_mat, y)\n",
    "\n",
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print(\"%f (%f) with: %r\" % (mean, stdev, param))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Results from various grid search show that optimizer='Adam\",  epochs=20, activation='relu' produce a better result."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 6: Mixed Input Model with City, Rate Type and Cost of Living Indexes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Use Functional API:  https://machinelearningmastery.com/keras-functional-api-deep-learning/\n",
    "\n",
    "Multi-input full example: https://www.pyimagesearch.com/2019/02/04/keras-multiple-inputs-and-mixed-data/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Explore continuous variables and categorical variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Title</th>\n",
       "      <th>Company</th>\n",
       "      <th>Link</th>\n",
       "      <th>Review</th>\n",
       "      <th>Salary</th>\n",
       "      <th>Description</th>\n",
       "      <th>Location2</th>\n",
       "      <th>Rate_by</th>\n",
       "      <th>low</th>\n",
       "      <th>...</th>\n",
       "      <th>City</th>\n",
       "      <th>Cost of Living Index</th>\n",
       "      <th>Rent Index</th>\n",
       "      <th>Cost of Living Plus Rent Index</th>\n",
       "      <th>Groceries Index</th>\n",
       "      <th>Restaurant Price Index</th>\n",
       "      <th>Local Purchasing Power Index</th>\n",
       "      <th>City_Bins</th>\n",
       "      <th>combined_text</th>\n",
       "      <th>Salary_Bins</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>2021048 Data Scientist $215,000.00</td>\n",
       "      <td>B4CORP</td>\n",
       "      <td>https://www.indeed.com/pagead/clk?mo=r&amp;ad=-6NY...</td>\n",
       "      <td>None</td>\n",
       "      <td>Up to $215,000 a year</td>\n",
       "      <td>Position Requires a Top Secret (TS/SCI) Cleara...</td>\n",
       "      <td>Arlington, VA</td>\n",
       "      <td>Yearly</td>\n",
       "      <td>Up to $215,000 a year</td>\n",
       "      <td>...</td>\n",
       "      <td>Arlington, VA</td>\n",
       "      <td>80.159143</td>\n",
       "      <td>58.838571</td>\n",
       "      <td>69.961143</td>\n",
       "      <td>76.886286</td>\n",
       "      <td>78.608571</td>\n",
       "      <td>115.594571</td>\n",
       "      <td>medium</td>\n",
       "      <td>2021048 Data Scientist $215,000.00 Position Re...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Data Science/Data Analyst</td>\n",
       "      <td>CRAiC LLC</td>\n",
       "      <td>https://www.indeed.com/company/CRAiC-LLC/jobs/...</td>\n",
       "      <td>None</td>\n",
       "      <td>$60,000 a year</td>\n",
       "      <td>Cost Readiness Analytics (CRAiC LLC) is lookin...</td>\n",
       "      <td>Arlington, VA</td>\n",
       "      <td>Yearly</td>\n",
       "      <td>$60,000 a year</td>\n",
       "      <td>...</td>\n",
       "      <td>Arlington, VA</td>\n",
       "      <td>80.159143</td>\n",
       "      <td>58.838571</td>\n",
       "      <td>69.961143</td>\n",
       "      <td>76.886286</td>\n",
       "      <td>78.608571</td>\n",
       "      <td>115.594571</td>\n",
       "      <td>medium</td>\n",
       "      <td>Data Science/Data Analyst Cost Readiness Analy...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Data Science Intern</td>\n",
       "      <td>MilliporeSigma</td>\n",
       "      <td>https://www.indeed.com/company/Serono/jobs/Dat...</td>\n",
       "      <td>3.9</td>\n",
       "      <td>$20 - $25 an hour</td>\n",
       "      <td>Job Title: Data Science Intern\\rA career with ...</td>\n",
       "      <td>Arlington, VA</td>\n",
       "      <td>Hourly</td>\n",
       "      <td>$20</td>\n",
       "      <td>...</td>\n",
       "      <td>Arlington, VA</td>\n",
       "      <td>80.159143</td>\n",
       "      <td>58.838571</td>\n",
       "      <td>69.961143</td>\n",
       "      <td>76.886286</td>\n",
       "      <td>78.608571</td>\n",
       "      <td>115.594571</td>\n",
       "      <td>medium</td>\n",
       "      <td>Data Science Intern Job Title: Data Science In...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Jr. Data Scientist</td>\n",
       "      <td>Numero Data LLC</td>\n",
       "      <td>https://www.indeed.com/company/Numero-Data-LLC...</td>\n",
       "      <td>None</td>\n",
       "      <td>$70,000 - $85,000 a year</td>\n",
       "      <td>Ideal candidate should have a degree in a quan...</td>\n",
       "      <td>Arlington, VA</td>\n",
       "      <td>Yearly</td>\n",
       "      <td>$70,000</td>\n",
       "      <td>...</td>\n",
       "      <td>Arlington, VA</td>\n",
       "      <td>80.159143</td>\n",
       "      <td>58.838571</td>\n",
       "      <td>69.961143</td>\n",
       "      <td>76.886286</td>\n",
       "      <td>78.608571</td>\n",
       "      <td>115.594571</td>\n",
       "      <td>medium</td>\n",
       "      <td>Jr. Data Scientist Ideal candidate should have...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Business Data Quality Engineer</td>\n",
       "      <td>HRU Technical Resources</td>\n",
       "      <td>https://www.indeed.com/pagead/clk?mo=r&amp;ad=-6NY...</td>\n",
       "      <td>None</td>\n",
       "      <td>$45 - $55 an hour</td>\n",
       "      <td>Please note that this is not a corp to corp or...</td>\n",
       "      <td>Arlington, VA</td>\n",
       "      <td>Hourly</td>\n",
       "      <td>$45</td>\n",
       "      <td>...</td>\n",
       "      <td>Arlington, VA</td>\n",
       "      <td>80.159143</td>\n",
       "      <td>58.838571</td>\n",
       "      <td>69.961143</td>\n",
       "      <td>76.886286</td>\n",
       "      <td>78.608571</td>\n",
       "      <td>115.594571</td>\n",
       "      <td>medium</td>\n",
       "      <td>Business Data Quality Engineer Please note tha...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0                               Title                  Company  \\\n",
       "0           0  2021048 Data Scientist $215,000.00                   B4CORP   \n",
       "1           1           Data Science/Data Analyst                CRAiC LLC   \n",
       "2           2                 Data Science Intern           MilliporeSigma   \n",
       "3           3                  Jr. Data Scientist          Numero Data LLC   \n",
       "4           4      Business Data Quality Engineer  HRU Technical Resources   \n",
       "\n",
       "                                                Link Review  \\\n",
       "0  https://www.indeed.com/pagead/clk?mo=r&ad=-6NY...   None   \n",
       "1  https://www.indeed.com/company/CRAiC-LLC/jobs/...   None   \n",
       "2  https://www.indeed.com/company/Serono/jobs/Dat...    3.9   \n",
       "3  https://www.indeed.com/company/Numero-Data-LLC...   None   \n",
       "4  https://www.indeed.com/pagead/clk?mo=r&ad=-6NY...   None   \n",
       "\n",
       "                     Salary  \\\n",
       "0     Up to $215,000 a year   \n",
       "1            $60,000 a year   \n",
       "2         $20 - $25 an hour   \n",
       "3  $70,000 - $85,000 a year   \n",
       "4         $45 - $55 an hour   \n",
       "\n",
       "                                         Description      Location2 Rate_by  \\\n",
       "0  Position Requires a Top Secret (TS/SCI) Cleara...  Arlington, VA  Yearly   \n",
       "1  Cost Readiness Analytics (CRAiC LLC) is lookin...  Arlington, VA  Yearly   \n",
       "2  Job Title: Data Science Intern\\rA career with ...  Arlington, VA  Hourly   \n",
       "3  Ideal candidate should have a degree in a quan...  Arlington, VA  Yearly   \n",
       "4  Please note that this is not a corp to corp or...  Arlington, VA  Hourly   \n",
       "\n",
       "                     low  ...           City  Cost of Living Index  \\\n",
       "0  Up to $215,000 a year  ...  Arlington, VA             80.159143   \n",
       "1         $60,000 a year  ...  Arlington, VA             80.159143   \n",
       "2                   $20   ...  Arlington, VA             80.159143   \n",
       "3               $70,000   ...  Arlington, VA             80.159143   \n",
       "4                   $45   ...  Arlington, VA             80.159143   \n",
       "\n",
       "   Rent Index  Cost of Living Plus Rent Index  Groceries Index  \\\n",
       "0   58.838571                       69.961143        76.886286   \n",
       "1   58.838571                       69.961143        76.886286   \n",
       "2   58.838571                       69.961143        76.886286   \n",
       "3   58.838571                       69.961143        76.886286   \n",
       "4   58.838571                       69.961143        76.886286   \n",
       "\n",
       "   Restaurant Price Index Local Purchasing Power Index  City_Bins  \\\n",
       "0               78.608571                   115.594571     medium   \n",
       "1               78.608571                   115.594571     medium   \n",
       "2               78.608571                   115.594571     medium   \n",
       "3               78.608571                   115.594571     medium   \n",
       "4               78.608571                   115.594571     medium   \n",
       "\n",
       "                                       combined_text  Salary_Bins  \n",
       "0  2021048 Data Scientist $215,000.00 Position Re...            5  \n",
       "1  Data Science/Data Analyst Cost Readiness Analy...            1  \n",
       "2  Data Science Intern Job Title: Data Science In...            0  \n",
       "3  Jr. Data Scientist Ideal candidate should have...            2  \n",
       "4  Business Data Quality Engineer Please note tha...            3  \n",
       "\n",
       "[5 rows x 26 columns]"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "salary_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler, OneHotEncoder\n",
    "from sklearn.impute import SimpleImputer\n",
    "\n",
    "cont_features = ['Cost of Living Index','Rent Index','Cost of Living Plus Rent Index','Local Purchasing Power Index']\n",
    "X_cont_df = salary_df.loc[:, cont_features]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 963 entries, 0 to 962\n",
      "Data columns (total 4 columns):\n",
      "Cost of Living Index              939 non-null float64\n",
      "Rent Index                        939 non-null float64\n",
      "Cost of Living Plus Rent Index    939 non-null float64\n",
      "Local Purchasing Power Index      939 non-null float64\n",
      "dtypes: float64(4)\n",
      "memory usage: 77.6 KB\n"
     ]
    }
   ],
   "source": [
    "X_cont_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Cost of Living Index</th>\n",
       "      <th>Rent Index</th>\n",
       "      <th>Cost of Living Plus Rent Index</th>\n",
       "      <th>Local Purchasing Power Index</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>80.159143</td>\n",
       "      <td>58.838571</td>\n",
       "      <td>69.961143</td>\n",
       "      <td>115.594571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>80.159143</td>\n",
       "      <td>58.838571</td>\n",
       "      <td>69.961143</td>\n",
       "      <td>115.594571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>80.159143</td>\n",
       "      <td>58.838571</td>\n",
       "      <td>69.961143</td>\n",
       "      <td>115.594571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>80.159143</td>\n",
       "      <td>58.838571</td>\n",
       "      <td>69.961143</td>\n",
       "      <td>115.594571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>80.159143</td>\n",
       "      <td>58.838571</td>\n",
       "      <td>69.961143</td>\n",
       "      <td>115.594571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>958</th>\n",
       "      <td>87.740000</td>\n",
       "      <td>78.180000</td>\n",
       "      <td>83.170000</td>\n",
       "      <td>130.250000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>959</th>\n",
       "      <td>87.740000</td>\n",
       "      <td>78.180000</td>\n",
       "      <td>83.170000</td>\n",
       "      <td>130.250000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>960</th>\n",
       "      <td>87.740000</td>\n",
       "      <td>78.180000</td>\n",
       "      <td>83.170000</td>\n",
       "      <td>130.250000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>961</th>\n",
       "      <td>87.740000</td>\n",
       "      <td>78.180000</td>\n",
       "      <td>83.170000</td>\n",
       "      <td>130.250000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>962</th>\n",
       "      <td>87.740000</td>\n",
       "      <td>78.180000</td>\n",
       "      <td>83.170000</td>\n",
       "      <td>130.250000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>963 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     Cost of Living Index  Rent Index  Cost of Living Plus Rent Index  \\\n",
       "0               80.159143   58.838571                       69.961143   \n",
       "1               80.159143   58.838571                       69.961143   \n",
       "2               80.159143   58.838571                       69.961143   \n",
       "3               80.159143   58.838571                       69.961143   \n",
       "4               80.159143   58.838571                       69.961143   \n",
       "..                    ...         ...                             ...   \n",
       "958             87.740000   78.180000                       83.170000   \n",
       "959             87.740000   78.180000                       83.170000   \n",
       "960             87.740000   78.180000                       83.170000   \n",
       "961             87.740000   78.180000                       83.170000   \n",
       "962             87.740000   78.180000                       83.170000   \n",
       "\n",
       "     Local Purchasing Power Index  \n",
       "0                      115.594571  \n",
       "1                      115.594571  \n",
       "2                      115.594571  \n",
       "3                      115.594571  \n",
       "4                      115.594571  \n",
       "..                            ...  \n",
       "958                    130.250000  \n",
       "959                    130.250000  \n",
       "960                    130.250000  \n",
       "961                    130.250000  \n",
       "962                    130.250000  \n",
       "\n",
       "[963 rows x 4 columns]"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_cont_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAl0AAAIACAYAAAC4ktkCAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nOzde3zPdf/H8ed33zE5rLE2IldckdMSJdbKYnLezEauOXeaDliUscwlimSEIclVroSlZWPmtIVCOecXLaNEDqNtcpixZYfv7w8336s1p232/bDv437dvrdbn/Pr8+m65eX1fn3eH5PFYrEIAAAApcrB6AAAAADsAUkXAACADZB0AQAA2ABJFwAAgA2QdAEAANgASRcAAIANkHQBuO3l5eXpv//9rwIDA+Xv768uXbpoypQpunTpUqldc/v27fL19b3hfrNnz9a6deskSZGRkVq+fPktu36DBg00atSoQtv69++v5s2b3/Ac33zzjSIjI6+6bf369ZowYUKJ4wRw8xyNDgAAbmTcuHE6d+6cFixYoCpVqujixYsaMWKEwsPDNWXKFENj2759u+rVqydJeu21127pud3c3PT1118rKytLd911lyQpJSVFhw8fvqnjf/zxR507d+6q29q1a6d27drdslgB3BhJF4Db2vHjxxUfH69vv/1WlStXliRVrFhR48eP1+7duyVJ58+f1/jx47V//36ZTCa1bt1ar7/+uhwdHeXh4aF27dpp//79mjp1qnr37l1guWLFipo4caLOnj2rvLw89e/fXz179iwQw+HDh/X222/rwoULSk9PV8OGDTVjxgwtXbpUSUlJioiIkNls1vr161W/fn298MIL2rVrlyIiIpSVlaVy5cpp2LBh8vb2VmxsrL766is5ODjoyJEjqlChgiZPnqwHHnig0L27uLiodu3aWrdunfz8/CRJy5cvl5+fn5YsWSJJunjxosaNG6cjR47o7NmzqlSpkqZOnarz589ryZIlysvLU5UqVXT//fdr6dKlysrKUuXKlRUQEKCEhARFRkaqR48e6tOnj/r27asvv/xSn332maKjo62JHoBbg+FFALe1n376SfXq1bMmXFe4ubmpY8eOkqQJEybIxcVF8fHxiomJ0YEDBzR//nxJUk5Ojtq2bauEhAQ99NBDBZYbNWqkkJAQvfHGG4qNjdWiRYs0f/58/fDDDwWuFR0dre7duys6OlqJiYk6fvy4vvnmG/Xt21ceHh4aOXKk2rdvb93/zJkzCgkJUXh4uOLj4zV58mSFhobq2LFjkqSdO3fq3//+t1auXKmHH35Y8+bNu+b9d+/eXXFxcdblNWvWFBj23LRpk5ydnfXFF18oISFBHh4eWrx4sR5++GEFBQWpS5cuGj58uCTp4MGDWrhwoRYuXGg9vkKFCpo2bZpmzpypjRs3asaMGYqMjCThAkoBlS4AtzUHBwfl5+dfd59Nmzbp888/l8lkUvny5RUUFKQFCxZo0KBBkqQWLVoU2P/K8m+//aajR49q9OjR1m3Z2dnat29fgcpTaGiovvvuO/3nP//Rb7/9prS0NF28ePGa8ezdu1f/+Mc/9PDDD0uS6tevr0ceeUQ7duyQyWRSkyZNVKNGDUlS48aN9dVXX13zXG3bttW4ceN06tQpHTlyRP/85z919913W7d36tRJtWvX1sKFC3XkyBHt2LHjmv1eDRo0KJS8Xlk/ZMgQvfTSS3rvvff0z3/+85rxACg+ki4At7WmTZvq0KFDyszMLJAwpKam6t///rdmzpyp/Px8mUwm67b8/Hzl5uZalytWrFjgnFeWrwy9/bWSdOrUKVWpUqVAtev1119XXl6eOnfurDZt2ujkyZO63mdr8/LyCsQjSRaLRbm5uSpXrpwqVKhgXW8yma57rvLly6tDhw5atWqVDh48qICAgALbo6KiFB0drb59+8rPz08uLi46fvz4Vc/19+fwV7/88ovuuece7dmzR927d7/mfgCKj+FFALe16tWry8/PT6NHj1ZmZqYkKTMzU+PGjZOLi4sqVKigJ598UosWLZLFYtGlS5cUHR0tLy+vG567bt26qlChgjXpOnnypHx9fZWUlFRgv2+//VaDBw9Wly5dJEl79uxRXl6eJMlsNhdI8CSpWbNmOnTokPbu3SvpckKzc+dOtWzZsljPoHv37lq2bJl27typ1q1bF4otICBAzzzzjOrWrasNGzZcN7arSUxM1Pbt27VixQp999131rcxAdxaVLoA3PbeeustzZkzR0FBQTKbzbp06ZKefvppDR06VJI0ZswYTZgwQX5+fsrJyVHr1q318ssv3/C85cuX15w5czRx4kR9/PHHys3N1WuvvaZHH31U27dvt+43fPhwDR48WBUrVlTlypX12GOP6ejRo5IkHx8fTZs2TTk5Odb9q1WrpsjISL3zzjvKzs6WyWTSpEmTVLduXf3f//1fke+/efPmysrKko+PjxwdC/5n+/nnn9fYsWO1dOlSSZcTvp9//lmS5OnpqREjRuidd95RkyZNrnrukydP6q233tLcuXNVrVo1vffeexo8eLA8PDysQ6AAbg2T5Xp1bQAAANwSDC8CAADYgN0lXZmZmRo/frx8fX3l7++v/v3766effirWuaKjo7Vy5coiXTswMFC+vr4FJjeMjY1VWFhYof3Xr19/zdmkb2b7zbrW9a/Hx8fnms26AADcjjIzM+Xr63vVP7+Sk5MVGBiojh07Kjw83NoPeeLECfXt21edOnXSK6+8ogsXLhT7+naVdOXn5ys4OFh33323li9frri4OA0ePFjBwcE6c+ZMkc+3e/fuIn2GJDk5WeXLl9fKlStVt27dG+7frl27685wfaPtAADgsj179qh379767bffrro9NDRUY8eOVUJCgiwWi6KjoyVJ48ePV58+fbR27Vp5eHhozpw5xY7BrpKu7du36+TJkwoJCbE2o3p6emrSpEnWeYDmzp2rLl26yM/PT++9957y8vKUmZmpQYMGKTAwUIGBgVq/fr22bNmiDRs2aObMmdq8eXOB65w6dUovvfSS/Pz8FBAQoE2bNumPP/7Q6NGjdeDAgZtq8JX+V4Fav359gWMWLlyoCRMmFKhQ+fj4aMaMGerZs6e6du1qffvq559/tn6v7p133ikwgePVhIWFacKECerdu7d8fHwUExMjSTp79qyCg4Pl5+enYcOG6c8//5R0+dX4SZMmKSAgQN26ddOnn34qSfrss8/Ur18/WSwW7dq1Sx06dCjR3w4AACiJ6OhovfXWW3J3dy+0LSUlRdnZ2WrWrJkkKTAwUGvXrlVOTo527txpnYj5yvrisquka9++fWrYsKEcHAre9lNPPSVXV1dt3LhRGzZsUExMjJYtW6YjR45oyZIl+uqrr1SrVi3FxsZq4sSJ2rVrl7y8vOTj46OQkJBCr3C/88478vT0VHx8vGbOnKnRo0fLYrFowoQJ8vDw0Ny5c4sUt7e3t5KSkqzfUFu1apW6detWaD8XFxctXbpUQUFB+uijjyRdTqJee+01xcXFqXbt2tZXya/n999/V1RUlD788ENFRERIkmbOnKnGjRsrPj5effv21alTpyTJ+jeBZcuWaenSpVq/fr127dql/v37y2QyKSoqSmPGjNGkSZNUqVKlIt03AADXk5GRoePHjxf6ZWRkFNp34sSJhSZKviItLU1ubm7WZTc3N6WmpurMmTOqXLmytVBzZX1x2dWUEQ4ODnJycrrm9m3btqlr167Wz1/06NFDy5cv14gRIzRt2jSlpqaqTZs2Gjx48HWvs23bNk2YMEGSVLt2bT388MPas2fPVWeCvhnlypVT+/btlZiYqCeeeEJnz55V06ZNdfDgwQL7XUn+6tevr8TERJ09e1YpKSl66qmnrPfz2Wef3fB6TzzxhEwmkx588EGdPXtWkrRjxw69//77kqTHHntMtWvXliRt3bpVycnJ2rZtm6TL34E7cOCAWrRooXfffVd+fn7q3bu3Hn300Zu+35xTh256XxRP7o8bjA6hzHNwu9/oEOxCpUeeNTqEMi/3Ukqpnbuk/71f8PkqzZ49u9D6IUOGWKeUuRl/n2DZYrFYJy7++0THf18uCrtKujw8PBQVFVXoIU6bNk1eXl5X/dRIbm6u6tSpozVr1mjz5s36+uuvNX/+fK1evfqa1/n7LBwWi+WmKkzX4+/vr8jISJ07d8764du/u5JQXrk3s9l83Zmur+Xv57nyz389l9lslnR5eDE0NFQdOnSQJJ0+fdpa0Tpx4oQqVaqkffv2XfX/uAAAO5dfsj8bBw4cWOgrDZLk7OxcpPPUqFFD6enp1uVTp07J3d1d1apV0/nz55WXlyez2az09PSrDk/eLLsaXmzRooVcXV01e/ZsaxK0efNmxcbGql69evL09NSqVauUnZ2t3NxcxcTEyNPTU4sWLdKsWbPUuXNnvfXWWzp9+rQyMzNlNpuvmkx5enpaJyo8duyYdu/ebR0nLq5mzZopLS1NcXFxVx1avJoqVaqodu3a2rhxoyQpPj6+2Nd//PHHrbN279271zoxpKenp6Kjo5WTk6MLFy6oT58++uGHH3ThwgX9+9//1ocffqgKFSooKiqq2NcGAOBqnJ2ddd999xX6FTXpqlWrlpycnPT9999LkuLi4uTt7a1y5cqpRYsW1kLL8uXL5e3tXex47arSZTKZNGfOHE2aNEm+vr5ydHRU1apVNW/ePN1zzz1q27atkpOT1aNHD+Xm5urJJ59Uv379lJ2drddff11+fn4ym80KDQ2Vs7OzvLy8NG3aNFWpUkWdOnWyXic8PFxjx45VbGysJGnChAlyd3cvME3E38XHxyshIcG6/NJLLxXKpjt37qxvv/3WOrR3MyIiIjR69GjNmDFDDRo0KPDNt6IICQlRWFiYunbtqn/+85/WGIKCgnTkyBEFBAQoNzdXgYGBatWqlcaNG6ennnpKTZs21dixY9WrVy95e3sXKXYAQBlnuf7H7EtbcHCwQkJC9NBDD2nq1KkaM2aMMjMz1aRJEw0YMEDS5S9ihIWF6cMPP9S9996radOmFft6zEhfxs2ePVu9evWSu7u7EhMTFR8fr1mzZhkd1nXR01X66OkqffR02QY9XaWvVHu6TiaX6Phy9za6RZHYhl1VuuxRzZo19fzzz8vR0VHOzs6aOHGi0SEBACBJshhc6bI1kq4y7srcYgAA3Hau8gJbWWZXjfQAAABGodIFAACMwfAiAACADZRwnq47DUkXAAAwBpUuAAAAG6CRHgAAALcalS4AAGAI5ukCAACwBTsbXiTpAgAAxrCzShc9XQAAADZApQsAABiDeboAAABswM6GF0m6AACAMWikBwAAsAE7q3TRSA8AAGADVLoAAIAxGF4EAAAofRYLby8CAACUPjvr6SLpAgAAxrCz4UUa6QEAAGyAShcAADAGw4sAAAA2wGeAAAAAbMDOKl30dAEAANgAlS4AAGAMO3t7kaQLAAAYw86GF0m6AACAMah0AQAA2ICdJV000gMAANgAlS4AAGAIPngNAABgC3Y2vEjSBQAAjGFnby/S0wUAAGADVLoAAIAxGF4EAACwATsbXiTpAgAAxqDSBQAAYAN2VumikR4AAMAGqHQBAABjMLwIAABgAyRdAAAANmBnPV0kXQAAwBh2VumikR4AAMAGqHQBAABjMLwIAABgA3Y2vEjSBQAAjEGlCzBW7o8bjA6hzHN8yMfoEMq+vByjI7ALz9z7mNEhADeNpAsAABiD4UUAAAAbIOkCAACwAYvFZpeKj4/Xhx9+qNzcXA0cOFB9+/a1bktOTlZYWJh1+fTp07r77ru1cuVKLVu2TO+//75cXV0lSW3atNHw4cOLFQNJFwAAMIaNKl2pqamaPn26YmNjVb58eQUFBalVq1aqV6+eJKlRo0aKi4uTJGVlZemZZ57RuHHjJElJSUkKCwuTr69vieNgclQAAHBHysjI0PHjxwv9MjIyCuy3ZcsWeXp6ysXFRRUrVlTHjh21du3aq57zo48+0mOPPaYWLVpIkn788UctW7ZMfn5+GjFihM6dO1fseEm6AACAMfLzS/RbsGCB2rVrV+i3YMGCApdJS0uTm5ubddnd3V2pqamFwjl//ryio6M1ZMgQ6zo3Nze9+uqrWrFihe699169/fbbxb5dhhcBAIAxSjhP18CBAxUQEFBovbOzc4Hl/Px8mUym/13WYimwfMWKFSv09NNPW/u3JOmDDz6w/vOLL76o9u3bFzteki4AAGCMEvZ0OTs7F0qwrqZGjRratWuXdTk9PV3u7u6F9lu3bp1eeukl6/L58+cVExOjZ599VtLlZM1sNhc7XoYXAQCAMSyWkv1ukpeXl7Zu3arTp08rKytLiYmJ8vb2/lsoFv30009q3ry5dV3FihX18ccfa8+ePZKkRYsWUekCAAC4lurVq2v48OEaMGCAcnJy1LNnTzVt2lTBwcEKCQnRQw89pNOnT6tcuXJycnKyHmc2mzVjxgyNGzdO2dnZqlOnjiIiIoodh8liseEkGcBNyPr6Y6NDKPP4DJAN8BkgmxjQcpTRIZR5nx9ZXmrnzvrvyBIdf9dzxU+AjEClCwAAGIMZ6QEAAGyghG8v3mlopAcAALABKl0AAMAQlnz7aisn6QIAAMagpwsAAMAG7Kyni6QLAAAYw86GF2mkBwAAsAEqXQAAwBj0dAEAANgASRcAAIAN2NmXCEm6AACAMeys0kUjPQAAgA1Q6QIAAMawsykjSLoAAIAxmBwVAADABuys0kVPFwAAgA1Q6QIAAIaw2NnbiyRdAADAGHY2vEjSBQAAjEEjPQAAgA3YWaWLRnoAAAAboNIFAACMQSM9AACADdjZ8CJJFwAAMAaN9AAAADZgZ5UuGukBAABsgEoXAAAwBDPSAwAA2IKdDS+SdAEAAGPYWdJFTxcAAIANUOkCAADGYMoIAAAAG7Cz4UWSLgAAYAgLSRcAAIAN2FnSRSM9AACADVDpAgAAxmByVAAAABuws+FFki4AAGAMki4AAIDSZ7HYV9JFI/1t7Pjx4/Lw8JC/v7/8/f3l5+cnHx8fzZw5s9jnfPPNN5WSklJofVhYmGJjY4sUm4+PT7HjAADA3lDpus25u7srLi7OupyamqqOHTuqa9eueuCBB4p8vu3bt2vw4MG3MkQAAIqH4UXcztLT02WxWFSpUiVJ0rx587RmzRrl5eXpySefVGhoqFJSUjRkyBDVr19fycnJcnV1VWRkpKKjo5WWlqZBgwZp8eLFqlq16lWv4ePjo27duunbb79VVlaWJk+eLA8PD+3bt0/h4eGSpIYNG1r3P3XqlMaOHavff/9dJpNJb7zxhry8vDR06FA98MADGjZsmObOnavk5GRFRkaW/kMCANwZ7CzpYnjxNpeWliZ/f3916tRJrVq10owZMzR79mzVqFFDmzZtUlJSkpYuXarly5crNTVVK1askCTt379fzz33nFauXClnZ2fFx8dr0KBBcnd317x5866ZcF3h4uKipUuXKigoSB999JEkadSoURoxYoSWLVum++67z7rvxIkT1aNHD8XGxurDDz/U2LFjlZmZqXHjxik2NlYJCQmKjo7W+PHjS+9BAQDuOJZ8S4l+dxoqXbe5K8OL+fn5eu+99/Trr7/qiSeekCRt3bpVe/fuVWBgoCQpOztbNWvW1KOPPipXV1c1btxYklS/fn2dO3euSNdt3bq19djExESdPn1aaWlp1msHBgYqJiZGkrRlyxYdOnTI2muWm5urY8eOqVGjRgoLC1NISIg++ugjubi4lPyBAABwhyLpukM4ODho5MiR6t69uz755BMFBwcrLy9PAwcO1HPPPSdJysjIkNls1pkzZ+Tk5GQ91mQyFfkNkSvHm0ymq57DbDZb/zk/P18LFiywJlVpaWlydXWVJB06dEiurq5KSkpSmzZtin7jAICy6w6sVpUEw4t3EEdHR40cOVJz5sxRenq6PD09FRcXpwsXLig3N1eDBw9WQkLCdc9hNpuVl5dX5GtXrVpVNWvW1DfffCNJWrlypXWbp6enoqKiJEkHDx6Un5+fsrKylJycrGXLlik2NlaxsbHav39/ka8LACjD8kv4u8OQdN1hvL291bx5c0VGRsrHx0cdOnRQr1695Ovrq4YNGyogIOC6x7dp00aDBg3SsWPHinztKVOmaPbs2erevbuOHj1qXT9mzBjt2bNHfn5+Gj58uCIiIuTk5KSwsDC9+eabqlGjhkaOHKlRo0YpJyenyNcFAJRN9tbTZbLY28xkuO1lff2x0SGUeY4PMcdaqcvjLxi2MKDlKKNDKPM+P7K81M59tnfbEh3v8vnXtygS26DSBQAAYAM00gMAAGPcgX1ZJUGlCwAAGMKWPV3x8fHq0qWLOnTooMWLFxfaPnv2bLVt29b66b0r+yQnJyswMFAdO3ZUeHi4cnNzi32/VLoAAIAxbFTpSk1N1fTp0xUbG6vy5csrKChIrVq1Ur169az7JCUladq0aWrevHmBY0NDQzVhwgQ1a9ZMo0ePVnR0tPr06VOsOKh0AQCAO1JGRoaOHz9e6JeRkVFgvy1btsjT01MuLi6qWLGiOnbsqLVr1xbYJykpSR999JH8/Pz09ttv688//1RKSoqys7PVrFkzSZcnBv/7cUVBpQsAABiipNM+LFiwQLNnzy60fsiQIRo6dKh1OS0tTW5ubtZld3d37d2717p84cIFNWrUSKGhobr//vsVFhamOXPmqE2bNgWOc3NzU2pqarHjJekCAADGKOHw4sCBA686P6Wzs3PBy+TnW7+wIkkWi6XAcqVKlfSf//zHuvz8889r9OjR8vb2vu5xRUXSBQAADGEpYdLl7OxcKMG6mho1amjXrl3W5fT0dLm7u1uXT5w4oS1btqhnz56X47JY5OjoqBo1aig9Pd2636lTpwocV1T0dAEAAGPY6DNAXl5e2rp1q06fPq2srCwlJibK29vbur1ChQqaMmWKjh07JovFosWLF6t9+/aqVauWnJyc9P3330uS4uLiChxXVFS6AABAmVa9enUNHz5cAwYMUE5Ojnr27KmmTZsqODhYISEheuihh/T222/rlVdeUU5Ojh555BE999xzkqSpU6dqzJgxyszMVJMmTTRgwIBix8FngHDb4TNApY/PANkAnwGyCT4DVPpK8zNApzo/VaLj71mz8RZFYhtUugAAgDHsbEZ6ki4AAGCIkjbS32lIugAAgCHsLeni7UUAAAAboNIFAAAMYW+VLpIuAABgDEvxZ3e/E5F0AQAAQ9hbpYueLgAAABug0gUAAAxhyWd4EQAAoNTZ2/AiSRcAADCEhUZ6AACA0mdvlS4a6QEAAGyAShcAADAEjfQAAAA2YLEYHYFtkXQBAABDUOkCAACwAXtLumikBwAAsAEqXQAAwBD0dAEAANiAvQ0vknQBAABD2NuM9PR0AQAA2ACVLgAAYAh7+wwQSRcAADBEvp0NL5J0AQAAQ9hbTxdJFwAAMIS9vb1IIz0AAIANUOkCAACGYHJUAAAAG7C34UWSLgAAYAjeXgQAALABe3t7kUZ6AAAAG6DSBQAADEEjPQAAgA3Q0wUAAGAD9HQBAADglqPSBQAADEFPF2AwB7f7jQ6h7MvLMTqCss9czugI7EJK7nmjQ0AJ0NMFAABgA/bW00XSBQAADGFvlS4a6QEAAGyAShcAADCEnfXRk3QBAABj2NvwIkkXAAAwBI30AAAANpBvdAA2RiM9AACADVDpAgAAhrCI4UUAAIBSl29nry+SdAEAAEPk21mli54uAAAAG6DSBQAADGFvPV1UugAAgCHyS/grivj4eHXp0kUdOnTQ4sWLC21ft26d/P391a1bN7366qs6d+6cJGnZsmV68skn5e/vL39/f02fPr04tyqJShcAADCIrSpdqampmj59umJjY1W+fHkFBQWpVatWqlevniQpMzNT48aNU0xMjKpXr67IyEjNmjVLY8aMUVJSksLCwuTr61viOKh0AQAAQ5S00pWRkaHjx48X+mVkZBS4zpYtW+Tp6SkXFxdVrFhRHTt21Nq1a63bc3Jy9NZbb6l69eqSpAYNGujkyZOSpB9//FHLli2Tn5+fRowYYa2AFQdJFwAAuCMtWLBA7dq1K/RbsGBBgf3S0tLk5uZmXXZ3d1dqaqp1uWrVqmrfvr0kKTs7W/PmzdPTTz8tSXJzc9Orr76qFStW6N5779Xbb79d7HgZXgQAAIYo6WeABg4cqICAgELrnZ2dC14nP18m0/+GMi0WS4HlK86fP6/BgwerYcOG1vN+8MEH1u0vvviiNTkrDpIuAABgiJL2dDk7OxdKsK6mRo0a2rVrl3U5PT1d7u7uBfZJS0vTCy+8IE9PT40ePVrS5SQsJiZGzz777OV4LRaZzeZix8vwIgAAMES+qWS/m+Xl5aWtW7fq9OnTysrKUmJiory9va3b8/Ly9PLLL6tz584KDw+3VsEqVqyojz/+WHv27JEkLVq0iEoXAAC489hqRvrq1atr+PDhGjBggHJyctSzZ081bdpUwcHBCgkJ0e+//659+/YpLy9PCQkJkiQPDw9NnDhRM2bM0Lhx45Sdna06deooIiKi2HGYLBaLnX35CLe7P5O+MjqEMs/B7R9Gh1D2mcsZHYFdaPdwsNEhlHmbUtaX2rnjavQp0fH+v0fdokhsg0oXAAAwhL1VfUi6AACAIUr69uKdhqQLAAAYIv8q0zaUZby9CAAAYANUugAAgCHo6QIAALABeroAAABsoCgTnJYFJF0AAMAQtpoc9XZBIz0AAIANUOkCAACGoJEeAADABujpAgAAsAHeXgQAALABextepJEeAADABqh0AQAAQ9DTBQAAYAP0dAEAANiAvSVd9HQBAADYAJUuAABgCAs9XQAAAKXP3oYXSboAAIAhSLoAAABsgMlRAQAAcMtR6QIAAIZgclQAAAAboKcLAADABuwt6aKnCwAAwAaodAEAAEPY29uLJF0AAMAQ9tZIf9PDi5mZmRo/frx8fX3l7++v/v3766effirWRaOjo7Vy5cqb3j8zM1OBgYHy9fXV4cOHretjY2MVFhZWaP/169crMjLymue70fabNWvWLD3xxORMp9sAACAASURBVBPy9/eXv7+/OnbsqOnTp0uStm/frv79+5fo/MePH5eHh4f1/H5+fvLx8dHMmTOLfc4333xTKSkphdaHhYUpNja2SLH5+PgUOw4AAPJL+LvT3FSlKz8/X8HBwWrVqpWWL18uR0dHbdu2TcHBwVq1apWqVq1apIvu3r1bLVu2vOn9k5OTVb58+ZtOCtq1a6d27doVe3tRBAUFaejQoZKkixcvqkuXLmrRooXKly9/S87v7u6uuLg463Jqaqo6duyorl276oEHHijy+bZv367BgwffktgAACgJhhevYvv27Tp58qRCQkLk4HC5OObp6alJkyYpP/9yrjl37lytWLFCZrNZTzzxhEJDQ5WVlaXXX39dp06dkiQNHjxYd911lzZs2KBt27bJzc1NrVu3tl7n1KlTCg8P14kTJ+To6Kjhw4erSZMmGj16tE6dOqWXX35Zc+fOvWG8sbGx2rFjh9q3b68vv/zSeszChQt15MgRNW7cWDt27NB7770nHx8fdevWTd9++62ysrI0efJkeXh46Oeff1ZYWJjy8vLUokULbdq0SV999dV1r1uxYkU1bdpUv/zyi5o0aWJd379/fw0ZMkStWrXS8ePHNWDAAG3YsEHx8fH6+OOPZTabdd9992nKlClycnK67jXS09NlsVhUqVIlSdK8efO0Zs0a5eXl6cknn1RoaKhSUlI0ZMgQ1a9fX8nJyXJ1dVVkZKSio6OVlpamQYMGafHixddMlq/1TPbt26fw8HBJUsOGDQv8exs7dqx+//13mUwmvfHGG/Ly8tLQoUP1wAMPaNiwYZo7d66Sk5NvSYURAIA70U0NL+7bt08NGza0JlxXPPXUU3J1ddXGjRu1YcMGxcTEaNmyZTpy5IiWLFmir776SrVq1VJsbKwmTpyoXbt2ycvLSz4+PgoJCSmQcEnSO++8I09PT8XHx2vmzJkaPXq0LBaLJkyYIA8Pj5tKuP7K29tbSUlJOnfunCRp1apV6tatW6H9XFxctHTpUgUFBemjjz6SdHm47bXXXlNcXJxq166tvLy8G14vJSVFu3fv1sMPP3xT8c2YMUPz589XbGysatWqpUOHDhXaJy0tTf7+/urUqZNatWqlGTNmaPbs2apRo4Y2bdqkpKQkLV26VMuXL1dqaqpWrFghSdq/f7+ee+45rVy5Us7OzoqPj9egQYPk7u6uefPm3bA6ebVnMmrUKI0YMULLli3TfffdZ9134sSJ6tGjh2JjY/Xhhx9q7NixyszM1Lhx4xQbG6uEhARFR0dr/PjxN/VcAAD2IV+WEv3uNDeVdDk4OFy3ArNt2zZ17dpVd911lxwdHdWjRw9t3bpVzZs317p16/Tqq6/qxx9/vOGw1rZt29SzZ09JUu3atfXwww9rz549RbidgsqVK6f27dsrMTFRJ06c0NmzZ9W0adNC+11J/urXr6+zZ8/q7NmzSklJ0VNPPSVJ6tGjxzWvsWTJEmu/1eDBg/Xyyy/r0Ucfvan42rZtq969eysiIkIdO3ZUo0aNCu1zZXhx9erV8vf3l8Vi0RNPPCFJ2rp1q/bu3avAwEAFBAQoKSlJBw8elCS5urqqcePG1vu6knjerL8/k9OnTystLc167cDAQOu+W7Zs0cyZM+Xv76/g4GDl5ubq2LFjcnV1VVhYmEJCQjR27Fi5uLgUKQYAQNlGT9dVeHh4KCoqShaLRSbT/141mDZtmry8vKxDjH+Vm5urOnXqaM2aNdq8ebO+/vprzZ8/X6tXr77mdSwWS6Hlm6kwXY+/v78iIyN17tw5+fn5XXWfKwnllXszm82FYrmWv/Z0Xc+V8+Xm5lrXjRkzRvv379fGjRsVGhqqIUOGyN/f/6rHOzg4aOTIkerevbs++eQTBQcHKy8vTwMHDtRzzz0nScrIyJDZbNaZM2cKJMkmk+mm7+eKvz+Tv5/DbDZb/zk/P18LFiywJlVpaWlydXWVJB06dEiurq5KSkpSmzZtihQDAKBsu/NqVSVzU5WuFi1ayNXVVbNnz7YmQZs3b1ZsbKzq1asnT09PrVq1StnZ2crNzVVMTIw8PT21aNEizZo1S507d9Zbb72l06dPKzMzU2az+arJlKenp5YuXSpJOnbsmHbv3q1mzZqV6AabNWumtLQ0xcXFXXVo8WqqVKmi2rVra+PGjZKk+Pj4EsVQtWpVawVq3bp1ki4nXx06dFDVqlX10ksvyd/fX8nJydc9j6Ojo0aOHKk5c+YoPT1dnp6eiouL04ULF5Sbm6vBgwcrISHhuue41rO/mXuoWbOmvvnmG0kq8Papp6enoqKiJEkHDx6Un5+fsrKylJycrGXLlik2NlaxsbHav39/ka8LACi77K3SdVNJl8lk0pw5c3T06FH5+vrKz89P//nPfzRv3jzdc889atu2rdq0aaMePXqoa9euqlmzpvr166fu3bvr8OHD8vPzU9++fRUaGipnZ2d5eXlp7ty5Wrt2bYHrhIeHa9u2bfLz89Orr76qCRMmyN3d/bqxxcfHq3nz5tbf1fq+OnfurEqVKql27do3/WAiIiI0Z84cBQQEaO/evapQocJNH/t3L774oqKiohQQEKDs7GxJlxOokJAQPf/88woMDNSePXsUHBx8w3N5e3urefPmioyMlI+Pjzp06KBevXrJ19dXDRs2VEBAwHWPb9OmjQYNGqRjx44V+T6mTJmi2bNnq3v37jp69Kh1/ZgxY7Rnzx75+flp+PDhioiIkJOTk8LCwvTmm2+qRo0aGjlypEaNGqWcnJwiXxcAgLLAZCnquJOdmD17tnr16iV3d3clJiYqPj5es2bNMjosu/Bn0vXfEkXJObj9w+gQyj5zOaMjsAvtHr7xX1ZRMptS1pfaucfW6Vui49/+bfEtisQ2mJH+GmrWrKnnn39ejo6OcnZ21sSJE40OCQCAMuVOfAOxJEi6riEwMLDAG3oAAODWsq+UqwifAQIAAEDxUekCAACGuBPfQCwJki4AAGAIeroAAABswL5SLpIuAABgEHsbXqSRHgAAwAaodAEAAEPQ0wUAAGAD9pVykXQBAACD2FtPF0kXAAAwhMXOal000gMAANgASRcAADBEfgl/RREfH68uXbqoQ4cOWrx4caHtycnJCgwMVMeOHRUeHq7c3FxJ0okTJ9S3b1916tRJr7zyii5cuFCcW5VE0gUAAAySL0uJfjcrNTVV06dPV1RUlJYvX64vvvhCBw8eLLBPaGioxo4dq4SEBFksFkVHR0uSxo8frz59+mjt2rXy8PDQnDlzin2/JF0AAMAQlhL+MjIydPz48UK/jIyMAtfZsmWLPD095eLioooVK6pjx45au3atdXtKSoqys7PVrFkzSVJgYKDWrl2rnJwc7dy5Ux07diywvrhopAcAAHekBQsWaPbs2YXWDxkyREOHDrUup6Wlyc3Nzbrs7u6uvXv3XnO7m5ubUlNTdebMGVWuXFmOjo4F1hcXSRcAADBESSdHHThwoAICAgqtd3Z2Lnid/HyZTCbrssViKbB8re1/309SoeWiIOkCAACGKOk8Xc7OzoUSrKupUaOGdu3aZV1OT0+Xu7t7ge3p6enW5VOnTsnd3V3VqlXT+fPnlZeXJ7PZXOi4oqKnCwAAGMJSwv/dLC8vL23dulWnT59WVlaWEhMT5e3tbd1eq1YtOTk56fvvv5ckxcXFydvbW+XKlVOLFi20evVqSdLy5csLHFdUJF0AAMAQtpoyonr16ho+fLgGDBig7t27y9fXV02bNlVwcLB+/PFHSdLUqVM1adIkderUSRcvXtSAAQMkSW+99Zaio6PVpUsX7dq1S8OGDSv2/ZosFot9TQeL296fSV8ZHUKZ5+D2D6NDKPvM5YyOwC60ezjY6BDKvE0p60vt3M/X6Vmi4+f/tvQWRWIb9HQBAABD2NtngEi6AACAIfjgNQAAgA3k21mHE0kXAAAwhH2lXLy9CAAAYBNUugAAgCFKOiP9nYakCwAAGIK3FwEAAGzA3t5epKcLAADABqh0AQAAQ9DTBQAAYAP0dAEAANiAvfV0kXQBAABDWOxsRnoa6QEAAGyAShcAADAEjfQAAAA2QE8XAACADfD2IgAAgA3Y2/AijfQAAAA2QKULAAAYwt6mjCDpAgAAhqCRHgAAwAbsrZGeni4AAAAboNIFAAAMYW9vL5J0AQAAQ9BIDwAAYANUugCDVXrkWaNDKPOeufcxo0Mo81Jyzxsdgl1Yv+c/RoeAEqCRHgAAALcclS4AAGCIfHq6AAAASp99pVwkXQAAwCA00gMAANiAvSVdNNIDAADYAJUuAABgCCZHBQAAsAF7G14k6QIAAIZgclQAAADcclS6AACAIejpAgAAsAF6ugAAAGyAShcAAIAN2Fuli0Z6AAAAG6DSBQAADGFvU0aQdAEAAEPk09MFAABQ+qh0AQAA2IC9VbpopAcAALABKl0AAMAQDC8CAADYgL0NL5J0AQAAQ9hbpYueLgAAABug0gUAAAxh9PDiiRMnFBoaqj/++EN169bV1KlTValSpQL7pKWl6c0339SpU6fk4OCgkSNH6vHHH1dOTo5atWql2rVrW/eNjY2V2Wy+5vVIugAAgCGMHl4cP368+vTpo65du+qDDz7QnDlzFBoaWmCfiIgI+fj4qG/fvjp06JD69++vTZs26cCBA2revLk++eSTm74ew4sAAMAQFkt+iX4ZGRk6fvx4oV9GRsYNr52Tk6OdO3eqY8eOkqTAwECtXbu20H7t27eXr6+vJOn+++/Xn3/+qYsXL+rHH3/U6dOnFRgYqF69emnHjh03vCaVLgAAYIj8Ela6FixYoNmzZxdaP2TIEA0dOvS6x545c0aVK1eWo+PlVMjNzU2pqamF9ruSlEnSJ598okaNGqlKlSoymUxq166dXnrpJf3yyy8KDg5WfHy8qlWrds1rknQBAIA70sCBAxUQEFBovbOzc4HlNWvWaNKkSQXW3X///TKZTAXW/X35rz799FN98cUXWrRokSQpKCjIuq1x48Zq2rSpdu/eraeffvqa5yDpAgAAhrCUsJHe2dm5UIJ1NZ07d1bnzp0LrLvSCJ+Xlyez2az09HS5u7tf9fiIiAht3LhRixcvVo0aNSRJy5cv1yOPPKJ//OMf1nspV67cdeOgpwsAABgiX5YS/UqiXLlyatGihVavXi3pchLl7e1daL9PP/1U27dv1+eff25NuCTpwIEDmj9/viTp0KFDSk5O1qOPPnrda5osJU0zgVvMsXwto0Mo85659zGjQyjzUnLPGx2CXVi/5z9Gh1Dmlbvnn6V27lpVm5To+JQzP5Xs+JQUhYWF6Y8//tC9996radOm6e6779bnn3+utLQ0hYSEqGXLlqpcuXKBitq8efNUqVIljR49WocOHZLJZFJ4eLg8PT2vez2SLtx2SLpKH0lX6SPpsg2SrtJXlpMuW6OnCwAAGMLoyVFtjaQLAAAYwujJUW2NpAsAABjC3jqcSLoAAIAhSvoG4p2GKSMAAABsgEoXAAAwBMOLAAAANsDbiwAAADZApQsAAMAGaKQHAADALUelCwAAGILhRQAAABugkR4AAMAG7O0zQPR0AQAA2ACVLgAAYAiGFwEAAGyARnoAAAAboKfrL7Zv367+/fuX2sVnzZqlWbNmFVrv4+OjLl26yN/fX926dVNgYKC2bdtW4uvdqvuJjIzU+vXrS3ye/v37q3379tb79PPz0+rVq0t83qIqznPp37+/tm/fXkoRAQDsgcViKdHvTnPbVrrmzZun++67T5K0YcMGjRgxQt9++63BUV322muv3bJzTZgwQa1atZIkHThwQD179lTr1q1VpUqVW3YNAABgvGInXXPnztWKFStkNpv1xBNPKDQ0VGazWZ9++qk+//xzmc1mtW3bVqGhofr555/1zjvv6OLFizp9+rQGDRqk3r173/S1WrVqpfT0dJ05c0aTJ09Wy5YtFRgYKElq0KCBDhw4oFmzZumHH37QyZMn1a9fPzVv3lxjx45Vdna27r77bk2dOlWSdPr0aQUHB+vo0aOqW7euZs6cqfLly2v69OnaunWrzp07J3d3d02fPl133323Ro8erV9++UWS1KdPH/Xq1UthYWFq2bKlWrZsqSFDhqh+/fpKTk6Wq6urIiMj5eLiotWrV2vmzJmqWLGiGjVqpLy8PL333nvXvc8GDRqoYsWKOnLkiB544AGNGTNGBw4ckMlk0gsvvCBfX1+1bt1aX331lSpXrqygoCD5+Pho0KBBWrlypb7//nuNGTNGERER2rFjh/Ly8hQYGKhnn31W27dv15QpU5Sfn6/69etr8uTJV41h1qxZSk1N1ZEjR5SSkqJnnnlGr7zyii5duqTw8HAlJSWpVq1aOnPmjPWYefPmac2aNcrLy9OTTz6p0NBQbdiwQREREVqxYoV+//139e/fX19++aWqV69+0//eAQBl251YrSqJYiVdGzdu1IYNGxQTE6Ny5cpp6NChWrJkiR566CFFRUUpJiZGd911l1588UUlJSUpLi5Or776qh5//HEdO3ZM3bp1K1LStXLlStWpU0dVq1a97n6XLl2yDs917dpVI0aMUNu2bRUVFaUFCxaoTZs2OnHihObOnatatWqpV69e2rJli+rWratDhw5pyZIlcnBw0MiRI7VixQp5eHjo3LlzWr58uVJTU/X++++rV69eBa65f/9+vfvuu2rcuLGGDh2q+Ph4de3aVe+++65iYmLk5uamkJAQVa5c+Yb3uXnzZklS3bp1NWvWLFWtWlUrV67U6dOn9cwzz6hhw4by9PTUzp071bJlS504cUI7d+7UoEGDtHnzZnXp0kXR0dGSpGXLlunSpUt64YUX5OHhIUn67bff9PXXX9+winbgwAEtXrxY58+f19NPP62+ffvqyy+/lCStWbNGv/32m7p16yZJ2rRpk5KSkrR06VKZTCaFhoZqxYoV8vf3V2JioubOnavt27dr1KhRJFwAgALsK+UqZtK1bds2de3aVXfddZckqUePHlq+fLmys7PVtm1b6x/qn376qSSpUaNG2rx5sz766CP9/PPPunjx4g2vMWjQIJUrV045OTm69957NWPGjBse07RpU0mXq1np6elq27atpMsVKuly71LDhg1Vu3ZtSdIDDzygM2fOqE2bNho1apS+/PJLHT58WD/88IP+8Y9/qH79+jp8+LBeeOEFeXt7a+TIkYWu6erqqsaNG0uS6tevr3PnzmnXrl1q3ry5Ncno3r271q1bd9WYx4wZo4oVKyovL0933323ZsyYoUqVKmnbtm169913JUnVqlVTu3bttGPHDj311FPaunWrHBwcrD1gOTk52rVrl95++22FhoYqOTnZ2gN38eJFHThwQPXq1VPdunVvatiyVatWKl++vFxdXeXi4qLz589rx44d+te//iVJqlOnjpo3by5J2rp1q/bu3WutPGZnZ6tmzZqSpPDwcHXp0kWPPPKIunbtesPrXpF7KeWm9wUA3Lns7b/3xUq68vPzC63Lzc2Vo6OjTCaTdV1qaqruuusuhYeHy9nZWW3btlWXLl20cuXKG17jrz1df2UymazlyJycnALbKlSoIEkqV65cgTj+/PNPpaWlSZIcHf93y1fOlZSUpDfeeEPPPvusOnbsKAcHB1ksFlWtWlWrVq3Sd999p40bNyogIECrVq0qcE0nJ6dC53NwcLjqM7qav/Z0/dXfS64Wi0V5eXny9vbWf//7X5nNZj3++OM6dOiQli5dqgcffFBOTk7Ky8tTaGioOnToIOlyAlqpUiX98MMP1udzI1e7p78+d+l/zzEvL08DBw7Uc889J0nKyMiQ2WyWJJ06dUpms1mHDh3Sn3/+WeC8AADYm2LNSO/p6alVq1YpOztbubm5iomJkaenp1q0aKGNGzfqwoULys3N1RtvvKGkpCR99913CgkJ0dNPP61NmzZJuvyHdXG4uLjo4MGDknTN6lGVKlVUvXp1a+N9XFycIiMjr3nOK8N1vXv3Vp06dfTNN98oLy9P69evV2hoqNq0aWOtSJ08efKGMT7yyCP68ccflZaWJovFotWrVxdIAm+Gp6enli5dKuly4rR+/Xq1bNlS1apVU4UKFfT111/r0Ucflaenp+bMmWOt6nl6eio6Olo5OTm6cOGC+vTpox9++KFI176axx9/XPHx8crPz1dKSop2795tvV5cXJz13/ngwYOVkJCgvLw8vfnmmwoPD1fLli2v+/wBALAHN6x0XRkqu8LPz09vv/22kpOT1aNHD+Xm5urJJ59Uv3795OjoqH79+ikoKEj5+flq3769vLy8NHToUPXp00dOTk5q2LChatWqpePHjxcr4N69e2vYsGHy8/OTp6en3NzcrrrflClTNG7cOE2ZMkVVq1ZVRESEDh8+fNV9u3TpoiFDhsjPz0+S5OHhoePHj2vw4MFKTExU165d5eTkpG7duqlBgwY3jLFatWoaM2aMnn/+eZUvX1733XefnJ2di3SfgwcP1rhx4+Tn56e8vDy9/PLLatKkiSTJ29tbGzduVKVKleTp6al3331XTz31lCQpKChIR44cUUBAgHJzcxUYGKhWrVqVeHqHPn366JdfflHnzp1Vq1YtPfjgg5IuT++xf/9+9erVS3l5eWrdurUCAgL08ccfy9XVVR06dJCXl5d8fX3VoUMHNWvWrERxAABwpzJZ7O3VARs4c+aMFi5cqCFDhsjBwUETJkzQ/fffX6pzngEAgNvbbTtP153MxcVFGRkZ8vX1ldlsVpMmTQq99QgAAOwLlS4AAAAbKFYjPQAAAIqGpAsAAMAGSLoAAABsgKQLKIasrKxC61JS7GtmZSMUd34/FA3P+da6Mjn3X+3du9eASGA0ki6gGLp3715g0tmoqCjrZ5JwawwfPlznz5+3Lu/fv189e/Y0MKKyiedc+p555hmtWbNG0uVvBE+ZMkXDhg0zOCoYgSkjgGKYOHGi3nzzTfn4+Gjfvn1ycnKyfmwct0aDBg0UEBCgsWPHWj+qPmrUKKPDKnN4zqXvs88+0+jRo5WQkKBff/1VrVq10ooVK4wOCwZgygigmKKiojRlyhRVqlRJc+fOlYeHh9EhlTmJiYkKCQnRPffco5iYGOtH5HFr8ZxL16VLlzR37lx9+eWXMpvNGjNmjJ5++mmjw4IBGF4EiqF///5KSEhQfHy83n//fb3xxht67733jA6rTImMjNS7776radOmKSgoSP369dOGDRuMDqvM4TmXPj8/P508eVKrV6/W/Pnz9fHHH2vIkCFGhwUDMLwIFEOHDh2sn3W67777FBsbq6lTpxocVdny66+/KjY2VtWqVZMktWvXzjqki1uH51z6Ro0aZX2eVapUUVRUlObPn29wVDACw4tAMX3//ff6+eef1aNHD+3Zs0ePPfaY0SGVORcvXtTRo0fVoEEDZWVlqXz58nJ05O+KtxrPufTFx8fr4MGDevnll5WQkKDu3bsbHRIMwPAiUAwLFizQjBkz9Omnn+rChQsaO3asPvnkE6PDKlO2bt0qf39/vfrqqzp16pTatm2rbdu2GR1WmcNzLn1Tp07Vxo0blZiYqLy8PMXExNCOYKdIuoBiWLZsmT755BPdddddqlq1qpYuXaqYmBijwypTpk2bpqioKDk7O8vNzU2LFy9WRESE0WGVOTzn0vftt99qypQpcnJyUuXKlfXf//5XmzZtMjosGICkCygGBwcHlS9f3rrs5OQks9lsYERlT35+vtzc3KzL9erVMzCasovnXPocHC7/UWsymSRdfpvxyjrYFwbtgWJo2bKlJk+erKysLK1bt05ffPGFPD09jQ6rTKlRo4a+/vprmUwmZWRkaPHixapZs6bRYZU5POfS16lTJw0bNkznzp3Tp59+qhUrVsjX19fosGAAGumBYsjPz1d0dLS2bNmi/Px8eXp6KigoiObjW+iPP/7QxIkTCzzjMWPGyN3d3ejQyhSes21s3ry5wDNu27at0SHBACRdQBGcOHHiutupEAC4YufOndfdzhvP9oekCygCHx8fmUwm/fnnn/rjjz9Uu3ZtOTg46OjRo6pdu7YSEhKMDvGOd+UZX8v69ettGE3ZxXMufVfm8jt79qyOHTum5s2by8HBQf/3f/+nBx98UEuWLDE4QtgaYyFAEVyZqXv48OHq27evWrRoIUnau3evPv74YyNDKzMWLlwoi8WiDz74QLVr11ZgYKDMZrPi4+N1/Phxo8MrM3jOpW/hwoWSpODgYM2ePVv333+/JCklJUVjx441MjQYhKQLKIZff/3VmnBJUtOmTXX48GEDIyo7atWqJUk6cOCAJk2aZF3//PPPKzAw0Kiwyhyes+2cOHHCmnBJl9sQbtSqgLKJpAsohho1aigyMlJdunSRxWJRXFyc6tSpY3RYZc7WrVv1+OOPS5I2btzItBylhOdcupo0aaJRo0apc+fOslgsio+PL/CXNtgPerqAYjh37pxmzpypHTt2SJK8vLw0dOhQVa5c2eDIyo59+/Zp1KhRSk9Pl8ViUa1atRQREcE8UrcYz7n0Xbp0SYsWLSrw34s+ffrwtrMdIukCcFs7c+aMTCaTXFxcjA6lTOM5l67MzEydP39ef/0jl7ed7Q9pNlAMsbGxmjx5sjIyMiRJFotFJpNJycnJBkdWduzbt09z587VuXPnCvxB9dlnnxkYVdnDcy59c+fO1bx58+Ti4iKTyWT97wVviNofki6gGObMmaOFCxfqwQcfNDqUMmvUqFH617/+pfr16193agOUDM+59C1dulTr1q1TtWrVjA4FBiPpAorB3d2dhKuUVahQQf369fv/9u40KKorfQP4040C2syoaHCCQMpkVIzMuCQCLrSAKUZjmh214qA1JmEYBkVTSdDByaTGcmcqYrs7KoPGJTrIogQDbqFQFoM6yKKkrDIuRFFiG9oFevl/SNklfxOT7nT3wXufsjYnjAAAGQlJREFU3yfuuX546qRSvJz33HNEx5A8zrPjPf/88+jVq5foGNQFcE8XkQ2WLFmCGzduYNy4cXBzc7OMR0dHC0wlLVlZWfD09MT48eM7zTH3wdgX59nx/v73v+PixYsICgqCq6urZTw1NVVgKhKBK11ENmhra4NKpcLZs2c7jbPosp/8/HwAwPbt2y1j3Adjf5xnx+vfvz/69+8vOgZ1AVzpIiIiInICrnQRWWHhwoVPff/4yd5km7y8vKe+52qifXCeHS8xMfGpHyfwC1H5YdFFZIXAwEDRESSvsrLyqe9ZDNgH59nx5syZIzoCdTFsLxIRERE5gVJ0ACIiIiI5YNFFRERE5AQsuoioS9LpdE+MXbt2TUASaeM8O95PfYBD8sGN9EQ2iIiIgNFotDwrFAq4u7vjxRdfRHp6OgYMGCAw3bOtubkZZrMZSUlJ2LJli+U+QKPRiHfeeQfFxcWCE0oD59l5Ll68CL1eD5VKJToKCcaii8gGarUaPj4+iI+PBwAUFBSgtrYW4eHhyMjIQHZ2ttiAz7A1a9agsrISN2/exIwZMyzj3bp1Q2hoqLhgEsN5dh6lUomwsDAMHDiw06n/PDJCfvj1IpENYmJicODAgU5jsbGxyM3N/cF3ZL3NmzcjKSlJdAzJ4zw7XlVV1Q+O8wga+eFKF5ENlEolysrKEBISAgAoKyuDq6srbt26BYPBIDidNEybNg2ffPIJ7ty5g8f/NuR9dfbFeXa8wMBAfPnll7h48SLi4uJw7tw5jB49WnQsEoArXUQ2uHjxIhYsWGDZcOzn54fly5ejuLgY3t7eiImJEZzw2fenP/0Jv/rVrzBo0KBOp3qzGLAvzrPj/ec//0FpaSlu3ryJPXv24M0330R8fDzeeust0dHIyVh0Ef0COp0OLi4u8PDwEB1FcjQaDQoLC0XHkDzOs+NFR0fj008/xdSpU5GXlwe9Xo+EhAQUFRWJjkZOxvYikQ3q6+uxceNG6HS6Ti0Zboy1n6FDh6KxsRH+/v6io0ga59nxlEolXF1dLc9ubm5wcXERmIhEYdFFZIP09HRMmzbtiZYM2U9TUxNiYmLQt29fuLm5wWw2Q6FQ4MiRI6KjSQrn2fECAwOxYsUK3L9/H6Wlpdi7dy+Cg4NFxyIB2F4kskFCQgL27dsnOoak/dgBnTwDzb44z45nMpnw6aef4uTJkzCZTAgODsb06dPRrRvXPeSGRReRDbKysuDp6Ynx48d3OnfH29tbYCrpKSwsxFdffYXk5GQcPnwY0dHRoiNJEufZsRYvXozQ0FAEBQV1ajOS/LDoIrJBeHj4E2NsydhXZmYmvvnmG9TV1WHfvn34y1/+gmHDhmHBggWio0kK59nxDh06hLKyMpw+fRpDhgxBWFgYJkyYgOeee050NHIyFl1E1CVFR0fjwIEDiImJQV5eHgwGAyIjI/nFl51xnp3HYDBg//79WL9+PVpaWtDQ0CA6EjkZG8pEVtBqtZgzZ86PXmC7bNkyJyeSLqVSCQCWDxXa29stY2Q/nGfH+/e//43q6mo0NTVh6NChePvtt7mRXqZYdBFZYdiwYQB4fYczTJo0CfPmzYNOp0N2djYKCgowZcoU0bEkh/PseKWlpWhuboZGo0FwcDBeeeUV9OjRQ3QsEoDtRSIbLF26FJGRkQgICBAdRdLKyso6ffEVFhYmOpIkcZ4d7969e6iurkZVVRWOHTuGX//619izZ4/oWORkLLqIbLBz504cOnQIOp0OGo0GGo0GPj4+omNJxqVLl6BSqdC/f3/L2O3bt5GVlYV//vOfApNJC+fZOR4VXCdPnkRlZSXc3d2hVquRkpIiOho5GYsuol+gubkZRUVFKCgogEqlwq5du0RHeuZptVps27YNALBu3TqMHTsWW7duxbp16zBy5Ehs3bpVcEJp4Dw7z9ixYzFmzBio1WqEhITA09NTdCQShEUXkY2+++47HD58GEVFRbh58yYmT56Mv/71r6JjPfMmTpyI3bt34+bNm1izZg1MJhNu3LiBDz74ACEhIaLjSQbn2XmMRiO++uorVFdXw2AwIDg4mNcuyRQ30hPZIDk5GXV1dYiIiEBaWhqGDx8uOpJkqFQqeHl5wcvLC//73/8QHR2NTZs28a46O+M8O8/Bgweh1Wrx2muvwWQyISUlBSkpKYiPjxcdjZyMRReRDaZOnQq1Ws1rPBzg8eMK+vTpw0M6HYTz7Dzbtm3Dvn370KdPHwDf/9E2c+ZMFl0yxN8YRFZ4dE5XSUkJSkpKnnjPc7p+uccvEHd3dxeYRNo4z85jMpksBRcAeHp6dpp/kg8WXURW4DldjtfU1ISJEycCAG7cuGH52Ww286olO+I8O8+QIUOwZMkSy8rW/v37uadLpriRnshONm/ejKSkJNExnnnXrl176vsBAwY4KYm0cZ6d58GDB9BqtaioqIDZbEZwcDBSUlLg4eEhOho5GYsuIjsZNWoUampqRMcgoi6kra0NX3/9NQYOHMhT6Am8YIvITvj3CxE97rPPPsPYsWMxe/ZshIeHo6qqSnQkEoxFF5GdcGMsET1uw4YN2L9/PyoqKrBy5UpotVrRkUgwbqQnssLChQt/9F17e7sTkxDZR3t7Oy5dugR/f38UFhaivr4e77zzDk9NtwOFQoHBgwcDAEJCQrBy5UrBiUg0Fl1EVnjaV4v8opGeRe+//z58fHzw8OFDaLVaREVFYeHChdi0aZPoaM+8x89CA8Bz/YhFF5E1YmJiREcgsqurV68iKysLq1atQnx8PJKSkhAXFyc6liTo9XqcPn3ast/z3r17nZ5Hjx4tMh4JwKKLiEjGjEYjWltbUVpaCq1Wi5aWFjx8+FB0LEno378/srKyLM9eXl6WZ4VCgZycHFHRSBAWXURWuHz5Ml544QXRMYjs5q233sLUqVMRHh6OwYMH4w9/+APS0tJEx5KEHTt2iI5AXQzP6SKygkajQWFhIVJSUrB+/XrRcYjszmg08tJrIgdh0UVkhdjYWLi5ueHChQsICAh44j3bBfSsCQ8P/8HjTngNEJH9sb1IZIWcnBw0NDQgIyMDqampouMQ/WKPt8AMBgNKSkp4/AmRg3Cli8gGra2tAIBz587BaDRixIgR6Nevn+BURPYRGxuL3Nxc0TEk4/+f76dQKODu7o6XXnoJCQkJcHV1FZSMnI0rXUQ2qKurw9/+9jeMGDECJpMJH374IZYsWYKwsDDR0YisUl1dbfnZbDajqamJXy/amYuLC3Q6HaKjowEARUVF0Ov1UCqV+Mc//oFly5YJTkjOwqKLyAYff/wxdu3aBV9fXwDAlStXkJqayqKLnjlr1qyx/KxQKNCnTx8sX75cYCLpaWhowH//+1/Lc3h4OBISEpCVlYXIyEiBycjZWHQR2cBgMFgKLgDw9fWFyWQSmIjINjzWwPHu3buHlpYWPPfccwCA27dvW1YTjUajyGjkZCy6iGzg7e2N7OxsxMfHAwD279+PAQMGCE5F9PMlJiY+9ZJ2folrP3PmzEFsbCxGjhwJk8mE8+fPIyMjA1qtFmPHjhUdj5yIG+mJbHD79m0sXrwYFRUVMJvNCA4ORkZGBry8vERHI/pZqqqqnvqed4naV2trK7788ksolUqMHDkSnp6euHPnDnr37i06GjkRiy4iIpkqLy9HU1MThg8fjpEjR4qOI1l3795FYWEh7ty5g8d/5fLYGflR/vQ/ISIiqVm9ejUWLVqEmpoapKWlYdeuXaIjSVZaWhoqKyu575O4p4uISI4OHz6MoqIi9OjRA9euXcOcOXPw5ptvio4lSbdu3cL27dtFx6AugCtdRDYoLy9/Yuzzzz8XkITINm5ubujRowcAYMCAATAYDIITSdfQoUPR2NgoOgZ1AVzpIrJCUVER2tvbsWbNGsydO9cy3tHRgc2bNyMiIkJgOqKf7/9/uchLrh2nqakJMTEx6Nu3L9zc3GA2m6FQKHi/pQyx6CKygl6vR01NDfR6PSorKy3jLi4umD9/vsBkRNZpaWnB2rVrf/SZm7zt5/F5JXnj14tENjh16hTGjBljeW5ra4OHh4fARETW+alCgEXXL3fs2DGEhYUhLy/vB98/uhaI5IMrXUQ2uH//PlatWoWUlBTEx8ejtbUV6enpiI2NFR2N6GdhUeV4tbW1CAsL67Qq/jgWXfLDlS4iG8TFxWHJkiWora3F6dOn8eGHHyIxMRG5ubmioxFRF9bW1obm5mYMGjRIdBQSgF8vEtnI398fx48fR3h4OFQqFTo6OkRHIqIuaN++fViwYAFaW1vx+uuvY+7cudi4caPoWCQAiy4iG/Tr1w+LFy9GbW0tQkJCsHz5cnh7e4uORURd0O7du/Huu+/i4MGDmDhxIgoLC3nEjExxTxeRDf71r3+htLQUs2bNQs+ePeHr68s9MvRMioiIgNFotDwrFAq4u7vjxRdfRHp6Oi9ytxMvLy+cOHECM2fORLdu3fDw4UPRkUgAFl1ENlCpVNDr9cjMzITBYEBQUBB69uwpOhaR1dRqNXx8fBAfHw8AKCgoQG1tLcLDw5GRkYHs7GyxASXgt7/9Lf785z/j6tWrGDNmDObNm4ff/e53omORANxIT2SDFStW4PLly4iLi4PZbEZubi68vb2xaNEi0dGIrBITE4MDBw50GouNjUVubu4PviPrGQwGnDlzBoMGDULv3r1x9OhRTJgwgQfSyhBXuohsUF5ejry8PCiV32+LDA0NhUajEZyKyHpKpRJlZWUICQkBAJSVlcHV1RW3bt3i1UB2cvfuXdTV1aGqqgpmsxkmkwnFxcVYuXKl6GjkZCy6iGxgNBphMBjg6upqeeZfrfQsWrZsGRYsWID33nsPAODn54fly5dj7969mD17tuB00jBv3jw8//zzOHv2LF577TUcP36c7UWZYnuRyAYbN27E8ePHMWXKFADAoUOHEBoaiuTkZMHJiGyj0+ng4uLCmxUcYNKkSSguLsaKFSswadIk+Pn5YdasWSgoKBAdjZyMK11ENkhOTsbLL7+MU6dOwWw2Izk5GaGhoaJjEVmtvr4eGzduhE6nw+N/g+fk5AhMJS29evUCAAwcOBCNjY0YPny44EQkCosuIivpdDoYjUao1Wqo1WpUVlbydGl6ZqWnp2PatGkYNGgQFAqF6DiSFBwcjLlz5yI9PR2zZ89GXV0d3N3dRcciAdheJLJCfX09kpKSsHTpUqjVagDAxx9/jNzcXGzZsgX+/v6CExJZJyEhAfv27RMdQ/K+/vpr+Pn5oa6uDtXV1Xj99dfh5eUlOhY5GYsuIivMmjULKSkpCAoK6jReVlaGrVu38kwjeuZkZWXB09MT48ePh5ubm2WcNyzYT0dHB06ePIlvv/220zgvvJYftheJrHD37t0nCi4ACAkJQWZmpoBERL9Mfn4+AGD79u2WMYVCgSNHjoiKJDlpaWloaWnBSy+91KmFy6JLflh0EVnBYDDAZDJZzud6xGQy8cJreiYdPXpUdATJu3TpEoqLi0XHoC6ARReRFUaPHo21a9di7ty5ncbXr1+PgIAAQamIrKfVajFnzhwsXLjwB98vW7bMyYmky8/PD9evX2fLllh0EVnj3XffRVJSEvLy8uDv7w83NzfU19fD09MTGzZsEB2P6GcbNmwYACAwMFBwEulKTEyEQqFAa2srNBoN/P394eLiArPZDIVCwWM5ZIgb6YmsZDabUVFRgYaGBiiVSgQEBODVV18VHYvIJkuXLkVkZCRXah2gqqrqqe9Z8MoPiy4iIhnbuXMnDh06BJ1OB41GA41GAx8fH9GxJOXGjRvIycnB+++/jytXrkCr1eKDDz5Av379REcjJ2PRRUREaG5uRlFREQoKCqBSqbBr1y7RkSQjMTERU6ZMwfTp09He3o68vDwUFxdj27ZtoqORkyl/+p8QEZGUfffddygvL0d5eTmMRiPGjRsnOpKk6HQ6TJ8+HQDg6uqKqVOnPnFmF8kDN9ITEclYcnIy6urqEBERgbS0NN4L6ADu7u44ceIEJkyYAAA4efIkevToITgVicD2IhGRjB09ehRqtRrduvFvcEdpbGzEe++9h5aWFigUCvzmN7/BqlWreGerDPH/MiIiGXp0TldJSQlKSkqeeM9zuuzn7NmzOHjwIL799lt0794dHh4eoiORICy6iIhkiOd0Oc/OnTsxffp09OnTR3QUEoztRSIi6mTz5s1ISkoSHUMy3n77bbS3t2P48OGdLhVPTU0VmIpE4NeLRETUycaNG0VHkJQRI0YgMDCwU8FF8sSVLiIi6mTkyJE4c+aM6BhEksM9XURE1IlCoRAdQVL8/f2fmFMvLy+cOHFCUCIShUUXEZEMLVy48Efftbe3OzGJ9DU2Nlp+7ujoQGlpKc6ePSswEYnCoouISIae9tUiv2h0nO7du2Py5MncNydTLLqIiGQoJiZGdATZyMvLs/xsNpvR1NTEw2hliv/ViYiIHKiysrLTc58+fbB69WpBaUgkfr1IRCRDly9fxgsvvCA6hizodDr06tVLdAzqAnhOFxGRDD06mDMlJUVwEumqrKzE+PHjERwcjEmTJuHChQuiI5FgXOkiIpKh2NhYuLm54cKFCwgICHjifU5OjoBU0hIXF4fU1FQEBQWhsLAQpaWl2LJli+hYJBD3dBERyVBOTg4aGhqQkZHB62gcxGAwICwsDAAwbdo0FrLEoouISI48PDwwevRo7NmzBwBw7tw5GI1GjBgxAv369ROcThqUys47eFxdXQUloa6Ce7qIiGSsrq4OUVFRyM3NxYEDBxAZGYljx46JjiUJHR0daG5uxvXr13H9+vUnnkl+uKeLiEjGYmNjkZWVBV9fXwDAlStXkJqaivz8fMHJnn3h4eFQKBT4oV+zCoUCR44cEZCKRGJ7kYhIxgwGg6XgAgBfX1+YTCaBiaTj6NGjoiNQF8P2IhGRjHl7eyM7OxttbW1oa2tDdnY2BgwYIDoWkSSxvUhEJGO3b9/G4sWLUVFRAbPZjODgYGRkZMDLy0t0NCLJYdFFRERE5ATc00VEROQAa9eufep7no8mP9zTRUREROQEbC8SEclYeXk5xo0b12ns888/R0REhKBE0mc2m3H16tVOX42SPLC9SEQkQ0VFRWhvb8eaNWswd+5cy3hHRwc2b97MosuO9u7dixUrVuD+/fuWMR8fH5SUlAhMRSKw6CIikiG9Xo+amhro9XpUVlZaxl1cXDB//nyByaRn06ZNyM/Px+rVqzF//nycOHECNTU1omORACy6iIhkKCEhAQkJCTh16hTGjBljGW9ra4OHh4fAZNLTt29f+Pr6YsiQIbh48SJmzJiB3bt3i45FAnAjPRGRjN2/fx+rVq2CXq/H5MmTMXHiROTm5oqOJSk9evRARUUFhgwZgmPHjqGlpQUPHjwQHYsEYNFFRCRj69atg0ajQVFREX7/+9/j6NGj2Llzp+hYkrJo0SIcO3YMISEhuHPnDiZPnow//vGPomORAGwvEhHJnL+/P7RaLSIjI6FSqdDR0SE6kqQMHjwYUVFRUCqVWLp0Kc6fP9+ppUvywZUuIiIZ69evHxYvXoza2lqEhIRg+fLl8Pb2Fh1LUjIzM5GZmQng+3bu+vXrodVqBaciEXhOFxGRjLW1taG0tBSjRo2Cn58fPvnkE0RFRXEzvR298cYbyM/Ph4uLCwDAYDAgJiYGhYWFgpORs7G9SEQkYyqVCnq9HpmZmTAYDAgKCkLPnj1Fx5IUg8GABw8eQKVSAQDbtzLGoouISMZWrlyJy5cvIy4uDmazGbm5ubhy5QoWLVokOppkTJ8+HbGxsQgPDwcAfPHFF5gxY4bgVCQC24tERDIWGRmJvLw8KJXfb/E1GAzQaDT47LPPBCeTltraWlRXV6N79+545ZVX8PLLL4uORAJwpYuISMaMRiMMBgNcXV0tz4/2HpF9mM1m1NbW4syZMzAajTCZTPD397cUuiQfLLqIiGRMo9Fg5syZmDJlCgDg0KFDeOONNwSnkha2cOkRtheJiGTuiy++wKlTp2A2mxEcHIzQ0FDRkSSFLVx6hCtdREQypdPpYDQaoVaroVarUVlZiUGDBomOJTls4dIjbCgTEclQfX09pkyZgvPnz1vGTp48iaioKDQ2NgpMJj2PWrg7duzAjh07MGvWLEs7l+SF7UUiIhmaNWsWUlJSEBQU1Gm8rKwMW7duRXZ2tphgEsUWLgEsuoiIZCkmJgYHDhz4wXdRUVHIz893ciJ5+eijj/DRRx+JjkFOxvYiEZEMGQwGmEymJ8ZNJhNPTHeCgoIC0RFIABZdREQyNHr0aKxdu/aJ8fXr1yMgIEBAInlhk0me2F4kIpKhtrY2JCUl4ZtvvoG/vz/c3NxQX18PT09PbNiwAb179xYdUdJGjRqFmpoa0THIyVh0ERHJlNlsRkVFBRoaGqBUKhEQEIBXX31VdCzJSExMhEKheGLcbDajpqYGdXV1AlKRSCy6iIiIHKCqquqp7wMDA52UhLoKFl1ERERETsCN9EREREROwKKLiIiIyAlYdBERERE5AYsuIiIiIidg0UVERETkBP8HYUwiWthNYYYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 576x432 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Look at the correlation among continous variables\n",
    "import numpy as np\n",
    "np.random.seed(0)\n",
    "import seaborn as sns\n",
    "sns.set()\n",
    "plt.figure(figsize=(8,6))\n",
    "ax = sns.heatmap(X_cont_df.corr())\n",
    "plt.title(\"Correlation Matrix\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Looks like Cost of Living Index largely correlates with Rent Index and negatively correlates with Local Purchasing Power Index. We will keep Cost of Living Plus Rent Index and Local Purchasing Power Index in this model. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Keep Cost of Living Plus Rent Index, Local Purchasing Power and Fill missing values with the mean\n",
    "\n",
    "cont_imp = SimpleImputer(strategy='mean')\n",
    "cont_imp.fit(X_cont_df[['Cost of Living Plus Rent Index','Local Purchasing Power Index']])\n",
    "X_cont = cont_imp.transform(X_cont_df[['Cost of Living Plus Rent Index','Local Purchasing Power Index']])\n",
    "\n",
    "#Standardized inputs\n",
    "\n",
    "sc=StandardScaler()\n",
    "sc.fit(X_cont)\n",
    "X_cont_scaled = sc.transform(X_cont)\n",
    "\n",
    "#Continous Features Dataframe\n",
    "cont_scaled_df = pd.DataFrame(X_cont_scaled, columns = ['Cost of Living Plus Rent Index','Local Purchasing Power Index'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Try unscaled Index\n",
    "con_unscaled_df = pd.DataFrame(X_cont_df, columns = ['Cost of Living Plus Rent Index','Local Purchasing Power Index'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 963 entries, 0 to 962\n",
      "Data columns (total 2 columns):\n",
      "Cost of Living Plus Rent Index    939 non-null float64\n",
      "Local Purchasing Power Index      939 non-null float64\n",
      "dtypes: float64(2)\n",
      "memory usage: 62.6 KB\n"
     ]
    }
   ],
   "source": [
    "con_unscaled_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "#salary_df['City_Bins'] = salary_df['City_Bins'].astype('category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Categorical Features\n",
    "\n",
    "cat_features=['Rate_by','City_Bins']\n",
    "X_cat = salary_df.loc[:, cat_features]\n",
    "\n",
    "# Replace NaNs with 'missing'\n",
    "cat_imp = SimpleImputer(strategy='constant', fill_value='missing')\n",
    "cat_imp.fit(X_cat)\n",
    "X_cat = cat_imp.transform(X_cat)\n",
    "\n",
    "#Encode Categorical Features\n",
    "ohe = OneHotEncoder(handle_unknown='ignore')\n",
    "ohe.fit(X_cat)\n",
    "X_cat_ohe = ohe.transform(X_cat)\n",
    "\n",
    "#Create categorical features dataframe\n",
    "cat_df = pd.DataFrame(X_cat_ohe.todense(), columns=ohe.get_feature_names(input_features=cat_features))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Rate_by_Daily</th>\n",
       "      <th>Rate_by_Hourly</th>\n",
       "      <th>Rate_by_Monthly</th>\n",
       "      <th>Rate_by_Yearly</th>\n",
       "      <th>City_Bins_high</th>\n",
       "      <th>City_Bins_low</th>\n",
       "      <th>City_Bins_medium</th>\n",
       "      <th>City_Bins_missing</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>963.000000</td>\n",
       "      <td>963.000000</td>\n",
       "      <td>963.000000</td>\n",
       "      <td>963.000000</td>\n",
       "      <td>963.000000</td>\n",
       "      <td>963.000000</td>\n",
       "      <td>963.000000</td>\n",
       "      <td>963.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.001038</td>\n",
       "      <td>0.305296</td>\n",
       "      <td>0.032191</td>\n",
       "      <td>0.661475</td>\n",
       "      <td>0.305296</td>\n",
       "      <td>0.376947</td>\n",
       "      <td>0.292835</td>\n",
       "      <td>0.024922</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.032225</td>\n",
       "      <td>0.460772</td>\n",
       "      <td>0.176599</td>\n",
       "      <td>0.473454</td>\n",
       "      <td>0.460772</td>\n",
       "      <td>0.484873</td>\n",
       "      <td>0.455300</td>\n",
       "      <td>0.155969</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Rate_by_Daily  Rate_by_Hourly  Rate_by_Monthly  Rate_by_Yearly  \\\n",
       "count     963.000000      963.000000       963.000000      963.000000   \n",
       "mean        0.001038        0.305296         0.032191        0.661475   \n",
       "std         0.032225        0.460772         0.176599        0.473454   \n",
       "min         0.000000        0.000000         0.000000        0.000000   \n",
       "25%         0.000000        0.000000         0.000000        0.000000   \n",
       "50%         0.000000        0.000000         0.000000        1.000000   \n",
       "75%         0.000000        1.000000         0.000000        1.000000   \n",
       "max         1.000000        1.000000         1.000000        1.000000   \n",
       "\n",
       "       City_Bins_high  City_Bins_low  City_Bins_medium  City_Bins_missing  \n",
       "count      963.000000     963.000000        963.000000         963.000000  \n",
       "mean         0.305296       0.376947          0.292835           0.024922  \n",
       "std          0.460772       0.484873          0.455300           0.155969  \n",
       "min          0.000000       0.000000          0.000000           0.000000  \n",
       "25%          0.000000       0.000000          0.000000           0.000000  \n",
       "50%          0.000000       0.000000          0.000000           0.000000  \n",
       "75%          1.000000       1.000000          1.000000           0.000000  \n",
       "max          1.000000       1.000000          1.000000           1.000000  "
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "#cat_df['City_Bins']= salary_df['City_Bins']\n",
    "#drop Rate_by_Daily and City_Bins_missing since too many zeros\n",
    "cat_df_clean = cat_df.drop(['Rate_by_Daily','City_Bins_missing'], axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Combine contious and categorical variables into one dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Other_inputs = pd.concat([cont_scaled_df, cat_df], axis=1)\n",
    "Other_inputs = pd.concat([cont_scaled_df, cat_df_clean], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 963 entries, 0 to 962\n",
      "Data columns (total 8 columns):\n",
      "Cost of Living Plus Rent Index    963 non-null float64\n",
      "Local Purchasing Power Index      963 non-null float64\n",
      "Rate_by_Hourly                    963 non-null float64\n",
      "Rate_by_Monthly                   963 non-null float64\n",
      "Rate_by_Yearly                    963 non-null float64\n",
      "City_Bins_high                    963 non-null float64\n",
      "City_Bins_low                     963 non-null float64\n",
      "City_Bins_medium                  963 non-null float64\n",
      "dtypes: float64(8)\n",
      "memory usage: 60.3 KB\n"
     ]
    }
   ],
   "source": [
    "Other_inputs.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Text inputs - embedding layer\n",
    "tokenizer = text.Tokenizer() \n",
    "tokenizer.num_words = 1000  #limit the number of words to keep most common 1000 words\n",
    "tokenizer.fit_on_texts(list(salary_df['combined_text']))\n",
    "list_tokenized_jd = tokenizer.texts_to_sequences(salary_df['combined_text'])\n",
    "\n",
    "X_t = sequence.pad_sequences(list_tokenized_jd, maxlen=500, padding = 'post') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = to_categorical(target_class.values, num_classes=6, dtype=\"float32\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Train Test split on text inputs use Keras embedding layer\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_t, y, test_size=0.1, random_state=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Train Test split on other inputs\n",
    "\n",
    "X2_train, X2_test, y_train, y_test = train_test_split(Other_inputs, y, test_size=0.1, random_state=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(866, 8)"
      ]
     },
     "execution_count": 168,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X2_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(866, 500)"
      ]
     },
     "execution_count": 169,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(866, 6)"
      ]
     },
     "execution_count": 170,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0      5\n",
       "1      1\n",
       "2      0\n",
       "3      2\n",
       "4      3\n",
       "      ..\n",
       "958    3\n",
       "959    3\n",
       "960    3\n",
       "961    2\n",
       "962    2\n",
       "Name: Salary_Bins, Length: 963, dtype: category\n",
       "Categories (6, int64): [0, 1, 2, 3, 4, 5]"
      ]
     },
     "execution_count": 171,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "target_class"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Put them together"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "from keras.layers.convolutional import Conv2D\n",
    "from keras.layers.convolutional import MaxPooling2D\n",
    "from keras.layers.core import Activation\n",
    "from keras.layers.core import Dropout\n",
    "from keras.layers.core import Dense\n",
    "from keras.layers import Flatten\n",
    "from keras.layers import Input\n",
    "from keras.models import Model\n",
    "from keras.layers import concatenate\n",
    "from keras.optimizers import Adam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "UpLPFthfde5J"
   },
   "outputs": [],
   "source": [
    "# define two sets of inputs\n",
    "inputA = Input(shape=(8,)) #non-words input\n",
    "inputB = Input(shape=(500,))  #word vectors \n",
    "\n",
    "# the first branch operates on the first input\n",
    "#hiddenA1 = Dense(16, activation=\"relu\")(inputA)\n",
    "#hiddenA2 = Dense(16, activation=\"relu\")(hiddenA1)\n",
    "outputA = Dense(4, activation=\"relu\")(inputA)\n",
    "modelA = Model(inputs=inputA, outputs=outputA)\n",
    "\n",
    "# the second branch opreates on the second input\n",
    "embedB1 =  Embedding(input_dim = 1000, output_dim = 32, input_length = 500)(inputB) #each word onto a 32 length real valued vector\n",
    "convB1 = Conv1D(filters=32, kernel_size = 3, padding='same', activation='relu')(embedB1)\n",
    "poolB1 = MaxPooling1D(pool_size=2)(convB1)\n",
    "LSTMB1 = LSTM(128, return_sequences=True)(poolB1)\n",
    "hiddenB1 = Dense(100, activation='relu')(LSTMB1)\n",
    "dropB1 = Dropout(0.5)(hiddenB1)\n",
    "flat = Flatten()(dropB1)\n",
    "outputB = Dense(128, activation='relu')(flat)\n",
    "modelB = Model(inputs=inputB, outputs=outputB)\n",
    "\n",
    "# combine the output of the two branches\n",
    "combined = concatenate([modelA.output, modelB.output])\n",
    "# apply a FC layer and then a regression prediction on the\n",
    "# combined outputs\n",
    "z = Dense(16, activation=\"relu\")(combined)\n",
    "z = Dense(6, activation=\"softmax\")(z)\n",
    "# our model will accept the inputs of the two branches and\n",
    "# then output a single value\n",
    "model = Model(inputs=[modelA.input, modelB.input], outputs=z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_4\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_5 (InputLayer)            (None, 500)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_8 (Embedding)         (None, 500, 32)      32000       input_5[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_11 (Conv1D)              (None, 500, 32)      3104        embedding_8[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling1d_11 (MaxPooling1D) (None, 250, 32)      0           conv1d_11[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "lstm_6 (LSTM)                   (None, 250, 128)     82432       max_pooling1d_11[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dense_14 (Dense)                (None, 250, 100)     12900       lstm_6[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "dropout_3 (Dropout)             (None, 250, 100)     0           dense_14[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "input_4 (InputLayer)            (None, 8)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "flatten_7 (Flatten)             (None, 25000)        0           dropout_3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_13 (Dense)                (None, 2)            18          input_4[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_15 (Dense)                (None, 128)          3200128     flatten_7[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 130)          0           dense_13[0][0]                   \n",
      "                                                                 dense_15[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_16 (Dense)                (None, 16)           2096        concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_17 (Dense)                (None, 6)            102         dense_16[0][0]                   \n",
      "==================================================================================================\n",
      "Total params: 3,332,780\n",
      "Trainable params: 3,332,780\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ehW6a2HGde5O"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 866 samples, validate on 97 samples\n",
      "Epoch 1/40\n",
      "866/866 [==============================] - 6s 7ms/step - loss: 1.3559 - accuracy: 0.4908 - val_loss: 1.8045 - val_accuracy: 0.3918\n",
      "Epoch 2/40\n",
      "866/866 [==============================] - 5s 5ms/step - loss: 1.0848 - accuracy: 0.5589 - val_loss: 1.6868 - val_accuracy: 0.3711\n",
      "Epoch 3/40\n",
      "866/866 [==============================] - 5s 5ms/step - loss: 0.9230 - accuracy: 0.6443 - val_loss: 2.0205 - val_accuracy: 0.3814\n",
      "Epoch 4/40\n",
      "866/866 [==============================] - 5s 5ms/step - loss: 0.7673 - accuracy: 0.7206 - val_loss: 2.7269 - val_accuracy: 0.4124\n",
      "Epoch 5/40\n",
      "866/866 [==============================] - 5s 5ms/step - loss: 0.6674 - accuracy: 0.7506 - val_loss: 2.3498 - val_accuracy: 0.4124\n",
      "Epoch 6/40\n",
      "866/866 [==============================] - 5s 5ms/step - loss: 0.5904 - accuracy: 0.7760 - val_loss: 2.9971 - val_accuracy: 0.4330\n",
      "Epoch 7/40\n",
      "866/866 [==============================] - 5s 6ms/step - loss: 0.4852 - accuracy: 0.8210 - val_loss: 3.0341 - val_accuracy: 0.4021\n",
      "Epoch 8/40\n",
      "866/866 [==============================] - 5s 6ms/step - loss: 0.4319 - accuracy: 0.8487 - val_loss: 2.9734 - val_accuracy: 0.4227\n",
      "Epoch 9/40\n",
      "866/866 [==============================] - 5s 5ms/step - loss: 0.3808 - accuracy: 0.8845 - val_loss: 4.3127 - val_accuracy: 0.4536\n",
      "Epoch 10/40\n",
      "866/866 [==============================] - 5s 5ms/step - loss: 0.3045 - accuracy: 0.9099 - val_loss: 3.5683 - val_accuracy: 0.4639\n",
      "Epoch 11/40\n",
      "866/866 [==============================] - 5s 5ms/step - loss: 0.2599 - accuracy: 0.9122 - val_loss: 4.1240 - val_accuracy: 0.4227\n",
      "Epoch 12/40\n",
      "866/866 [==============================] - 5s 5ms/step - loss: 0.2129 - accuracy: 0.9388 - val_loss: 3.7800 - val_accuracy: 0.4330\n",
      "Epoch 00012: early stopping\n"
     ]
    }
   ],
   "source": [
    "\n",
    "opt = Adam(lr=1e-3, decay=1e-3/200)\n",
    "model.compile(loss = 'categorical_crossentropy', metrics=['accuracy'], optimizer=opt)\n",
    "\n",
    "es = EarlyStopping(monitor='val_loss', patience = 10, mode='min', verbose=1)\n",
    "\n",
    "history = model.fit([X2_train,X_train], y_train, validation_data = ([X2_test, X_test], y_test), epochs=40, batch_size=128, callbacks=[es])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Try Word2Vec Embedding in Mixed Input Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [],
   "source": [
    "#X_train, X_test using pad sequences from Word2Vec model\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(jd_pad, y, test_size=0.1, random_state=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define two sets of inputs\n",
    "inputA = Input(shape=(8,)) #non-words input\n",
    "inputB = Input(shape=(500,))  #word vectors \n",
    "\n",
    "# the first branch operates on the first input\n",
    "#hiddenA1 = Dense(16, activation=\"relu\")(inputA)\n",
    "#hiddenA2 = Dense(16, activation=\"relu\")(hiddenA1)\n",
    "outputA = Dense(4, activation=\"relu\")(inputA)\n",
    "modelA = Model(inputs=inputA, outputs=outputA)\n",
    "\n",
    "# the second branch opreates on the second input\n",
    "\n",
    "#Word2Vec Embedding layer\n",
    "embedding_size = 100  #word2vec has a 100 dimension space\n",
    "embedding_layer = Embedding(input_dim = num_words, #input_dim: size of the vocabulary\n",
    "                    output_dim = embedding_size,  #dimension of the dense embedding\n",
    "                    embeddings_initializer = Constant(embedding_matrix),\n",
    "                    input_length = 500, #length of input sequences\n",
    "                    trainable = False)\n",
    "\n",
    "\n",
    "embedded_sequences = embedding_layer(inputB)\n",
    "#embedB1 =  Embedding(input_dim = 1000, output_dim = 32, input_length = 500)(inputB) #each word onto a 32 length real valued vector\n",
    "convB1 = Conv1D(filters=32, kernel_size = 3, padding='same', activation='relu')(embedded_sequences)\n",
    "poolB1 = MaxPooling1D(pool_size=2)(convB1)\n",
    "LSTMB1 = LSTM(128, return_sequences=True)(poolB1)\n",
    "dropB1 = Dropout(0.5)(poolB1)\n",
    "hiddenB1 = Dense(100, activation='relu')(dropB1)\n",
    "dropB2 = Dropout(0.5)(hiddenB1)\n",
    "flat = Flatten()(dropB2)\n",
    "outputB = Dense(128, activation='relu')(flat)\n",
    "modelB = Model(inputs=inputB, outputs=outputB)\n",
    "\n",
    "# combine the output of the two branches\n",
    "combined = concatenate([modelA.output, modelB.output])\n",
    "# apply a FC layer and then a regression prediction on the\n",
    "# combined outputs\n",
    "z = Dense(64, activation=\"relu\")(combined)\n",
    "z = Dense(32, activation=\"relu\")(z)\n",
    "z = Dense(6, activation=\"softmax\")(z)\n",
    "# our model will accept the inputs of the two branches and\n",
    "# then output a single value\n",
    "model = Model(inputs=[modelA.input, modelB.input], outputs=z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_9\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_10 (InputLayer)           (None, 500)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_10 (Embedding)        (None, 500, 100)     2652000     input_10[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "conv1d_13 (Conv1D)              (None, 500, 32)      9632        embedding_10[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling1d_13 (MaxPooling1D) (None, 250, 32)      0           conv1d_13[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_6 (Dropout)             (None, 250, 32)      0           max_pooling1d_13[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dense_22 (Dense)                (None, 250, 100)     3300        dropout_6[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_7 (Dropout)             (None, 250, 100)     0           dense_22[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "input_9 (InputLayer)            (None, 8)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "flatten_9 (Flatten)             (None, 25000)        0           dropout_7[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "dense_21 (Dense)                (None, 4)            36          input_9[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_23 (Dense)                (None, 128)          3200128     flatten_9[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_2 (Concatenate)     (None, 132)          0           dense_21[0][0]                   \n",
      "                                                                 dense_23[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_24 (Dense)                (None, 64)           8512        concatenate_2[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dense_25 (Dense)                (None, 32)           2080        dense_24[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "dense_26 (Dense)                (None, 6)            198         dense_25[0][0]                   \n",
      "==================================================================================================\n",
      "Total params: 5,875,886\n",
      "Trainable params: 3,223,886\n",
      "Non-trainable params: 2,652,000\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 866 samples, validate on 97 samples\n",
      "Epoch 1/40\n",
      "866/866 [==============================] - 2s 3ms/step - loss: 1.8434 - accuracy: 0.1744 - val_loss: 1.7935 - val_accuracy: 0.1856\n",
      "Epoch 2/40\n",
      "866/866 [==============================] - 2s 2ms/step - loss: 1.7875 - accuracy: 0.2125 - val_loss: 1.7746 - val_accuracy: 0.2474\n",
      "Epoch 3/40\n",
      "866/866 [==============================] - 2s 2ms/step - loss: 1.7552 - accuracy: 0.2529 - val_loss: 1.7660 - val_accuracy: 0.2268\n",
      "Epoch 4/40\n",
      "866/866 [==============================] - 2s 2ms/step - loss: 1.7289 - accuracy: 0.2483 - val_loss: 1.7403 - val_accuracy: 0.2784\n",
      "Epoch 5/40\n",
      "866/866 [==============================] - 2s 2ms/step - loss: 1.7012 - accuracy: 0.2968 - val_loss: 1.7506 - val_accuracy: 0.2990\n",
      "Epoch 6/40\n",
      "866/866 [==============================] - 2s 2ms/step - loss: 1.6694 - accuracy: 0.2991 - val_loss: 1.7042 - val_accuracy: 0.3196\n",
      "Epoch 7/40\n",
      "866/866 [==============================] - 2s 2ms/step - loss: 1.6168 - accuracy: 0.3464 - val_loss: 1.7028 - val_accuracy: 0.2577\n",
      "Epoch 8/40\n",
      "866/866 [==============================] - 2s 2ms/step - loss: 1.6064 - accuracy: 0.3591 - val_loss: 1.6715 - val_accuracy: 0.3093\n",
      "Epoch 9/40\n",
      "866/866 [==============================] - 2s 2ms/step - loss: 1.5051 - accuracy: 0.4145 - val_loss: 1.6844 - val_accuracy: 0.3505\n",
      "Epoch 10/40\n",
      "866/866 [==============================] - 2s 2ms/step - loss: 1.4609 - accuracy: 0.4400 - val_loss: 1.6489 - val_accuracy: 0.3093\n",
      "Epoch 11/40\n",
      "866/866 [==============================] - 2s 2ms/step - loss: 1.5135 - accuracy: 0.4099 - val_loss: 1.6254 - val_accuracy: 0.3505\n",
      "Epoch 12/40\n",
      "866/866 [==============================] - 1s 2ms/step - loss: 1.4219 - accuracy: 0.4527 - val_loss: 1.6001 - val_accuracy: 0.3814\n",
      "Epoch 13/40\n",
      "866/866 [==============================] - 2s 2ms/step - loss: 1.3285 - accuracy: 0.5092 - val_loss: 1.6141 - val_accuracy: 0.3608\n",
      "Epoch 14/40\n",
      "866/866 [==============================] - 2s 2ms/step - loss: 1.3067 - accuracy: 0.5104 - val_loss: 1.6200 - val_accuracy: 0.3093\n",
      "Epoch 15/40\n",
      "866/866 [==============================] - 2s 2ms/step - loss: 1.2805 - accuracy: 0.5323 - val_loss: 1.6689 - val_accuracy: 0.2577\n",
      "Epoch 16/40\n",
      "866/866 [==============================] - 2s 2ms/step - loss: 1.2489 - accuracy: 0.5312 - val_loss: 1.5890 - val_accuracy: 0.3608\n",
      "Epoch 17/40\n",
      "866/866 [==============================] - 2s 2ms/step - loss: 1.2078 - accuracy: 0.5612 - val_loss: 1.5230 - val_accuracy: 0.4330\n",
      "Epoch 18/40\n",
      "866/866 [==============================] - 2s 2ms/step - loss: 1.1008 - accuracy: 0.5912 - val_loss: 1.5369 - val_accuracy: 0.3608\n",
      "Epoch 19/40\n",
      "866/866 [==============================] - 2s 2ms/step - loss: 1.1429 - accuracy: 0.5589 - val_loss: 1.6621 - val_accuracy: 0.3402\n",
      "Epoch 20/40\n",
      "866/866 [==============================] - 2s 2ms/step - loss: 1.1246 - accuracy: 0.5901 - val_loss: 1.5539 - val_accuracy: 0.3505\n",
      "Epoch 21/40\n",
      "866/866 [==============================] - 2s 2ms/step - loss: 1.0190 - accuracy: 0.6201 - val_loss: 1.5700 - val_accuracy: 0.3918\n",
      "Epoch 22/40\n",
      "866/866 [==============================] - 2s 2ms/step - loss: 0.9594 - accuracy: 0.6790 - val_loss: 1.5544 - val_accuracy: 0.4021\n",
      "Epoch 23/40\n",
      "866/866 [==============================] - 2s 2ms/step - loss: 1.0218 - accuracy: 0.6386 - val_loss: 1.5388 - val_accuracy: 0.4021\n",
      "Epoch 24/40\n",
      "866/866 [==============================] - 2s 2ms/step - loss: 0.9932 - accuracy: 0.6501 - val_loss: 1.5127 - val_accuracy: 0.4330\n",
      "Epoch 25/40\n",
      "866/866 [==============================] - 2s 2ms/step - loss: 0.9247 - accuracy: 0.6605 - val_loss: 1.4798 - val_accuracy: 0.4433\n",
      "Epoch 26/40\n",
      "866/866 [==============================] - 2s 2ms/step - loss: 0.8984 - accuracy: 0.6848 - val_loss: 1.4777 - val_accuracy: 0.4433\n",
      "Epoch 27/40\n",
      "866/866 [==============================] - 2s 2ms/step - loss: 0.8228 - accuracy: 0.7206 - val_loss: 1.5112 - val_accuracy: 0.3608\n",
      "Epoch 28/40\n",
      "866/866 [==============================] - 2s 2ms/step - loss: 0.7956 - accuracy: 0.7148 - val_loss: 1.7503 - val_accuracy: 0.3711\n",
      "Epoch 29/40\n",
      "866/866 [==============================] - 2s 2ms/step - loss: 0.8483 - accuracy: 0.6905 - val_loss: 1.6575 - val_accuracy: 0.3918\n",
      "Epoch 30/40\n",
      "866/866 [==============================] - 2s 2ms/step - loss: 0.7856 - accuracy: 0.7217 - val_loss: 1.6503 - val_accuracy: 0.3814\n",
      "Epoch 31/40\n",
      "866/866 [==============================] - 2s 2ms/step - loss: 0.8345 - accuracy: 0.7102 - val_loss: 1.4842 - val_accuracy: 0.4124\n",
      "Epoch 32/40\n",
      "866/866 [==============================] - 2s 2ms/step - loss: 0.7086 - accuracy: 0.7667 - val_loss: 1.6357 - val_accuracy: 0.4124\n",
      "Epoch 33/40\n",
      "866/866 [==============================] - 1s 2ms/step - loss: 0.6974 - accuracy: 0.7667 - val_loss: 1.5167 - val_accuracy: 0.4021\n",
      "Epoch 34/40\n",
      "866/866 [==============================] - 1s 2ms/step - loss: 0.6180 - accuracy: 0.7991 - val_loss: 1.7042 - val_accuracy: 0.3711\n",
      "Epoch 35/40\n",
      "866/866 [==============================] - 2s 2ms/step - loss: 0.7309 - accuracy: 0.7413 - val_loss: 1.5102 - val_accuracy: 0.4021\n",
      "Epoch 36/40\n",
      "866/866 [==============================] - 2s 2ms/step - loss: 0.6248 - accuracy: 0.8025 - val_loss: 1.4790 - val_accuracy: 0.4433\n",
      "Epoch 00036: early stopping\n"
     ]
    }
   ],
   "source": [
    "opt = RMSprop(lr=1e-4)\n",
    "model.compile(loss = 'categorical_crossentropy', metrics=['accuracy'], optimizer=opt)\n",
    "\n",
    "es = EarlyStopping(monitor='val_loss', patience = 10, mode='min', verbose=1)\n",
    "\n",
    "history = model.fit([X2_train,X_train], y_train, validation_data = ([X2_test, X_test], y_test), epochs=40, batch_size=128, callbacks=[es])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Word2Vec was not able to improve the performance =  ="
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model 7: Use Job Title along with City, Cost of Living Index in Classification Models \n",
    "\n",
    "Model 1-5 uses job description and job title as the combined text in the text classification model. However, due to the large number of words in one job description, using the whole text may not be the best way to extract information as we can see many unrelevant words show up even with TF-IDF. Now we are going to use Title as a separate set of features to predict salary. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Explore Title"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 963 entries, 0 to 962\n",
      "Data columns (total 26 columns):\n",
      "Unnamed: 0                        963 non-null int64\n",
      "Title                             963 non-null object\n",
      "Company                           963 non-null object\n",
      "Link                              963 non-null object\n",
      "Review                            963 non-null object\n",
      "Salary                            963 non-null object\n",
      "Description                       963 non-null object\n",
      "Location2                         958 non-null object\n",
      "Rate_by                           963 non-null object\n",
      "low                               963 non-null object\n",
      "high                              762 non-null object\n",
      "min                               963 non-null float64\n",
      "max                               963 non-null float64\n",
      "adjusted_salary                   963 non-null float64\n",
      "adjusted_yearly_salary            963 non-null float64\n",
      "Rank                              939 non-null float64\n",
      "City                              939 non-null object\n",
      "Cost of Living Index              939 non-null float64\n",
      "Rent Index                        939 non-null float64\n",
      "Cost of Living Plus Rent Index    939 non-null float64\n",
      "Groceries Index                   939 non-null float64\n",
      "Restaurant Price Index            939 non-null float64\n",
      "Local Purchasing Power Index      939 non-null float64\n",
      "City_Bins                         939 non-null object\n",
      "combined_text                     963 non-null object\n",
      "Salary_Bins                       963 non-null int64\n",
      "dtypes: float64(11), int64(2), object(13)\n",
      "memory usage: 203.1+ KB\n"
     ]
    }
   ],
   "source": [
    "salary_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Drop NA values\n",
    "salary_df_new = salary_df.dropna(axis = 0, subset = ['City_Bins'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 939 entries, 0 to 962\n",
      "Data columns (total 26 columns):\n",
      "Unnamed: 0                        939 non-null int64\n",
      "Title                             939 non-null object\n",
      "Company                           939 non-null object\n",
      "Link                              939 non-null object\n",
      "Review                            939 non-null object\n",
      "Salary                            939 non-null object\n",
      "Description                       939 non-null object\n",
      "Location2                         939 non-null object\n",
      "Rate_by                           939 non-null object\n",
      "low                               939 non-null object\n",
      "high                              743 non-null object\n",
      "min                               939 non-null float64\n",
      "max                               939 non-null float64\n",
      "adjusted_salary                   939 non-null float64\n",
      "adjusted_yearly_salary            939 non-null float64\n",
      "Rank                              939 non-null float64\n",
      "City                              939 non-null object\n",
      "Cost of Living Index              939 non-null float64\n",
      "Rent Index                        939 non-null float64\n",
      "Cost of Living Plus Rent Index    939 non-null float64\n",
      "Groceries Index                   939 non-null float64\n",
      "Restaurant Price Index            939 non-null float64\n",
      "Local Purchasing Power Index      939 non-null float64\n",
      "City_Bins                         939 non-null object\n",
      "combined_text                     939 non-null object\n",
      "Salary_Bins                       939 non-null int64\n",
      "dtypes: float64(11), int64(2), object(13)\n",
      "memory usage: 198.1+ KB\n"
     ]
    }
   ],
   "source": [
    "salary_df_new.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['2021048 Data Scientist $215,000.00', 'Data Science/Data Analyst',\n",
       "       'Data Science Intern', 'Jr. Data Scientist',\n",
       "       'Business Data Quality Engineer', 'Data Scientist',\n",
       "       'Machine Learning / C++ Engineer (DoD Clearance Required)',\n",
       "       'Undergraduate Internship/Co-op Program - Data Scientist',\n",
       "       'Data Science Community Lead', 'Data Analyst/Data Scientist',\n",
       "       'Self-employed Hardware/ Data Centre Field Engineer',\n",
       "       'DATA SCIENCE & TAGGING ANALYST', 'Data Engineer',\n",
       "       'HPC / Computational Data Scientist with Clearance',\n",
       "       'Operations Research Analyst (Data Scientist)',\n",
       "       'Social Science Analyst', 'Data Center Engineer',\n",
       "       'Data Scientist II', 'Graduate Studies Program - Data Scientist',\n",
       "       'Machine Learning Data Engineer',\n",
       "       '2021049 Data Scientist $100,000.00', 'Principal Data Scientist',\n",
       "       'Data Center Technician/Customer Engineer',\n",
       "       'Experienced Data Scientist (locals only)',\n",
       "       'Systems Engineer / Data Analyst (TS/SCI w/POLY)',\n",
       "       'Data Scientist (TS/SCI w/CI POLY) / D2P20',\n",
       "       'Analyst & Data Scientist - CDC Contact Tracing',\n",
       "       'Data Science Capabilities Expert',\n",
       "       'Big Data Analyst (TS/SCI w/POLY)',\n",
       "       'Big Data Systems Engineer (TS/SCI w/Poly)',\n",
       "       'Human Resources Analyst', 'Office Coordinator',\n",
       "       'Marketing Associate', 'Senior Business Analyst',\n",
       "       'Digital Marketing Specialist', 'National Development Director',\n",
       "       'Jr. Business Analyst',\n",
       "       'Upcoming Opportunities for a Director SAP SAC / S4 Reporting...',\n",
       "       'Financial Analyst', 'IT Financial Analyst',\n",
       "       'Data Analyst (W2 Contract | No C2C)', 'Data Specialist - ATL',\n",
       "       'Data and Reporting Analyst',\n",
       "       'DHS/OIT-Gateway Help Desk Analyst-(IT Business Analyst 2) -0...',\n",
       "       'Sponsor Funded Professiona | Full Stack Engineer II',\n",
       "       'Business Development Representative',\n",
       "       'Data Loss Prevention Analyst', 'Azure Data Engineer',\n",
       "       'Outside Sales Representative', 'Business Analyst',\n",
       "       'Healthcare Data Analyst', 'Senior Media Buyer- REMOTE',\n",
       "       \"Marketing Manager (5+ years' HVACR experience required)\",\n",
       "       'Evaluation Specialists / Data Analysts', 'Website Programmer',\n",
       "       'Security Solutions Engineer',\n",
       "       'Performance and Optimization Engineer',\n",
       "       'Entry Level Business Analyst',\n",
       "       'Business Analyst - Strategy Consulting', 'IT Business Analyst',\n",
       "       'Jr . Business Analyst', 'Business & IT Analyst',\n",
       "       'Operations Analyst', 'Payroll Business Analyst',\n",
       "       'Business Intelligence Manager',\n",
       "       'Sales and Use Tax Business Analyst', 'Business Analyst, Spectrum',\n",
       "       'Senior Programmer Analyst - Business Solutions',\n",
       "       'SQL SSAS/ SSIS Developer', 'Business Systems Programmer Analyst',\n",
       "       'Inside Sales support Analyst - Account Manager',\n",
       "       'Senior Consultant - Power BI Data Analyst',\n",
       "       'Email Development Operations Analyst',\n",
       "       'SUPPORT SERVICES DIVISION MANAGER', 'Senior Data Scientist',\n",
       "       'Clinical Research AI Analyst',\n",
       "       'Artificial Intelligence Researcher',\n",
       "       'Artificial Intelligence Engineer',\n",
       "       'Statistician / Economist - Public Health Research',\n",
       "       'DPS - FIN - CAPPS Financial Senior Data Analyst',\n",
       "       'Data Analyst/Business Analyst_0745', 'Business Analyst 2',\n",
       "       'DPS - ICT - Criminal Intelligence Analyst I',\n",
       "       'Chief Lending Officer (CLO)', 'Marketing Coordinator',\n",
       "       'Senior Pega Business Analyst', 'Business Analyst (Mid Level)',\n",
       "       'Technical Architect, HHSC', 'SEO Specialist', 'Lab Analyst',\n",
       "       'Special Education Data Support Analyst (Data Analyst IV)',\n",
       "       'Account Executive', 'Strategic Initiatives Analyst',\n",
       "       'Digital Marketer', 'Model Risk Analyst (Reporting & Governance)',\n",
       "       'Data Analyst', 'Pay Per Click Specialist',\n",
       "       'Sales Engineer - Workforce Management',\n",
       "       'Analyst for Educator Preparation Improvement (Data Analyst I...',\n",
       "       'Call Center Agent (3rd Shift)',\n",
       "       'SQL Data Analyst - Local Candidates only',\n",
       "       'Big Data Hadoop Instructor', 'Business Analyst III', 'Webmaster',\n",
       "       'Digital Marketing Manager',\n",
       "       'Data Analyst with Winbatch Exp (w2 only)',\n",
       "       'Technical architect - (Data analytics or Dataware housing pr...',\n",
       "       'Infection Prevention RN Supervisor',\n",
       "       'Board Certified Behavior Analyst - Must be licensed in Texas',\n",
       "       'Lead Electrical Estimator', 'Digital Media Coordinator',\n",
       "       'Asset Protection Coordinator', 'Lab Supervisor',\n",
       "       'Analyst II, Data Science',\n",
       "       'Senior Data Solutions Analyst, GRM Data Operations',\n",
       "       'Digital Analyst', 'Opioid Surveillance Epidemiologist',\n",
       "       'Senior Product Manager- Remote based out of Boston MA',\n",
       "       'Marketing / Front End Developer',\n",
       "       'Senior Computer Vision Engineer For Surgical Robotics Compan...',\n",
       "       'Public Affairs Specialist', 'Sales Administrator',\n",
       "       'Sr Business Analyst ( FIX, SWIFT. Trade)',\n",
       "       'Associate Product Manager', 'Data Quality Analyst',\n",
       "       'Senior Actuarial Manager', 'Associate Business Systems Analyst',\n",
       "       'Education Data Analyst (ESC)',\n",
       "       'Social Media, E-newsletters and Website Content Management',\n",
       "       'Business System Analyst', 'Applications Administrator',\n",
       "       'Business intelligence Developer',\n",
       "       'Urgent Need! - IT Architect - Remote start, then Cambridge,...',\n",
       "       'Social Media Strategist',\n",
       "       'Coordinator, Media & Technology - Office of Public Health Pr...',\n",
       "       'Senior Software Architect - Population Health Management',\n",
       "       'Statistical Process Control Development Engineer',\n",
       "       'Behavior Technician',\n",
       "       'Senior Director I, Product Research, New Mobility Products',\n",
       "       'Accounting Clerk', 'Commercial Marketing Analytics',\n",
       "       'Data Analytics/Digital Marketing Analyst',\n",
       "       'Analyst/Senior Analyst, Advanced Analytics',\n",
       "       'Senior Analyst - Digital Marketing',\n",
       "       'Data Engineer/Senior Data Engineer - Data & Analytics Engine...',\n",
       "       'Boston Healthcare Software Startup - Server Software Enginee...',\n",
       "       'DATA TOOLS ENGINEER',\n",
       "       'Assistant Director/Director, Office of Data Science',\n",
       "       'Director II, Data Science - GRM Actuarial',\n",
       "       'Staff Data Engineer/Data Architect', 'Lead Data Engineer',\n",
       "       'NLP Expert in Machine Learning', 'Staff Data Engineer',\n",
       "       'Senior Software Engineer | Machine Learning | Boston',\n",
       "       'Sr. Data Scientist/ NLP Engineer',\n",
       "       'Lead Machine Learning Engineer', 'Bioinformatics Data Scientist',\n",
       "       'Data Engineer, Retail Analytics',\n",
       "       'Senior Director I, Data Science - GRS Predictive Analytics',\n",
       "       'Data Brick Engineer', 'Senior Big Data Engineer - Health-tech',\n",
       "       'Lead Software Engineer - Health-tech data',\n",
       "       'Senior Machine Learning Engineer', 'Lead Data Scientist',\n",
       "       'Machine Learning Engineer',\n",
       "       'Entry level Business Analyst/Software Eng',\n",
       "       'Surveillance and Evaluation Data Manager',\n",
       "       'UI BENEFITS CALL CENTER REPRESENTATIVE - CDE', 'Program Director',\n",
       "       'Supply Chain Planning Analyst',\n",
       "       'Work From Home - Part Time Controller / Accountant',\n",
       "       'Laboratory Technologist',\n",
       "       'Mail Machine Inserter Operator (Production)',\n",
       "       'Applied Behavior Analysis Therapist / Registered Behavior Te...',\n",
       "       'Marketing Intern', 'High School Guidance Counselor',\n",
       "       'Behavior Technician For Children with Autism',\n",
       "       'Registered Behavior Technician (RBT) / Applied Behavior Anal...',\n",
       "       'Business/Data Analyst', 'ABA Therapist', 'Development Associate',\n",
       "       'Home Builder Accountant (work from home near Charlotte, NC)',\n",
       "       'Quality Control Technician', 'Sr. Accountant', 'Design Engineer',\n",
       "       'Search Engine Optimization Lead', 'Usability Researcher',\n",
       "       'Machine/Deep Learning Research Scientist',\n",
       "       'QA Tester (Avaya/ IVR/ Cyara)', 'F.O.G. COMPLIANCE OFFICER',\n",
       "       'Sr. Reimbursement Specialist 1354',\n",
       "       'Board Certified Behavior Analyst (BCBA-D/BCBA)',\n",
       "       'Surety Underwriting Specialist', 'Senior Buyer (Procurement)',\n",
       "       '2nd Shift Metrology/Quality Technician',\n",
       "       'Virtual Open House-Call Center Agents! - Tues 5/19 at 11am',\n",
       "       'SQL Data Analysis Expert', 'Data Analyst II',\n",
       "       'ERP Systems/Data Analyst',\n",
       "       'Virtual Open House-Call Center Agents! - Tues 5/19 at 11am!',\n",
       "       'ERP Analyst', 'Finance Manager',\n",
       "       'Business Analyst with Treasury domain background',\n",
       "       'Chemist/Lab Technician', 'Account Support Administrator',\n",
       "       'SQL Developer', 'Sr. Reimbursement Analyst',\n",
       "       'Senior Behavior Analyst (BCBA w. LP / LPA Certification)',\n",
       "       'Inventory Project Manager', 'Analytics Consultant',\n",
       "       'Contracts Attorney', 'Data Analytics Instructor',\n",
       "       'Marketing Analyst', 'Director of Consulting',\n",
       "       'Customer Service Call Center Manager- Join our winning team!',\n",
       "       'Communications Manager',\n",
       "       'Technical Product Manager (Healthcare Tech Startup)',\n",
       "       'Applications Analyst',\n",
       "       'Marketing Manager (healthcare tech startup)',\n",
       "       'Junior Data Engineer w/ ETL',\n",
       "       'Marketing Automation Specialist - Infusionsoft / Keap - Cont...',\n",
       "       'Senior Software Engineer',\n",
       "       'Cloud Security Engineer - Azure, DevOps, HIPAA Compliance',\n",
       "       'DevOps Security Engineer - Azure',\n",
       "       'Visual & UX/UI Designer (healthcare tech startup)',\n",
       "       'MDS Director RN', 'VP Sales & Marketing',\n",
       "       'Big Data/Cloud Architect', 'Marketing and Social Media Manager',\n",
       "       'Management Associate',\n",
       "       'Sr. Data Analyst - Analytics, Marketing, CPG', 'IAM Engineer',\n",
       "       'Content Writer - Work From Home', 'Remote Wholesale Underwriter',\n",
       "       'Content and Design Specialist', 'Paid Search Specialist',\n",
       "       'Remote Marketing Specialist', 'Commercial Analytics Manager',\n",
       "       'Full Stack Data Engineer- SQL/Power BI/ETL/C#',\n",
       "       'Data Security Engineer - Information Technology',\n",
       "       'Sr. Data Engineer-Data Architecture,Design for Big Data,Anal...',\n",
       "       'Big Data Engineer, COBOL, PySpark',\n",
       "       'Senior Data Analytics Process Engineer', 'Hadoop Data Engineer',\n",
       "       'Cloud Data Engineer', 'Senior Data Engineer',\n",
       "       'Credit Risk Analyst', 'Data Engineer(only W2)',\n",
       "       'Senior Cloud Data Engineer',\n",
       "       'Principal Project Engineer - Streamlined Project Delivery an...',\n",
       "       'Sr. SOC Analyst', 'Web Designer',\n",
       "       'Drug Testing Technician (TXDAL002)', 'Digital Media Manager',\n",
       "       'SAP Business Analyst', 'Senior QC Analyst: Data Integrity SME',\n",
       "       'Data Warehouse team Lead - W2 candidates',\n",
       "       'VP of Sales - Health Plan Experience - Remote Role',\n",
       "       'Sales Associate', 'Data Analyst/Statistician Position #1119',\n",
       "       'Senior Python Developer',\n",
       "       'DORA/Public Utilities Commission: Analyst VI',\n",
       "       'Director of Operations', 'Digital Communications Manager',\n",
       "       'ETL Analyst- $50/hr- 10 mo CTH- Broomfield CO',\n",
       "       'Staff Software Engineer',\n",
       "       'CBD/Hemp National Sales Account Executive', 'District Manager',\n",
       "       'PURCHASING AGENT I', 'Digital Communications Specialist',\n",
       "       'National Account Executive', 'DevOps Developer',\n",
       "       'Inside Sales & Customer Service (Hemp/CBD)',\n",
       "       'Marketing and Brand Manager', 'Remote Sales Representative',\n",
       "       'BCBA or RBT', 'Accountant IV: Projects', 'Lead Software Engineer',\n",
       "       'Data Center Engineer with a Top Secret clearance',\n",
       "       'Data Scientist/Machine Learning Engineer',\n",
       "       'Python Data Analyst (NumPy Modbus IEC61850)',\n",
       "       'REMOTE Sr . Database Engineer ( Iot Data Management DDS)',\n",
       "       'Big Data Engineer',\n",
       "       'Sr. VCP6-DCV Certified Engineer (VMware Data Center Aerospac...',\n",
       "       'Data Scientist - Finance & Business Operations Specialist',\n",
       "       'Associate Research Analyst', 'Chemist', 'Chemist II (PhD)',\n",
       "       'Digital Marketing Analyst (Content and Graphics)',\n",
       "       'Database Analyst', 'Quality Audit Administrator',\n",
       "       'Principal Administrative Analyst',\n",
       "       'Research Analyst (Public Utilities Regulatory Authority)',\n",
       "       'Hydrologist, GS-1315-07/09 (RG-TC)', 'Data Analyst Lead',\n",
       "       'Data Modeler/Data Analyst', 'Business Analyst - Healthcare',\n",
       "       'Credit Analyst and Data Coordinator', 'Behavior Technician (ABA)',\n",
       "       'Accounting Careers Trainee (Revenue Examiner 1)',\n",
       "       'Senior Systems Analyst', 'Sr Marketing Analyst',\n",
       "       'Insurance Solutions Analyst', 'Lead Data Analyst',\n",
       "       'Commissions Analyst',\n",
       "       'Procurement Policy Development Coordinator',\n",
       "       'Clinical Genomics Technologist / Geneticist / Genomics Lab S...',\n",
       "       'Student Trainee (Hydrology) GS-1399-09 CI-BL',\n",
       "       'Board Certified Behavior Analyst -BCBA',\n",
       "       'Junior Bid Data Engineer', 'Sr. Hadoop Data Engineer',\n",
       "       'Chemist 6 + Month Contract $27.40', 'Project Manager',\n",
       "       'Compensation/HRIS Specialist', 'Public Health Section Chief',\n",
       "       'Fusion Center Intelligence Director',\n",
       "       'ABA/Behavioral Therapist - FT',\n",
       "       'Clinical Associate-Applied Behavioral Analysis',\n",
       "       'Environmental Protection Assistant Division Director',\n",
       "       'Manager Of Research And Planning', 'Behavior Tech',\n",
       "       'Director Advanced Analytics', 'FARO Arm Inspector',\n",
       "       '6th Grade Science Teacher', 'Licensed Vocational Nurse (LVN)',\n",
       "       'Graphic Artist & Content Manager', 'Telephone Sales',\n",
       "       'Facebook Community Manager',\n",
       "       \"Sr. Pricing Analyst [SQL or Tableau Req'd] for a Nationwide...\",\n",
       "       'Software Architect', 'Principal Software Engineer',\n",
       "       'Temporary Assembly Team Member (AM Shift)',\n",
       "       'Temporary Assembly Team Member (PM Shift)',\n",
       "       'Director of Marketing ($110,000-$125,000)',\n",
       "       'Marketing Manager ($77,000 - $97,000)',\n",
       "       'Sr Project Accountant ($110M+ distinguished architectural fi...',\n",
       "       'Category Buyer (Chinese Speaking)',\n",
       "       'Director of Digital Marketing', 'Web Developer',\n",
       "       'Director of Corporate Marketing', 'Social Media Manager',\n",
       "       'Product Manager', 'Category Management Analyst - Sales',\n",
       "       'Bookkeeper', 'Facebook Ads Specialist',\n",
       "       'Tag Management Implementation Engineer',\n",
       "       'National Sales Director - Equipment Finance (Relo Assist Ava...',\n",
       "       'Amazon Marketing Specialist', 'Senior Financial Analyst',\n",
       "       'Junior Product Manager',\n",
       "       'Social Marketing Assistant (Social - Podcast - YouTube - Ema...',\n",
       "       'Online Marketing Assistant', 'Marketing Assistant',\n",
       "       'Sponsorship pitch deck creator',\n",
       "       'Director Paid Digital Advertising [Virtual/Job Share]',\n",
       "       'Technology Assistant/Infusionsoft Expert (2+ Yrs Exp. Requir...',\n",
       "       'Compliance Administrator',\n",
       "       'School Climate Evaluation and Data Analyst',\n",
       "       'Digital Marketing Representative', 'Senior Systems Administrator',\n",
       "       'Systems Administrator', 'Business Analyst (SCADA Systems) - 1485',\n",
       "       'Controller', 'Senior Engineer/Architect',\n",
       "       'Enterprise Risk Manager',\n",
       "       'Supervisory Survey Statistician (Regional Survey Manager), G...',\n",
       "       'Sr. E-Commerce Manager _ÑÐ Bilingual Chinese',\n",
       "       'Software Developer ( Back End )', 'Cost Accountant',\n",
       "       'Marketing Manager', 'Web Desginer - UI Desginer',\n",
       "       'Director of Digital Media', 'Senior SEO Manager',\n",
       "       'Google Adwords (PPC) & Social Advertising Specialist (Onsite...',\n",
       "       'Operations Analyst - Work from Home',\n",
       "       'Regional Operations Analyst',\n",
       "       'SaaS/ERP Enterprise Blockchain IT Sales Representative',\n",
       "       'Senior Director of Post-Secondary Pathways',\n",
       "       'Machined Parts Sales for Aerospace, Automotive, Medical OEM...',\n",
       "       'Business Intelligence Consultant',\n",
       "       'AVP - Workday Lead / Project Manager',\n",
       "       'FULL-STACK Blockchain Engineer (JS, C++, Node.js, React, Blo...',\n",
       "       'Director of Growth & Customer Acquisition',\n",
       "       'Senior BI Reporting Analyst', 'Vice President of Sales',\n",
       "       'Social Media Specialist',\n",
       "       'Part Manufacturing Estimator and Fulfillment Coordinator',\n",
       "       'Radar Telemetry Data Analyst', 'Accounting Manager',\n",
       "       'Nurse Supervisor Case Management RN',\n",
       "       'Nurse Supervisor Utilization Review RN',\n",
       "       'Salesforce Business Analyst/Administrator', 'Program Analyst',\n",
       "       'Salesforce Center of Excellence Business Analyst',\n",
       "       'Sr. Salesforce Business Analyst (CPQ,ERP)',\n",
       "       'Business Intelligence Consultant for NYC Investment Bank',\n",
       "       'Senior Data Analyst M/WBE, Enterprise Data Services',\n",
       "       'Business Analyst- Archer, Compliance/Risk',\n",
       "       'AML - Data Remediation Analyst',\n",
       "       'Entry Level Data Analyst Position', 'Senior Data Analyst',\n",
       "       'Junior Business Analyst', 'Data Collector and Service Analyst',\n",
       "       'Temporary Data Analyst', 'Junior Level Business Analyst',\n",
       "       'Business Intelligence/Analytics Manager',\n",
       "       'Business Analyst- Risk Management experience',\n",
       "       'Temporary Data Analyst - Portuguese', 'Management Analyst',\n",
       "       'Business Intelligence Analyst',\n",
       "       'Business Analyst with Insurance and P&C exp',\n",
       "       'Full-time job opportunity for (BA/ DA/ SAS) with Immigration Support',\n",
       "       'Senior Data Analyst _ÑÐ Team Lead', 'Compliance Data Analyst',\n",
       "       'Lead Data Insights Analyst',\n",
       "       'Business Analyst - Digital Channels',\n",
       "       'Technical Business Analyst', 'Program Associate',\n",
       "       'HR Operations Analyst', 'Biostatistician',\n",
       "       'Director of Research and Evaluation, I/DD',\n",
       "       'Ad Operations Specialist',\n",
       "       'Full-time job opportunity for Business Analyst with Immigrat...',\n",
       "       'Playlist Curator', 'SAP FICO Functional Consultant',\n",
       "       'Marketing Manager - Education', 'Auditor',\n",
       "       'VP Sports Sponsorship Research & Analytics',\n",
       "       'MBA Summer Associate - Business Analyst - Remote',\n",
       "       'Business Development Analyst',\n",
       "       'Digital/Paid Social Media Planner',\n",
       "       'Full-time job opportunity for Data Analyst with Immigration...',\n",
       "       'Business Analyst / Regulatory Reporting',\n",
       "       '2020 ONLINE SUMMER CONSUMER MARKETING INTERNSHIP (UNPAID)',\n",
       "       'Head of Growth Marketing - D2C', 'External Affairs Coordinator',\n",
       "       'Data Engineer FT - (Informatica ETL) NYC',\n",
       "       'Jr Software Data Engineer',\n",
       "       'Data Engineer (w/ Healthcare Domain experience)',\n",
       "       'Data Science Online Course Instructor (Part-Time, Contract)',\n",
       "       'System/data Engineer with Etrading and Capital Markets exper...',\n",
       "       'Software Data Engineer', 'Sr. Data Scientist',\n",
       "       'Machine Learning Engineer Position',\n",
       "       'Data Scientist Intern (Unpaid)', 'Cloud/Data Engineer',\n",
       "       'Mathematical Scientist', 'Decision Scientist',\n",
       "       'Machine Learning and Artificial Intelligence Researcher',\n",
       "       'Data Scientist, Analytics - Payment Risk',\n",
       "       'Machine Learning Developer',\n",
       "       'Machine Learning Intern, Instruction Assistant',\n",
       "       'Marketing Analyst Intern', 'AI / Machine Learning Engineer',\n",
       "       'Machine learning Apache engineer',\n",
       "       'Sr. Research Engineer \\x8a\\x97\\x96 Machine Learning/Image Processing',\n",
       "       'Enginner-Automation/ Instrumentation-11 (Associate)',\n",
       "       'Insurance Agent - work from home, great sales opportunity',\n",
       "       'Data Engineer - Python, ELK is a plus',\n",
       "       'Sales Representative - Anti-Corrosive Protective Coatings',\n",
       "       'Care Manager', 'RN Care Manager', 'Network Engineer',\n",
       "       'Accounts Payable Supervisor',\n",
       "       'Python Developer _ÑÐ ELK/Machine Learning',\n",
       "       'Email Marketing Manager', 'Recruiter',\n",
       "       'Mortgage Quality Control Specialist',\n",
       "       'Board Certified Behavior Analyst (BCBA)',\n",
       "       'Accounts Payable Manager', 'SAP BW/BO/BW on HANA Developer',\n",
       "       'Healthcare Data Analyst(Remote)-only w2',\n",
       "       'Business Analyst _ÑÐ Compliance, Operations',\n",
       "       'Content Marketing Manager - Healthcare App',\n",
       "       'Data Warehouse Manager', 'Cyber Security Engineer',\n",
       "       'RWD Business Analyst with Pharma Exp. need in West Point, PA',\n",
       "       'Drafter II',\n",
       "       'Business Analyst _ÑÐ Compliance, Process Management and Operat...',\n",
       "       'Cisco Data Engineer', 'Sr. Data Engineer',\n",
       "       'Supervisory Economist or Supervisory Statistician (Economics...',\n",
       "       'Sr. Business Systems Analyst - Finance Analytics',\n",
       "       'Data Modeler/Architect', 'Operations and Policy Analyst',\n",
       "       'Ethernet Marketing Business Operations lead',\n",
       "       'Medical Records Technician- Multnomah County,OR',\n",
       "       'PPC/SEM/Google Adwords/', 'Business Analyst 3',\n",
       "       'Senior Internal Auditor', 'Social Media / Marketing Manager',\n",
       "       'Business Systems Analyst',\n",
       "       'Marketing & Creative Design Strategist',\n",
       "       'Security Solutions Architect', 'Warehouse Supervisor',\n",
       "       'Social Media Marketing Manager',\n",
       "       'Vice President of Operations _ÑÐ High-Growth, Global, SaaS/Tel...',\n",
       "       'Now Seeking Account Resolution Specialists',\n",
       "       'Supervisory IT Specialist', 'Senior Technical Recruiter',\n",
       "       'Remote Full Stack Engineer', 'Research Health Science Specialist',\n",
       "       'Social Media Coordinator', 'Process Quality Engineer',\n",
       "       '(Part-Time) Behavior Technician - ABA Therapy',\n",
       "       'Chief Development Officer',\n",
       "       '3PL Business Development & Sales Manager',\n",
       "       'Pharmacist - Informatics Program Manager (ADPAC)',\n",
       "       'Research Study Coordinator', 'Research Coordinator',\n",
       "       '342-1 Business Analyst IV (Insurance)',\n",
       "       'Lead Medical Support Assistant',\n",
       "       'Informatica Data Warehouse Engineer',\n",
       "       'Big Data Engineer - Deloitte (Remote / Contract)',\n",
       "       'Data Analyst, Data Warehouse', 'Data Analyst Intern',\n",
       "       'Data Science Technical Writer - PhD or Graduates Preferred',\n",
       "       'Back-End Engineer (Data Engineer)',\n",
       "       'Python with ETL/Data Science Back Ground Needed',\n",
       "       'Infrastructure Data Engineer',\n",
       "       'Clinical Contract Associate II (Data Science)',\n",
       "       'AWS/ Machine Learning Engineer',\n",
       "       'Lead Data Scientist (Machine Learning)',\n",
       "       'Data Science Intern (Unpaid)',\n",
       "       'Senior Data Scientist(Healthcare)',\n",
       "       'Staff Machine Learning Engineer',\n",
       "       'Data Management & Integration Engineer',\n",
       "       'SENIOR DATA INFRASTRUCTURE ENGINEER',\n",
       "       'Tech Lead - Machine Learning',\n",
       "       'Data Warehouse Test Engineer - REMOTE',\n",
       "       'Sr. Data/Infrastructure Engineer', 'Sr. Genomics Data Engineer',\n",
       "       'Technical Writer (Machine Learning)',\n",
       "       'Senior Data Engineer - Scala',\n",
       "       'Senior NLP Machine Learning Engineer',\n",
       "       'Data Warehouse Support Engineer - REMOTE',\n",
       "       'Lecturer - Data Science - School of Information',\n",
       "       'Data Warehouse Analyst/Business Analyst',\n",
       "       'Data Engineer lead / Jr.Architect', 'Data Science Engineer',\n",
       "       'Senior Data Analyst- Product',\n",
       "       'Business Systems Analyst - Workday Payroll',\n",
       "       'Business Systems Analyst with Pharma Domain',\n",
       "       'Business Analyst - only local', 'Senior Business Systems Analyst',\n",
       "       'Sr. Business Systems Analyst', 'Business Systems Analyst 3',\n",
       "       'Business Analyst with molecular biology',\n",
       "       'Business Analyst ( Digital Sales & Marketing )',\n",
       "       'FINANCIAL ANALYST (12 MONTH REGISTER) (SENIOR BUSINESS APPRA...',\n",
       "       '1053 Business Analyst Epic Willow', 'IS Business Analyst',\n",
       "       'Contract Business Systems Analyst 3',\n",
       "       'Business Systems Analyst, Presidential Correspondence',\n",
       "       'Pharmaceutical Business Systems Analyst',\n",
       "       'Business Continuity Analyst', 'IS Business Analyst-Senior',\n",
       "       'IS Business Analyst II-Limited Term (Open)',\n",
       "       'JD Edwards Supply Chain MFG Business Analyst',\n",
       "       'Business Analyst Manager', 'Global Legal Operations Analyst',\n",
       "       '1052 Business Analyst:Epic Inpatient Clinical Documentation',\n",
       "       'Business Systems Analyst (SAP Human Resources/Payroll)',\n",
       "       '9976 Technology Expert 1 - Epic willow Business Analyst',\n",
       "       '122 HEALTHCARE ANALYTICS - Senior Business Intelligence Anal...',\n",
       "       'Oracle Fusion Order Management and Supply Chain BSA',\n",
       "       'Business Analyst - Pharma experience - Master data managemen...',\n",
       "       '11749-IT- Business Analyst - Senior',\n",
       "       '11637- IT- Business Analyst - Senior',\n",
       "       'Analyst, Clinical Biospecimen Management Technology Operatio...',\n",
       "       'Associate IT Consultant / Web Developer',\n",
       "       'HR Operations & Technology Analyst',\n",
       "       'College Business Analyst - Associate, Academic and Instituti...',\n",
       "       'IS Business Analyst-Senior, Budget System Functional Analyst',\n",
       "       'Global Security Operations Center (GSOC) Analyst - Full Time',\n",
       "       'Senior Business Analyst (Criminal Justice)',\n",
       "       'Senior Functional Analyst, PeopleSoft Financials (FSCM)',\n",
       "       'Sr. Analyst, Commercial Business Intelligence',\n",
       "       'Financial Aid Business Analyst - Administrative Analyst/Spec...',\n",
       "       'E - Commerce Manager', 'Forensic Accountant',\n",
       "       'Digital Business Analyst - only local',\n",
       "       'Senior Natural Language AI & Machine Learing Engineer',\n",
       "       'IBM Sterling Connect Direct Consultant', 'Azure DevOps Engineer',\n",
       "       'IT Sales Manager',\n",
       "       'Data Engineering With Java Development experience',\n",
       "       'Cyber Risk Senior Associate', 'Analytics Consultant V',\n",
       "       'Product Manager (Growth)', 'HR Data Analyst',\n",
       "       'Technical Product Manager', 'Tableau Developer',\n",
       "       'Warehouse Associate', 'VP Finance', 'Creative Director',\n",
       "       'QA Automation Engineer', 'Software Engineer',\n",
       "       'Senior DevOps Engineer', 'DevOps Engineer (Predictions Engine)',\n",
       "       'Customer Support and Business Operations (Enterprise SaaS St...',\n",
       "       'Hadoop Security', 'Total Customer Experience Analyst',\n",
       "       'Software Engineer (Microservices)', 'Retail Buyer/Category Buyer',\n",
       "       'Marketing and Social Media Coordinator',\n",
       "       'Business Intelligence Associate: Operations Insights',\n",
       "       'Back End Engineer', 'Health care Actuary',\n",
       "       'Hospital Senior Actuarial Associate',\n",
       "       'DevOps Engineer (Python, Hive, Kubernetes, Docker)',\n",
       "       'UX Researcher / Designer Intern (Summer)', 'Fraud Risk Analyst',\n",
       "       'Hadoop Admin (Security Support)',\n",
       "       'Administrative Officer, Clinical Practice & Professional Dev...',\n",
       "       'Administrative Support Assistant, Clinical Practice & Profes...',\n",
       "       'Laboratory Unit Manager - Genetic Counselor',\n",
       "       'Business Intelligence Director', 'Senior Financial Analyst (FT)',\n",
       "       'Python Engineer', 'Microsoft Business Intelligence Consultant',\n",
       "       'Loss Prevention Officer', 'Analyst',\n",
       "       'Microsoft Power BI - Technical Product Manager',\n",
       "       'Digital Marketing Strategist',\n",
       "       'Medical Records Technician - King County, WA',\n",
       "       'Back-end Engineer', 'SSAS Developers (W2 role) (USC, GC)',\n",
       "       'Cloud DevOps Engineer (Azure)', 'Power BI Designer',\n",
       "       'Systems Administrator & Webmaster, Part-time, Remote',\n",
       "       'Government Relations Administrator',\n",
       "       'Digital Marketing Manager Position', 'Impact Assistant',\n",
       "       'Community Manager and Customer Support',\n",
       "       'Sales Hunter (Business Development Specialist)',\n",
       "       'Android Developer', 'Cloud Software Engineer', 'DevOps Engineer',\n",
       "       'Experienced Mental Health Clinical Referral Coordinator',\n",
       "       'Senior Cloud Engineer', 'Senior Security Engineer',\n",
       "       'Senior Staff Cloud Engineer', 'Data Engineer/Analytics',\n",
       "       'Software Engineer with Kusto Redmond, WA Locals only',\n",
       "       'Cloud Engineer/Developer',\n",
       "       'Epidemiologist II - Analytics &Informatics COVID',\n",
       "       'Account Manager - Customer Success Team',\n",
       "       'Senior Technical Writer', 'HRIS Business Analyst',\n",
       "       'Business Analyst - TA0R076', 'Sr. Business Intelligence Analyst',\n",
       "       'Digital Business Analyst',\n",
       "       'Certified Business Analyst \\x8a\\x97\\x96 IT Security and Industrial Cont...',\n",
       "       'Data Scientist / Data Analytics', 'Data Visualization Engineer',\n",
       "       'Jr Data Analyst',\n",
       "       'Immediate Interview for Data Scientist in Seattle, WA Area',\n",
       "       'Azure Security Data Engineer',\n",
       "       'Sr Global Strategy Engineer - Data Center Infrastructure',\n",
       "       'Data Engineer with Full Stack (Cosmos and C#)',\n",
       "       'Business Development Manager', 'Employee Relations Specialist',\n",
       "       'SOC Analyst', 'Data Engineering II', 'IR Day Shift',\n",
       "       'Security Engineer', 'FDP Analyst',\n",
       "       'Junior MS Access/ VBA Developer'], dtype=object)"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "salary_df_new['Title'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "salary_df_new = salary_df_new.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>Title</th>\n",
       "      <th>Company</th>\n",
       "      <th>Link</th>\n",
       "      <th>Review</th>\n",
       "      <th>Salary</th>\n",
       "      <th>Description</th>\n",
       "      <th>Location2</th>\n",
       "      <th>Rate_by</th>\n",
       "      <th>low</th>\n",
       "      <th>...</th>\n",
       "      <th>City</th>\n",
       "      <th>Cost of Living Index</th>\n",
       "      <th>Rent Index</th>\n",
       "      <th>Cost of Living Plus Rent Index</th>\n",
       "      <th>Groceries Index</th>\n",
       "      <th>Restaurant Price Index</th>\n",
       "      <th>Local Purchasing Power Index</th>\n",
       "      <th>City_Bins</th>\n",
       "      <th>combined_text</th>\n",
       "      <th>Salary_Bins</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>2021048 Data Scientist $215,000.00</td>\n",
       "      <td>B4CORP</td>\n",
       "      <td>https://www.indeed.com/pagead/clk?mo=r&amp;ad=-6NY...</td>\n",
       "      <td>None</td>\n",
       "      <td>Up to $215,000 a year</td>\n",
       "      <td>Position Requires a Top Secret (TS/SCI) Cleara...</td>\n",
       "      <td>Arlington, VA</td>\n",
       "      <td>Yearly</td>\n",
       "      <td>Up to $215,000 a year</td>\n",
       "      <td>...</td>\n",
       "      <td>Arlington, VA</td>\n",
       "      <td>80.159143</td>\n",
       "      <td>58.838571</td>\n",
       "      <td>69.961143</td>\n",
       "      <td>76.886286</td>\n",
       "      <td>78.608571</td>\n",
       "      <td>115.594571</td>\n",
       "      <td>medium</td>\n",
       "      <td>2021048 Data Scientist $215,000.00 Position Re...</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>Data Science/Data Analyst</td>\n",
       "      <td>CRAiC LLC</td>\n",
       "      <td>https://www.indeed.com/company/CRAiC-LLC/jobs/...</td>\n",
       "      <td>None</td>\n",
       "      <td>$60,000 a year</td>\n",
       "      <td>Cost Readiness Analytics (CRAiC LLC) is lookin...</td>\n",
       "      <td>Arlington, VA</td>\n",
       "      <td>Yearly</td>\n",
       "      <td>$60,000 a year</td>\n",
       "      <td>...</td>\n",
       "      <td>Arlington, VA</td>\n",
       "      <td>80.159143</td>\n",
       "      <td>58.838571</td>\n",
       "      <td>69.961143</td>\n",
       "      <td>76.886286</td>\n",
       "      <td>78.608571</td>\n",
       "      <td>115.594571</td>\n",
       "      <td>medium</td>\n",
       "      <td>Data Science/Data Analyst Cost Readiness Analy...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>Data Science Intern</td>\n",
       "      <td>MilliporeSigma</td>\n",
       "      <td>https://www.indeed.com/company/Serono/jobs/Dat...</td>\n",
       "      <td>3.9</td>\n",
       "      <td>$20 - $25 an hour</td>\n",
       "      <td>Job Title: Data Science Intern\\rA career with ...</td>\n",
       "      <td>Arlington, VA</td>\n",
       "      <td>Hourly</td>\n",
       "      <td>$20</td>\n",
       "      <td>...</td>\n",
       "      <td>Arlington, VA</td>\n",
       "      <td>80.159143</td>\n",
       "      <td>58.838571</td>\n",
       "      <td>69.961143</td>\n",
       "      <td>76.886286</td>\n",
       "      <td>78.608571</td>\n",
       "      <td>115.594571</td>\n",
       "      <td>medium</td>\n",
       "      <td>Data Science Intern Job Title: Data Science In...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>Jr. Data Scientist</td>\n",
       "      <td>Numero Data LLC</td>\n",
       "      <td>https://www.indeed.com/company/Numero-Data-LLC...</td>\n",
       "      <td>None</td>\n",
       "      <td>$70,000 - $85,000 a year</td>\n",
       "      <td>Ideal candidate should have a degree in a quan...</td>\n",
       "      <td>Arlington, VA</td>\n",
       "      <td>Yearly</td>\n",
       "      <td>$70,000</td>\n",
       "      <td>...</td>\n",
       "      <td>Arlington, VA</td>\n",
       "      <td>80.159143</td>\n",
       "      <td>58.838571</td>\n",
       "      <td>69.961143</td>\n",
       "      <td>76.886286</td>\n",
       "      <td>78.608571</td>\n",
       "      <td>115.594571</td>\n",
       "      <td>medium</td>\n",
       "      <td>Jr. Data Scientist Ideal candidate should have...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>Business Data Quality Engineer</td>\n",
       "      <td>HRU Technical Resources</td>\n",
       "      <td>https://www.indeed.com/pagead/clk?mo=r&amp;ad=-6NY...</td>\n",
       "      <td>None</td>\n",
       "      <td>$45 - $55 an hour</td>\n",
       "      <td>Please note that this is not a corp to corp or...</td>\n",
       "      <td>Arlington, VA</td>\n",
       "      <td>Hourly</td>\n",
       "      <td>$45</td>\n",
       "      <td>...</td>\n",
       "      <td>Arlington, VA</td>\n",
       "      <td>80.159143</td>\n",
       "      <td>58.838571</td>\n",
       "      <td>69.961143</td>\n",
       "      <td>76.886286</td>\n",
       "      <td>78.608571</td>\n",
       "      <td>115.594571</td>\n",
       "      <td>medium</td>\n",
       "      <td>Business Data Quality Engineer Please note tha...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>934</th>\n",
       "      <td>958</td>\n",
       "      <td>Data Engineering II</td>\n",
       "      <td>Pueo Business Solutions</td>\n",
       "      <td>https://www.indeed.com/rc/clk?jk=90a6c966f45f8...</td>\n",
       "      <td>5</td>\n",
       "      <td>$108,000 - $118,000 a year</td>\n",
       "      <td>Data Engineering II\\rSalary: $108K-$118K\\rClea...</td>\n",
       "      <td>Washington, DC</td>\n",
       "      <td>Yearly</td>\n",
       "      <td>$108,000</td>\n",
       "      <td>...</td>\n",
       "      <td>Washington, DC</td>\n",
       "      <td>87.740000</td>\n",
       "      <td>78.180000</td>\n",
       "      <td>83.170000</td>\n",
       "      <td>86.440000</td>\n",
       "      <td>87.480000</td>\n",
       "      <td>130.250000</td>\n",
       "      <td>high</td>\n",
       "      <td>Data Engineering II Data Engineering II\\rSalar...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>935</th>\n",
       "      <td>959</td>\n",
       "      <td>IR Day Shift</td>\n",
       "      <td>Pueo Business Solutions</td>\n",
       "      <td>https://www.indeed.com/rc/clk?jk=e4416ec0bed30...</td>\n",
       "      <td>5</td>\n",
       "      <td>$89,000 - $99,000 a year</td>\n",
       "      <td>Tier 2 IR Day Shift\\rSalary Range: $89K-$99K\\r...</td>\n",
       "      <td>Washington, DC</td>\n",
       "      <td>Yearly</td>\n",
       "      <td>$89,000</td>\n",
       "      <td>...</td>\n",
       "      <td>Washington, DC</td>\n",
       "      <td>87.740000</td>\n",
       "      <td>78.180000</td>\n",
       "      <td>83.170000</td>\n",
       "      <td>86.440000</td>\n",
       "      <td>87.480000</td>\n",
       "      <td>130.250000</td>\n",
       "      <td>high</td>\n",
       "      <td>IR Day Shift Tier 2 IR Day Shift\\rSalary Range...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>936</th>\n",
       "      <td>960</td>\n",
       "      <td>Security Engineer</td>\n",
       "      <td>Pueo Business Solutions</td>\n",
       "      <td>https://www.indeed.com/rc/clk?jk=c06c7f39a6074...</td>\n",
       "      <td>5</td>\n",
       "      <td>$89,000 - $99,000 a year</td>\n",
       "      <td>Security Engineer\\rSalary Range: $89K-$99K\\rCl...</td>\n",
       "      <td>Washington, DC</td>\n",
       "      <td>Yearly</td>\n",
       "      <td>$89,000</td>\n",
       "      <td>...</td>\n",
       "      <td>Washington, DC</td>\n",
       "      <td>87.740000</td>\n",
       "      <td>78.180000</td>\n",
       "      <td>83.170000</td>\n",
       "      <td>86.440000</td>\n",
       "      <td>87.480000</td>\n",
       "      <td>130.250000</td>\n",
       "      <td>high</td>\n",
       "      <td>Security Engineer Security Engineer\\rSalary Ra...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>937</th>\n",
       "      <td>961</td>\n",
       "      <td>FDP Analyst</td>\n",
       "      <td>GC Associates USA</td>\n",
       "      <td>https://www.indeed.com/rc/clk?jk=84a4b28b5c409...</td>\n",
       "      <td>None</td>\n",
       "      <td>$80,000 - $90,000 a year</td>\n",
       "      <td>Position Title: FDP Analyst\\rCustomer: HQDA, G...</td>\n",
       "      <td>Washington, DC</td>\n",
       "      <td>Yearly</td>\n",
       "      <td>$80,000</td>\n",
       "      <td>...</td>\n",
       "      <td>Washington, DC</td>\n",
       "      <td>87.740000</td>\n",
       "      <td>78.180000</td>\n",
       "      <td>83.170000</td>\n",
       "      <td>86.440000</td>\n",
       "      <td>87.480000</td>\n",
       "      <td>130.250000</td>\n",
       "      <td>high</td>\n",
       "      <td>FDP Analyst Position Title: FDP Analyst\\rCusto...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>938</th>\n",
       "      <td>962</td>\n",
       "      <td>Junior MS Access/ VBA Developer</td>\n",
       "      <td>Empirical Concepts, Inc.</td>\n",
       "      <td>https://www.indeed.com/rc/clk?jk=916afc6c9b4a4...</td>\n",
       "      <td>None</td>\n",
       "      <td>$70,000 - $90,000 a year</td>\n",
       "      <td>If you want an accounting career with more opp...</td>\n",
       "      <td>Washington, DC</td>\n",
       "      <td>Yearly</td>\n",
       "      <td>$70,000</td>\n",
       "      <td>...</td>\n",
       "      <td>Washington, DC</td>\n",
       "      <td>87.740000</td>\n",
       "      <td>78.180000</td>\n",
       "      <td>83.170000</td>\n",
       "      <td>86.440000</td>\n",
       "      <td>87.480000</td>\n",
       "      <td>130.250000</td>\n",
       "      <td>high</td>\n",
       "      <td>Junior MS Access/ VBA Developer If you want an...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>939 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     Unnamed: 0                               Title                   Company  \\\n",
       "0             0  2021048 Data Scientist $215,000.00                    B4CORP   \n",
       "1             1           Data Science/Data Analyst                 CRAiC LLC   \n",
       "2             2                 Data Science Intern            MilliporeSigma   \n",
       "3             3                  Jr. Data Scientist           Numero Data LLC   \n",
       "4             4      Business Data Quality Engineer   HRU Technical Resources   \n",
       "..          ...                                 ...                       ...   \n",
       "934         958                 Data Engineering II   Pueo Business Solutions   \n",
       "935         959                        IR Day Shift   Pueo Business Solutions   \n",
       "936         960                   Security Engineer   Pueo Business Solutions   \n",
       "937         961                         FDP Analyst         GC Associates USA   \n",
       "938         962     Junior MS Access/ VBA Developer  Empirical Concepts, Inc.   \n",
       "\n",
       "                                                  Link Review  \\\n",
       "0    https://www.indeed.com/pagead/clk?mo=r&ad=-6NY...   None   \n",
       "1    https://www.indeed.com/company/CRAiC-LLC/jobs/...   None   \n",
       "2    https://www.indeed.com/company/Serono/jobs/Dat...    3.9   \n",
       "3    https://www.indeed.com/company/Numero-Data-LLC...   None   \n",
       "4    https://www.indeed.com/pagead/clk?mo=r&ad=-6NY...   None   \n",
       "..                                                 ...    ...   \n",
       "934  https://www.indeed.com/rc/clk?jk=90a6c966f45f8...      5   \n",
       "935  https://www.indeed.com/rc/clk?jk=e4416ec0bed30...      5   \n",
       "936  https://www.indeed.com/rc/clk?jk=c06c7f39a6074...      5   \n",
       "937  https://www.indeed.com/rc/clk?jk=84a4b28b5c409...   None   \n",
       "938  https://www.indeed.com/rc/clk?jk=916afc6c9b4a4...   None   \n",
       "\n",
       "                         Salary  \\\n",
       "0         Up to $215,000 a year   \n",
       "1                $60,000 a year   \n",
       "2             $20 - $25 an hour   \n",
       "3      $70,000 - $85,000 a year   \n",
       "4             $45 - $55 an hour   \n",
       "..                          ...   \n",
       "934  $108,000 - $118,000 a year   \n",
       "935    $89,000 - $99,000 a year   \n",
       "936    $89,000 - $99,000 a year   \n",
       "937    $80,000 - $90,000 a year   \n",
       "938    $70,000 - $90,000 a year   \n",
       "\n",
       "                                           Description       Location2  \\\n",
       "0    Position Requires a Top Secret (TS/SCI) Cleara...   Arlington, VA   \n",
       "1    Cost Readiness Analytics (CRAiC LLC) is lookin...   Arlington, VA   \n",
       "2    Job Title: Data Science Intern\\rA career with ...   Arlington, VA   \n",
       "3    Ideal candidate should have a degree in a quan...   Arlington, VA   \n",
       "4    Please note that this is not a corp to corp or...   Arlington, VA   \n",
       "..                                                 ...             ...   \n",
       "934  Data Engineering II\\rSalary: $108K-$118K\\rClea...  Washington, DC   \n",
       "935  Tier 2 IR Day Shift\\rSalary Range: $89K-$99K\\r...  Washington, DC   \n",
       "936  Security Engineer\\rSalary Range: $89K-$99K\\rCl...  Washington, DC   \n",
       "937  Position Title: FDP Analyst\\rCustomer: HQDA, G...  Washington, DC   \n",
       "938  If you want an accounting career with more opp...  Washington, DC   \n",
       "\n",
       "    Rate_by                    low  ...            City  Cost of Living Index  \\\n",
       "0    Yearly  Up to $215,000 a year  ...   Arlington, VA             80.159143   \n",
       "1    Yearly         $60,000 a year  ...   Arlington, VA             80.159143   \n",
       "2    Hourly                   $20   ...   Arlington, VA             80.159143   \n",
       "3    Yearly               $70,000   ...   Arlington, VA             80.159143   \n",
       "4    Hourly                   $45   ...   Arlington, VA             80.159143   \n",
       "..      ...                    ...  ...             ...                   ...   \n",
       "934  Yearly              $108,000   ...  Washington, DC             87.740000   \n",
       "935  Yearly               $89,000   ...  Washington, DC             87.740000   \n",
       "936  Yearly               $89,000   ...  Washington, DC             87.740000   \n",
       "937  Yearly               $80,000   ...  Washington, DC             87.740000   \n",
       "938  Yearly               $70,000   ...  Washington, DC             87.740000   \n",
       "\n",
       "     Rent Index  Cost of Living Plus Rent Index  Groceries Index  \\\n",
       "0     58.838571                       69.961143        76.886286   \n",
       "1     58.838571                       69.961143        76.886286   \n",
       "2     58.838571                       69.961143        76.886286   \n",
       "3     58.838571                       69.961143        76.886286   \n",
       "4     58.838571                       69.961143        76.886286   \n",
       "..          ...                             ...              ...   \n",
       "934   78.180000                       83.170000        86.440000   \n",
       "935   78.180000                       83.170000        86.440000   \n",
       "936   78.180000                       83.170000        86.440000   \n",
       "937   78.180000                       83.170000        86.440000   \n",
       "938   78.180000                       83.170000        86.440000   \n",
       "\n",
       "     Restaurant Price Index Local Purchasing Power Index  City_Bins  \\\n",
       "0                 78.608571                   115.594571     medium   \n",
       "1                 78.608571                   115.594571     medium   \n",
       "2                 78.608571                   115.594571     medium   \n",
       "3                 78.608571                   115.594571     medium   \n",
       "4                 78.608571                   115.594571     medium   \n",
       "..                      ...                          ...        ...   \n",
       "934               87.480000                   130.250000       high   \n",
       "935               87.480000                   130.250000       high   \n",
       "936               87.480000                   130.250000       high   \n",
       "937               87.480000                   130.250000       high   \n",
       "938               87.480000                   130.250000       high   \n",
       "\n",
       "                                         combined_text  Salary_Bins  \n",
       "0    2021048 Data Scientist $215,000.00 Position Re...            5  \n",
       "1    Data Science/Data Analyst Cost Readiness Analy...            1  \n",
       "2    Data Science Intern Job Title: Data Science In...            0  \n",
       "3    Jr. Data Scientist Ideal candidate should have...            2  \n",
       "4    Business Data Quality Engineer Please note tha...            3  \n",
       "..                                                 ...          ...  \n",
       "934  Data Engineering II Data Engineering II\\rSalar...            3  \n",
       "935  IR Day Shift Tier 2 IR Day Shift\\rSalary Range...            3  \n",
       "936  Security Engineer Security Engineer\\rSalary Ra...            3  \n",
       "937  FDP Analyst Position Title: FDP Analyst\\rCusto...            2  \n",
       "938  Junior MS Access/ VBA Developer If you want an...            2  \n",
       "\n",
       "[939 rows x 26 columns]"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "salary_df_new"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "salary_df_new = salary_df_new.drop(['Unnamed: 0'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "job_title = salary_df_new['Title']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "#use count vectorizer to examine title\n",
    "\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "cv = CountVectorizer(stop_words='english', ngram_range=(1,3), max_df=0.3, min_df=0.03)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "#fitting count vectorizer to job title\n",
    "\n",
    "cv.fit(job_title)\n",
    "\n",
    "title_words = pd.DataFrame(cv.transform(job_title).todense(), columns=cv.get_feature_names())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>analyst</th>\n",
       "      <th>analytics</th>\n",
       "      <th>business</th>\n",
       "      <th>business analyst</th>\n",
       "      <th>data</th>\n",
       "      <th>data analyst</th>\n",
       "      <th>data engineer</th>\n",
       "      <th>data scientist</th>\n",
       "      <th>digital</th>\n",
       "      <th>director</th>\n",
       "      <th>...</th>\n",
       "      <th>marketing</th>\n",
       "      <th>sales</th>\n",
       "      <th>science</th>\n",
       "      <th>scientist</th>\n",
       "      <th>senior</th>\n",
       "      <th>senior data</th>\n",
       "      <th>software</th>\n",
       "      <th>software engineer</th>\n",
       "      <th>specialist</th>\n",
       "      <th>systems</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>939.000000</td>\n",
       "      <td>939.000000</td>\n",
       "      <td>939.000000</td>\n",
       "      <td>939.000000</td>\n",
       "      <td>939.000000</td>\n",
       "      <td>939.000000</td>\n",
       "      <td>939.000000</td>\n",
       "      <td>939.000000</td>\n",
       "      <td>939.000000</td>\n",
       "      <td>939.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>939.000000</td>\n",
       "      <td>939.000000</td>\n",
       "      <td>939.000000</td>\n",
       "      <td>939.000000</td>\n",
       "      <td>939.000000</td>\n",
       "      <td>939.000000</td>\n",
       "      <td>939.000000</td>\n",
       "      <td>939.000000</td>\n",
       "      <td>939.000000</td>\n",
       "      <td>939.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.299255</td>\n",
       "      <td>0.033014</td>\n",
       "      <td>0.153355</td>\n",
       "      <td>0.103301</td>\n",
       "      <td>0.310969</td>\n",
       "      <td>0.057508</td>\n",
       "      <td>0.085197</td>\n",
       "      <td>0.071353</td>\n",
       "      <td>0.036209</td>\n",
       "      <td>0.040469</td>\n",
       "      <td>...</td>\n",
       "      <td>0.083067</td>\n",
       "      <td>0.040469</td>\n",
       "      <td>0.033014</td>\n",
       "      <td>0.074547</td>\n",
       "      <td>0.123536</td>\n",
       "      <td>0.034079</td>\n",
       "      <td>0.044728</td>\n",
       "      <td>0.031949</td>\n",
       "      <td>0.033014</td>\n",
       "      <td>0.033014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.474184</td>\n",
       "      <td>0.178768</td>\n",
       "      <td>0.360521</td>\n",
       "      <td>0.304514</td>\n",
       "      <td>0.509187</td>\n",
       "      <td>0.232935</td>\n",
       "      <td>0.290548</td>\n",
       "      <td>0.257550</td>\n",
       "      <td>0.186909</td>\n",
       "      <td>0.207694</td>\n",
       "      <td>...</td>\n",
       "      <td>0.276131</td>\n",
       "      <td>0.197161</td>\n",
       "      <td>0.178768</td>\n",
       "      <td>0.262800</td>\n",
       "      <td>0.329226</td>\n",
       "      <td>0.181528</td>\n",
       "      <td>0.216882</td>\n",
       "      <td>0.175958</td>\n",
       "      <td>0.178768</td>\n",
       "      <td>0.178768</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          analyst   analytics    business  business analyst        data  \\\n",
       "count  939.000000  939.000000  939.000000        939.000000  939.000000   \n",
       "mean     0.299255    0.033014    0.153355          0.103301    0.310969   \n",
       "std      0.474184    0.178768    0.360521          0.304514    0.509187   \n",
       "min      0.000000    0.000000    0.000000          0.000000    0.000000   \n",
       "25%      0.000000    0.000000    0.000000          0.000000    0.000000   \n",
       "50%      0.000000    0.000000    0.000000          0.000000    0.000000   \n",
       "75%      1.000000    0.000000    0.000000          0.000000    1.000000   \n",
       "max      2.000000    1.000000    1.000000          1.000000    3.000000   \n",
       "\n",
       "       data analyst  data engineer  data scientist     digital    director  \\\n",
       "count    939.000000     939.000000      939.000000  939.000000  939.000000   \n",
       "mean       0.057508       0.085197        0.071353    0.036209    0.040469   \n",
       "std        0.232935       0.290548        0.257550    0.186909    0.207694   \n",
       "min        0.000000       0.000000        0.000000    0.000000    0.000000   \n",
       "25%        0.000000       0.000000        0.000000    0.000000    0.000000   \n",
       "50%        0.000000       0.000000        0.000000    0.000000    0.000000   \n",
       "75%        0.000000       0.000000        0.000000    0.000000    0.000000   \n",
       "max        1.000000       2.000000        1.000000    1.000000    2.000000   \n",
       "\n",
       "       ...   marketing       sales     science   scientist      senior  \\\n",
       "count  ...  939.000000  939.000000  939.000000  939.000000  939.000000   \n",
       "mean   ...    0.083067    0.040469    0.033014    0.074547    0.123536   \n",
       "std    ...    0.276131    0.197161    0.178768    0.262800    0.329226   \n",
       "min    ...    0.000000    0.000000    0.000000    0.000000    0.000000   \n",
       "25%    ...    0.000000    0.000000    0.000000    0.000000    0.000000   \n",
       "50%    ...    0.000000    0.000000    0.000000    0.000000    0.000000   \n",
       "75%    ...    0.000000    0.000000    0.000000    0.000000    0.000000   \n",
       "max    ...    1.000000    1.000000    1.000000    1.000000    1.000000   \n",
       "\n",
       "       senior data    software  software engineer  specialist     systems  \n",
       "count   939.000000  939.000000         939.000000  939.000000  939.000000  \n",
       "mean      0.034079    0.044728           0.031949    0.033014    0.033014  \n",
       "std       0.181528    0.216882           0.175958    0.178768    0.178768  \n",
       "min       0.000000    0.000000           0.000000    0.000000    0.000000  \n",
       "25%       0.000000    0.000000           0.000000    0.000000    0.000000  \n",
       "50%       0.000000    0.000000           0.000000    0.000000    0.000000  \n",
       "75%       0.000000    0.000000           0.000000    0.000000    0.000000  \n",
       "max       1.000000    2.000000           1.000000    1.000000    1.000000  \n",
       "\n",
       "[8 rows x 25 columns]"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "title_words.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "#One-hot encoding of categorical feature City_Bins\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "labelencoder = LabelEncoder()\n",
    "\n",
    "salary_df_new['City_cat']=labelencoder.fit_transform(salary_df_new['City_Bins'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0      2\n",
       "1      2\n",
       "2      2\n",
       "3      2\n",
       "4      2\n",
       "      ..\n",
       "934    0\n",
       "935    0\n",
       "936    0\n",
       "937    0\n",
       "938    0\n",
       "Name: City_cat, Length: 939, dtype: int64"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "salary_df_new['City_cat']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Construct model dataset\n",
    "X = pd.concat([title_words, salary_df_new['City_cat'], salary_df_new['Cost of Living Plus Rent Index']], axis=1)\n",
    "y = salary_df_new['Salary_Bins'].astype('category')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Train-Test Split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.1, random_state=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>analyst</th>\n",
       "      <th>analytics</th>\n",
       "      <th>business</th>\n",
       "      <th>business analyst</th>\n",
       "      <th>data</th>\n",
       "      <th>data analyst</th>\n",
       "      <th>data engineer</th>\n",
       "      <th>data scientist</th>\n",
       "      <th>digital</th>\n",
       "      <th>director</th>\n",
       "      <th>...</th>\n",
       "      <th>sales</th>\n",
       "      <th>science</th>\n",
       "      <th>scientist</th>\n",
       "      <th>senior</th>\n",
       "      <th>senior data</th>\n",
       "      <th>software</th>\n",
       "      <th>software engineer</th>\n",
       "      <th>specialist</th>\n",
       "      <th>systems</th>\n",
       "      <th>Salary_Bins</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   analyst  analytics  business  business analyst  data  data analyst  \\\n",
       "0        0          0         0                 0     1             0   \n",
       "1        1          0         0                 0     2             1   \n",
       "2        0          0         0                 0     1             0   \n",
       "3        0          0         0                 0     1             0   \n",
       "4        0          0         1                 0     1             0   \n",
       "\n",
       "   data engineer  data scientist  digital  director  ...  sales  science  \\\n",
       "0              0               1        0         0  ...      0        0   \n",
       "1              0               0        0         0  ...      0        1   \n",
       "2              0               0        0         0  ...      0        1   \n",
       "3              0               1        0         0  ...      0        0   \n",
       "4              0               0        0         0  ...      0        0   \n",
       "\n",
       "   scientist  senior  senior data  software  software engineer  specialist  \\\n",
       "0          1       0            0         0                  0           0   \n",
       "1          0       0            0         0                  0           0   \n",
       "2          0       0            0         0                  0           0   \n",
       "3          1       0            0         0                  0           0   \n",
       "4          0       0            0         0                  0           0   \n",
       "\n",
       "   systems  Salary_Bins  \n",
       "0        0            5  \n",
       "1        0            1  \n",
       "2        0            0  \n",
       "3        0            2  \n",
       "4        0            3  \n",
       "\n",
       "[5 rows x 26 columns]"
      ]
     },
     "execution_count": 114,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Add City Bins and Local Purchase Power Index\n",
    "word_observe = pd.concat([title_words, y], axis=1)\n",
    "\n",
    "word_observe.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "table=pd.pivot_table(word_observe, index=['Salary_Bins'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>analyst</th>\n",
       "      <th>analytics</th>\n",
       "      <th>business</th>\n",
       "      <th>business analyst</th>\n",
       "      <th>data</th>\n",
       "      <th>data analyst</th>\n",
       "      <th>data engineer</th>\n",
       "      <th>data scientist</th>\n",
       "      <th>digital</th>\n",
       "      <th>director</th>\n",
       "      <th>...</th>\n",
       "      <th>marketing</th>\n",
       "      <th>sales</th>\n",
       "      <th>science</th>\n",
       "      <th>scientist</th>\n",
       "      <th>senior</th>\n",
       "      <th>senior data</th>\n",
       "      <th>software</th>\n",
       "      <th>software engineer</th>\n",
       "      <th>specialist</th>\n",
       "      <th>systems</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Salary_Bins</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.164179</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.037313</td>\n",
       "      <td>0.022388</td>\n",
       "      <td>0.141791</td>\n",
       "      <td>0.029851</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.022388</td>\n",
       "      <td>0.067164</td>\n",
       "      <td>0.007463</td>\n",
       "      <td>...</td>\n",
       "      <td>0.208955</td>\n",
       "      <td>0.029851</td>\n",
       "      <td>0.044776</td>\n",
       "      <td>0.022388</td>\n",
       "      <td>0.022388</td>\n",
       "      <td>0.007463</td>\n",
       "      <td>0.007463</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.067164</td>\n",
       "      <td>0.007463</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.400000</td>\n",
       "      <td>0.010811</td>\n",
       "      <td>0.189189</td>\n",
       "      <td>0.162162</td>\n",
       "      <td>0.172973</td>\n",
       "      <td>0.059459</td>\n",
       "      <td>0.010811</td>\n",
       "      <td>0.059459</td>\n",
       "      <td>0.027027</td>\n",
       "      <td>0.027027</td>\n",
       "      <td>...</td>\n",
       "      <td>0.102703</td>\n",
       "      <td>0.043243</td>\n",
       "      <td>0.021622</td>\n",
       "      <td>0.064865</td>\n",
       "      <td>0.037838</td>\n",
       "      <td>0.005405</td>\n",
       "      <td>0.005405</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.070270</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.033333</td>\n",
       "      <td>0.206667</td>\n",
       "      <td>0.140000</td>\n",
       "      <td>0.293333</td>\n",
       "      <td>0.140000</td>\n",
       "      <td>0.033333</td>\n",
       "      <td>0.053333</td>\n",
       "      <td>0.086667</td>\n",
       "      <td>0.026667</td>\n",
       "      <td>...</td>\n",
       "      <td>0.093333</td>\n",
       "      <td>0.066667</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.053333</td>\n",
       "      <td>0.086667</td>\n",
       "      <td>0.026667</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.033333</td>\n",
       "      <td>0.046667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.308140</td>\n",
       "      <td>0.046512</td>\n",
       "      <td>0.180233</td>\n",
       "      <td>0.098837</td>\n",
       "      <td>0.389535</td>\n",
       "      <td>0.069767</td>\n",
       "      <td>0.127907</td>\n",
       "      <td>0.069767</td>\n",
       "      <td>0.040698</td>\n",
       "      <td>0.040698</td>\n",
       "      <td>...</td>\n",
       "      <td>0.040698</td>\n",
       "      <td>0.023256</td>\n",
       "      <td>0.029070</td>\n",
       "      <td>0.069767</td>\n",
       "      <td>0.151163</td>\n",
       "      <td>0.029070</td>\n",
       "      <td>0.034884</td>\n",
       "      <td>0.017442</td>\n",
       "      <td>0.011628</td>\n",
       "      <td>0.087209</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.260870</td>\n",
       "      <td>0.062112</td>\n",
       "      <td>0.192547</td>\n",
       "      <td>0.124224</td>\n",
       "      <td>0.453416</td>\n",
       "      <td>0.018634</td>\n",
       "      <td>0.180124</td>\n",
       "      <td>0.105590</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.043478</td>\n",
       "      <td>...</td>\n",
       "      <td>0.049689</td>\n",
       "      <td>0.037267</td>\n",
       "      <td>0.049689</td>\n",
       "      <td>0.111801</td>\n",
       "      <td>0.180124</td>\n",
       "      <td>0.043478</td>\n",
       "      <td>0.062112</td>\n",
       "      <td>0.031056</td>\n",
       "      <td>0.012422</td>\n",
       "      <td>0.031056</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.109489</td>\n",
       "      <td>0.043796</td>\n",
       "      <td>0.080292</td>\n",
       "      <td>0.043796</td>\n",
       "      <td>0.416058</td>\n",
       "      <td>0.021898</td>\n",
       "      <td>0.160584</td>\n",
       "      <td>0.116788</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.102190</td>\n",
       "      <td>...</td>\n",
       "      <td>0.014599</td>\n",
       "      <td>0.043796</td>\n",
       "      <td>0.058394</td>\n",
       "      <td>0.124088</td>\n",
       "      <td>0.277372</td>\n",
       "      <td>0.102190</td>\n",
       "      <td>0.175182</td>\n",
       "      <td>0.160584</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.021898</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              analyst  analytics  business  business analyst      data  \\\n",
       "Salary_Bins                                                              \n",
       "0            0.164179   0.000000  0.037313          0.022388  0.141791   \n",
       "1            0.400000   0.010811  0.189189          0.162162  0.172973   \n",
       "2            0.500000   0.033333  0.206667          0.140000  0.293333   \n",
       "3            0.308140   0.046512  0.180233          0.098837  0.389535   \n",
       "4            0.260870   0.062112  0.192547          0.124224  0.453416   \n",
       "5            0.109489   0.043796  0.080292          0.043796  0.416058   \n",
       "\n",
       "             data analyst  data engineer  data scientist   digital  director  \\\n",
       "Salary_Bins                                                                    \n",
       "0                0.029851       0.000000        0.022388  0.067164  0.007463   \n",
       "1                0.059459       0.010811        0.059459  0.027027  0.027027   \n",
       "2                0.140000       0.033333        0.053333  0.086667  0.026667   \n",
       "3                0.069767       0.127907        0.069767  0.040698  0.040698   \n",
       "4                0.018634       0.180124        0.105590  0.000000  0.043478   \n",
       "5                0.021898       0.160584        0.116788  0.000000  0.102190   \n",
       "\n",
       "             ...  marketing     sales   science  scientist    senior  \\\n",
       "Salary_Bins  ...                                                       \n",
       "0            ...   0.208955  0.029851  0.044776   0.022388  0.022388   \n",
       "1            ...   0.102703  0.043243  0.021622   0.064865  0.037838   \n",
       "2            ...   0.093333  0.066667  0.000000   0.053333  0.086667   \n",
       "3            ...   0.040698  0.023256  0.029070   0.069767  0.151163   \n",
       "4            ...   0.049689  0.037267  0.049689   0.111801  0.180124   \n",
       "5            ...   0.014599  0.043796  0.058394   0.124088  0.277372   \n",
       "\n",
       "             senior data  software  software engineer  specialist   systems  \n",
       "Salary_Bins                                                                  \n",
       "0               0.007463  0.007463           0.000000    0.067164  0.007463  \n",
       "1               0.005405  0.005405           0.000000    0.070270  0.000000  \n",
       "2               0.026667  0.000000           0.000000    0.033333  0.046667  \n",
       "3               0.029070  0.034884           0.017442    0.011628  0.087209  \n",
       "4               0.043478  0.062112           0.031056    0.012422  0.031056  \n",
       "5               0.102190  0.175182           0.160584    0.000000  0.021898  \n",
       "\n",
       "[6 rows x 25 columns]"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "table"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "From the distribution above, we can tell some words appear more for higher salary bin than others."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x13edb0e80>"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEECAYAAAAmiP8hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAQ5klEQVR4nO3df6xfdX3H8efLVopiBgt0qC31stFtKZMxrXVOF3+waRFnlwySYjKZYVYTyVzcnHVuDIkuMJc1i+IfnaCI24rrZtaNTlwgaMa0tii/CtZdEaXrmK0gpiA/qu/98T0Nd19ve097b++399PnIyE953M+5573oe3rfvq553y+qSokSe16xqgLkCQdWQa9JDXOoJekxhn0ktQ4g16SGmfQS1Lj5o+6gGGnnHJKjY2NjboMSZpTbrvttj1VtXCyY0dd0I+NjbFt27ZRlyFJc0qSbx3omFM3ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYddS9M6dg0tvaGWb3e/VecN6vXk0bJEb0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktS4XkGfZGWSHUnGk6yd5PiCJNd3x7ckGevan5nk2iR3Jbk3yXtntnxJ0lSmDPok84CrgHOBZcCFSZYNdbsYeLiqzgDWAVd27RcAC6rqhcCLgbft/yYgSZodfUb0K4Dxqrqvqp4ENgCrhvqsAq7ttjcC5yQJUMAJSeYDzwKeBL4/I5VLknrpE/SLgAcm7O/s2ibtU1X7gEeAkxmE/qPA/wDfBv6yqh6aZs2SpEPQJ+gzSVv17LMC+CHwfOB04A+S/PSPXSBZk2Rbkm27d+/uUZIkqa8+Qb8TOG3C/mJg14H6dNM0JwIPAW8CPltVT1XVd4BbgeXDF6iq9VW1vKqWL1y48NDvQpJ0QH2CfiuwNMnpSY4DVgObhvpsAi7qts8Hbq6qYjBd85oMnAD8MvC1mSldktTHlEHfzblfAtwI3At8uqq2J7k8yRu7blcDJycZB94F7H8E8yrgOcDdDL5hfLyq7pzhe5AkHUSvz4ytqs3A5qG2SydsP87gUcrh8/ZO1i5Jmj2+GStJjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalx80ddwEwZW3vDrF7v/ivOm9XrSdLhckQvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMb1CvokK5PsSDKeZO0kxxckub47viXJ2IRjZyX5YpLtSe5KcvzMlS9JmsqUQZ9kHnAVcC6wDLgwybKhbhcDD1fVGcA64Mru3PnAp4C3V9WZwKuAp2aseknSlPqM6FcA41V1X1U9CWwAVg31WQVc221vBM5JEuC1wJ1VdQdAVX23qn44M6VLkvroE/SLgAcm7O/s2ibtU1X7gEeAk4GfBSrJjUm+kuSPpl+yJOlQ9PngkUzSVj37zAdeAbwEeAy4KcltVXXT/zs5WQOsAViyZEmPkiRJffUZ0e8ETpuwvxjYdaA+3bz8icBDXfvnq2pPVT0GbAZeNHyBqlpfVcuravnChQsP/S4kSQfUJ+i3AkuTnJ7kOGA1sGmozybgom77fODmqirgRuCsJM/uvgG8ErhnZkqXJPUx5dRNVe1LcgmD0J4HXFNV25NcDmyrqk3A1cB1ScYZjORXd+c+nOSvGHyzKGBzVc3uh7tK0jGu14eDV9VmBtMuE9sunbD9OHDBAc79FINHLCVJI+CbsZLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDWuV9AnWZlkR5LxJGsnOb4gyfXd8S1JxoaOL0myN8kfzkzZkqS+pgz6JPOAq4BzgWXAhUmWDXW7GHi4qs4A1gFXDh1fB/zb9MuVJB2q+T36rADGq+o+gCQbgFXAPRP6rAIu67Y3Ah9JkqqqJL8J3Ac8OmNVS3PM2NobZvV6919x3qxeT0e3PlM3i4AHJuzv7Nom7VNV+4BHgJOTnAC8B3j/9EuVJB2OPkGfSdqqZ5/3A+uqau9BL5CsSbItybbdu3f3KEmS1FefqZudwGkT9hcDuw7QZ2eS+cCJwEPAS4Hzk/wFcBLwoySPV9VHJp5cVeuB9QDLly8f/iYiSZqGPkG/FVia5HTgv4HVwJuG+mwCLgK+CJwP3FxVBfzq/g5JLgP2Doe8JOnImjLoq2pfkkuAG4F5wDVVtT3J5cC2qtoEXA1cl2ScwUh+9ZEsWpLUX58RPVW1Gdg81HbphO3HgQum+BqXHUZ9kqRp8s1YSWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjZs/6gLUz9jaG2b1evdfcd6sXk/SkeOIXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mN6xX0SVYm2ZFkPMnaSY4vSHJ9d3xLkrGu/deT3Jbkru7X18xs+ZKkqUwZ9EnmAVcB5wLLgAuTLBvqdjHwcFWdAawDruza9wC/UVUvBC4CrpupwiVJ/fQZ0a8Axqvqvqp6EtgArBrqswq4ttveCJyTJFX11ara1bVvB45PsmAmCpck9dMn6BcBD0zY39m1TdqnqvYBjwAnD/X5LeCrVfXE4ZUqSTocfT54JJO01aH0SXImg+mc1056gWQNsAZgyZIlPUqSJPXVZ0S/Ezhtwv5iYNeB+iSZD5wIPNTtLwY+A7y5qr4x2QWqan1VLa+q5QsXLjy0O5AkHVSfoN8KLE1yepLjgNXApqE+mxj8sBXgfODmqqokJwE3AO+tqltnqmhJUn9TBn03534JcCNwL/Dpqtqe5PIkb+y6XQ2cnGQceBew/xHMS4AzgD9Ncnv330/N+F1Ikg6o14eDV9VmYPNQ26UTth8HLpjkvA8AH5hmjZKkafDNWElqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhrXK+iTrEyyI8l4krWTHF+Q5Pru+JYkYxOOvbdr35HkdTNXuiSpjymDPsk84CrgXGAZcGGSZUPdLgYerqozgHXAld25y4DVwJnASuCj3deTJM2SPiP6FcB4Vd1XVU8CG4BVQ31WAdd22xuBc5Kka99QVU9U1TeB8e7rSZJmSZ+gXwQ8MGF/Z9c2aZ+q2gc8Apzc81xJ0hE0v0efTNJWPfv0OZcka4A13e7eJDt61DVTTgH2HOpJufIIVHJkeH+T8P6OCod1b3PIbN/fCw50oE/Q7wROm7C/GNh1gD47k8wHTgQe6nkuVbUeWN+jlhmXZFtVLR/FtWeD9ze3tXx/Ld8bHF3312fqZiuwNMnpSY5j8MPVTUN9NgEXddvnAzdXVXXtq7unck4HlgJfnpnSJUl9TDmir6p9SS4BbgTmAddU1fYklwPbqmoTcDVwXZJxBiP51d2525N8GrgH2Ae8o6p+eITuRZI0iT5TN1TVZmDzUNulE7YfBy44wLkfBD44jRqPtJFMGc0i729ua/n+Wr43OIruL4MZFklSq1wCQZIaZ9BLUuN6zdG3JMnPM3hjdxGDZ/p3AZuq6t6RFqZeut+/RcCWqto7oX1lVX12dJVNX5IVQFXV1m75kJXA17qfkTUnySer6s2jruNISPIKBqsA3F1Vnxt5PcfSHH2S9wAXMljGYWfXvJjBU0IbquqKUdV2pCV5S1V9fNR1TEeS3wPeAdwLnA28s6r+uTv2lap60Sjrm44kf8ZgPan5wL8DLwVuAX4NuLF7qGHOSjL8SHaAVwM3A1TVG2e9qBmU5MtVtaLbfiuDP6efAV4L/Muos+VYC/qvA2dW1VND7ccB26tq6WgqO/KSfLuqloy6julIchfwsqra262QuhG4rqr+OslXq+qXRlrgNHT3djawAHgQWFxV30/yLAb/ejlrpAVOU5KvMHjM+mM8/db83/P0o9ifH1110zfxz1+SrcDrq2p3khOAL1XVC0dZ37E2dfMj4PnAt4ban9cdm9OS3HmgQ8Cps1nLETJv/3RNVd2f5FXAxiQvYPLlNuaSfd07Jo8l+UZVfR+gqn6QZM7/2QSWA+8E3ge8u6puT/KDuR7wEzwjyU8y+Llnqmo3QFU9mmTfaEs79oL+94GbkvwXTy+2tgQ4A7hkZFXNnFOB1wEPD7UH+M/ZL2fGPZjk7Kq6HaAb2b8BuAYY6YhpBjyZ5NlV9Rjw4v2NSU6kgUFIVf0IWJfkH7pf/5e28udE4DYGf9cqyXOr6sEkz+EoGIQcU1M3AEmeweCHJIsY/AbsBLa28MZukquBj1fVf0xy7O+q6k0jKGvGJFnMYOT74CTHXl5Vt46grBmRZEFVPTFJ+ynA86rqrhGUdcQkOQ94eVX98ahrOZKSPBs4tVumfXR1HGtBL0nHGp+jl6TGGfSS1DiDXpIaZ9BrTkvyviTbk9yZ5PYkLz1I308kOf8I1vKJJN/s6vha9xLU/mMf6952lWZdS4836RiT5GXAG4AXVdUT3RMqx83g15/ffQbyoXh3VW1McjxwT/ea/zer6ndnqi7pUDmi11z2PGDP/scSq2pPVe1KcmmSrUnuTrI+yY89x3ygPkluSfLnST4PvK8boT+zO/YTSe7fvz+F47tfH53wdZd323uTfDDJHUm+lOTUrv2Crp47knxhuv9zpP0Mes1lnwNOS/L1JB9N8squ/SNV9ZKq+gXgWQxG/cMO1uekqnplVb2fwXoz53Xtq4F/HF5CY8iHktzO4P2MDVX1nUn67H8t/heBLwBv7dovBV7Xtc/ptV90dDHoNWd1yyG8GFgD7AauT/I7wKuTbOnWj3kNcOYkpx+sz/UTtj8GvKXbfgsw1cJw766qs4HnAuck+ZVJ+jwJ/Gu3fRsw1m3fCnyiWxRr3hTXkXpzjl5zWvdG8y3ALV1ovw04C1heVQ8kuYynp1EA6ObPP3qQPo9O+Pq3Jhnr/rUwr6ru7lnX3iS3AK/gx5efeKqeflPxh3R/D6vq7d0Pk88Dbu+We/hun+tJB+OIXnNWkp9LMnHF0bOBHd32nm6dkcmesjm+R5+JPslgpcXeyzwnmc9gqeFvHMI5P1NVW7rPY94DnNb3XOlgHNFrLnsO8OEkJwH7gHEG0zjfA+4C7ge2Dp9UVd9L8jcH6zPkb4EPMAj7qXwoyZ8wePrnJuCf+tzIhHOXMliD6SbgjkM4Vzog17qRptA9e7+qqn571LVIh8MRvXQQST7M4JOfXj/qWqTD5YheOkRJrgJePtT813P9oxrVLoNekhrnUzeS1DiDXpIaZ9BLUuMMeklqnEEvSY37P7ertHNYnwtsAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#'Word' digital mostly appear in lower bins\n",
    "table['digital'].plot.bar()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x142a08b70>"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEECAYAAAAmiP8hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAR/0lEQVR4nO3df6xfdX3H8efLVoo/MnDQobZ0l41uS5mMaa1zuvmDDYo4umSQFJOJhllNJHNxc9a5ITJdYC4jRjEZExRxrjA2s250ooGgGVNsQQQrVq+Ictc5iyCmKD+q7/3xPQ13X7/tPeXe3i/3w/ORND3ncz6f73kfaF/303PP+dxUFZKkdj1p3AVIkg4ug16SGmfQS1LjDHpJapxBL0mNM+glqXGLx13AsCOPPLImJibGXYYkLSg333zzPVW1dNSxx13QT0xMsG3btnGXIUkLSpJv7uuYt24kqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjXvcvTAlSY8nExuvmdfz3XXBqXP+mb1m9EnWJtmRZDLJxhHHfzPJLUn2JDl96NhZSb7W/TprrgqXJPUzY9AnWQRcDJwCrALOTLJqqNu3gNcAHxsa+9PAO4AXAGuAdyR5xuzLliT11WdGvwaYrKo7q+phYBOwbnqHqrqrqm4Dfjw09mTgU1V1b1XdB3wKWDsHdUuSeuoT9MuAu6ftT3VtfcxmrCRpDvQJ+oxoq56f32tskg1JtiXZtmvXrp4fLUnqo0/QTwFHT9tfDuzs+fm9xlbVJVW1uqpWL106cjllSdJj1CfotwIrkxyT5BBgPbC55+dfC5yU5BndN2FP6tokSfNkxqCvqj3AOQwC+g7gqqranuT8JKcBJHl+kingDODvkmzvxt4L/CWDLxZbgfO7NknSPOn1wlRVbQG2DLWdO217K4PbMqPGXgZcNosaJUmz4BIIktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMb1Cvoka5PsSDKZZOOI40uSXNkdvynJRNf+5CSXJ7k9yR1J3ja35UuSZjJj0CdZBFwMnAKsAs5Msmqo29nAfVV1LHARcGHXfgawpKqeAzwPeP3eLwKSpPnRZ0a/Bpisqjur6mFgE7BuqM864PJu+2rgxCQBCnhaksXAU4CHge/PSeWSpF76BP0y4O5p+1Nd28g+VbUHuB84gkHoPwD8D/At4G+q6t7hEyTZkGRbkm27du064IuQJO1bn6DPiLbq2WcN8CPg2cAxwB8n+bmf6Fh1SVWtrqrVS5cu7VGSJKmvPkE/BRw9bX85sHNffbrbNIcB9wKvAj5RVY9U1XeAG4HVsy1aktRfn6DfCqxMckySQ4D1wOahPpuBs7rt04Hrq6oY3K55eQaeBvwa8JW5KV2S1MeMQd/dcz8HuBa4A7iqqrYnOT/JaV23S4EjkkwCbwb2PoJ5MfB04EsMvmB8qKpum+NrkCTtx+I+napqC7BlqO3cadsPMniUcnjc7lHtkqT545uxktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1rlfQJ1mbZEeSySQbRxxfkuTK7vhNSSamHTs+yWeTbE9ye5JD5658SdJMZgz6JIuAi4FTgFXAmUlWDXU7G7ivqo4FLgIu7MYuBj4KvKGqjgNeCjwyZ9VLkmbUZ0a/Bpisqjur6mFgE7BuqM864PJu+2rgxCQBTgJuq6ovAlTVd6vqR3NTuiSpjz5Bvwy4e9r+VNc2sk9V7QHuB44AfgGoJNcmuSXJn86+ZEnSgVjco09GtFXPPouBFwPPB34AXJfk5qq67v8NTjYAGwBWrFjRoyRJUl99ZvRTwNHT9pcDO/fVp7svfxhwb9f+6aq6p6p+AGwBnjt8gqq6pKpWV9XqpUuXHvhVSJL2qU/QbwVWJjkmySHAemDzUJ/NwFnd9unA9VVVwLXA8Ume2n0BeAnw5bkpXZLUx4y3bqpqT5JzGIT2IuCyqtqe5HxgW1VtBi4FrkgyyWAmv74be1+Sv2XwxaKALVV1zUG6FknSCH3u0VNVWxjcdpnedu607QeBM/Yx9qMMHrGUJI2Bb8ZKUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcYvHXYCkhW1i4zXzer67Ljh1Xs/XAmf0ktQ4g16SGmfQS1LjegV9krVJdiSZTLJxxPElSa7sjt+UZGLo+Ioku5P8ydyULUnqa8agT7IIuBg4BVgFnJlk1VC3s4H7qupY4CLgwqHjFwH/MftyJUkHqs9TN2uAyaq6EyDJJmAd8OVpfdYB53XbVwPvT5KqqiS/C9wJPDBnVUsLjE+maJz63LpZBtw9bX+qaxvZp6r2APcDRyR5GvBW4J37O0GSDUm2Jdm2a9euvrVLknroE/QZ0VY9+7wTuKiqdu/vBFV1SVWtrqrVS5cu7VGSJKmvPrdupoCjp+0vB3buo89UksXAYcC9wAuA05P8NXA48OMkD1bV+2dduSSplz5BvxVYmeQY4L+B9cCrhvpsBs4CPgucDlxfVQX8xt4OSc4DdhvykjS/Zgz6qtqT5BzgWmARcFlVbU9yPrCtqjYDlwJXJJlkMJNffzCLliT112utm6raAmwZajt32vaDwBkzfMZ5j6E+SdIs+WasJDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMWj7sA9TOx8Zp5Pd9dF5w6r+dr/fqkcXJGL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjesV9EnWJtmRZDLJxhHHlyS5sjt+U5KJrv23k9yc5Pbu95fPbfmSpJnMGPRJFgEXA6cAq4Azk6wa6nY2cF9VHQtcBFzYtd8D/E5VPQc4C7hirgqXJPXTZ0a/Bpisqjur6mFgE7BuqM864PJu+2rgxCSpqi9U1c6ufTtwaJIlc1G4JKmfPkG/DLh72v5U1zayT1XtAe4Hjhjq83vAF6rqocdWqiTpseizemVGtNWB9ElyHIPbOSeNPEGyAdgAsGLFih4lSZL66jOjnwKOnra/HNi5rz5JFgOHAfd2+8uBjwOvrqqvjzpBVV1SVauravXSpUsP7AokSfvVJ+i3AiuTHJPkEGA9sHmoz2YG32wFOB24vqoqyeHANcDbqurGuSpaktTfjEHf3XM/B7gWuAO4qqq2Jzk/yWldt0uBI5JMAm8G9j6CeQ5wLPAXSW7tfv3MnF+FJGmfev2EqaraAmwZajt32vaDwBkjxr0LeNcsa5QkzYJvxkpS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXG9fjj4QjCx8Zp5Pd9dF5w6r+eTpMfKGb0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktS4XkGfZG2SHUkmk2wccXxJkiu74zclmZh27G1d+44kJ89d6ZKkPmYM+iSLgIuBU4BVwJlJVg11Oxu4r6qOBS4CLuzGrgLWA8cBa4EPdJ8nSZonfWb0a4DJqrqzqh4GNgHrhvqsAy7vtq8GTkySrn1TVT1UVd8AJrvPkyTNkz5Bvwy4e9r+VNc2sk9V7QHuB47oOVaSdBD1WY8+I9qqZ58+Y0myAdjQ7e5OsqNHXXPlSOCeAx2UCw9CJQeH1zeC1/e40PK1wfxf38/u60CfoJ8Cjp62vxzYuY8+U0kWA4cB9/YcS1VdAlzSo5Y5l2RbVa0ex7nng9e3sLV8fS1fGzy+rq/PrZutwMokxyQ5hME3VzcP9dkMnNVtnw5cX1XVta/vnso5BlgJfH5uSpck9THjjL6q9iQ5B7gWWARcVlXbk5wPbKuqzcClwBVJJhnM5Nd3Y7cnuQr4MrAHeGNV/eggXYskaYRePzO2qrYAW4bazp22/SBwxj7Gvht49yxqPNjGcstoHnl9C1vL19fytcHj6PoyuMMiSWqVSyBIUuMMeklqXK979C1J8ksM3thdxuCZ/p3A5qq6Y6yFqZfu/98y4Kaq2j2tfW1VfWJ8lc1ekjVAVdXWbvmQtcBXuu+RNSfJR6rq1eOu42BI8mIGqwB8qao+OfZ6nkj36JO8FTiTwTIOU13zcgZPCW2qqgvGVdvBluS1VfWhcdcxG0n+EHgjcAdwAvCmqvrX7tgtVfXccdY3G0newWA9qcXAp4AXADcAvwVc2z3UsGAlGX4kO8DLgOsBquq0eS9qDiX5fFWt6bZfx+DP6ceBk4B/G3e2PNGC/qvAcVX1yFD7IcD2qlo5nsoOviTfqqoV465jNpLcDrywqnZ3K6ReDVxRVe9N8oWq+tWxFjgL3bWdACwBvg0sr6rvJ3kKg3+9HD/WAmcpyS0MHrP+II++Nf+PPPoo9qfHV93sTf/zl2Qr8Iqq2pXkacDnquo546zviXbr5sfAs4FvDrU/qzu2oCW5bV+HgKPms5aDZNHe2zVVdVeSlwJXJ/lZRi+3sZDs6d4x+UGSr1fV9wGq6odJFvyfTWA18Cbg7cBbqurWJD9c6AE/zZOSPIPB9z1TVbsAquqBJHvGW9oTL+j/CLguydd4dLG1FcCxwDljq2ruHAWcDNw31B7gv+a/nDn37SQnVNWtAN3M/pXAZcBYZ0xz4OEkT62qHwDP29uY5DAamIRU1Y+Bi5L8U/f7/9JW/hwG3Mzg71oleWZVfTvJ03kcTEKeULduAJI8icE3SZYx+B8wBWxt4Y3dJJcCH6qq/xxx7GNV9aoxlDVnkixnMPP99ohjL6qqG8dQ1pxIsqSqHhrRfiTwrKq6fQxlHTRJTgVeVFV/Nu5aDqYkTwWO6pZpH18dT7Sgl6QnGp+jl6TGGfSS1DiDXpIaZ9BrQUvy9iTbk9yW5NYkL9hP3w8nOf0g1vLhJN/o6vhK9xLU3mMf7N52leZdS4836QkmyQuBVwLPraqHuidUDpnDz1/c/QzkA/GWqro6yaHAl7vX/L9RVX8wV3VJB8oZvRayZwH37H0ssaruqaqdSc5NsjXJl5JckuQnnmPeV58kNyT5qySfBt7ezdCf3B37qSR37d2fwaHd7w9M+9zV3fbuJO9O8sUkn0tyVNd+RlfPF5N8Zrb/caS9DHotZJ8Ejk7y1SQfSPKSrv39VfX8qvpl4CkMZv3D9tfn8Kp6SVW9k8F6M6d27euBfx5eQmPIe5LcyuD9jE1V9Z0Rffa+Fv8rwGeA13Xt5wInd+0Leu0XPb4Y9FqwuuUQngdsAHYBVyZ5DfCyJDd168e8HDhuxPD99bly2vYHgdd2268FZloY7i1VdQLwTODEJL8+os/DwL932zcDE932jcCHu0WxFs1wHqk379FrQeveaL4BuKEL7dcDxwOrq+ruJOfx6G0UALr75x/YT58Hpn3+jUkmun8tLKqqL/Wsa3eSG4AX85PLTzxSj76p+CO6v4dV9Ybum8mnArd2yz18t8/5pP1xRq8FK8kvJpm+4ugJwI5u+55unZFRT9kc2qPPdB9hsNJi72WekyxmsNTw1w9gzM9X1U3dz2O+Bzi671hpf5zRayF7OvC+JIcDe4BJBrdxvgfcDtwFbB0eVFXfS/L3++sz5B+AdzEI+5m8J8mfM3j65zrgX/pcyLSxKxmswXQd8MUDGCvtk2vdSDPonr1fV1W/P+5apMfCGb20H0nex+AnP71i3LVIj5UzeukAJbkYeNFQ83sX+o9qVLsMeklqnE/dSFLjDHpJapxBL0mNM+glqXEGvSQ17v8Apasuy0yNo7QAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#'Word' director mostly appear in higher bins\n",
    "table['director'].plot.bar()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x110615a90>"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAEECAYAAAAmiP8hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAVXklEQVR4nO3df5BdZ33f8fcHKTK/BkONhhBJjpRaaSsX6sJalPIrwa2R49TqTOVWplNshkbJJJqm04ZGlESAAx1c2rqZYGZQsLExpbLjhlYtagS1BzKl4Eg2xkY2IotwrEWlrGMDI4gxsr/94x4Nl8uV9qx2V+t9/H7NaHTOc57nnO+xpc89evacc1NVSJLa9YzFLkCStLAMeklqnEEvSY0z6CWpcQa9JDXOoJekxi1f7AJGvfCFL6y1a9cudhmStKTcddddD1fVynHbnnJBv3btWg4cOLDYZUjSkpLkz062zakbSWqcQS9JjTPoJalxBr0kNc6gl6TG9Qr6JJuSHEoymWTHmO2vTXJ3kuNJtoxsOzfJJ5M8kOT+JGvnp3RJUh8zBn2SZcB1wCXABuCKJBtGuj0EXAV8bMwuPgK8r6r+GrAR+OZcCpYkzU6f++g3ApNVdRggyW5gM3D/iQ5V9WC37cnhgd0HwvKq+lTX79j8lC1J6qtP0K8CjgytTwGv6Ln/nwW+leQPgXXA/wJ2VNUTs6pSkhbJ2h2fOKPHe/C9l877PvvM0WdMW9+vpVoOvAb4DeBC4GcYTPH86AGSbUkOJDkwPT3dc9eSpD76BP0UsGZofTVwtOf+p4AvVNXhqjoO/FfgZaOdqmpXVU1U1cTKlWNf1SBJOk19gn4/sD7JuiQrgK3Anp773w+8IMmJ9H49Q3P7kqSFN2PQd1fi24F9wAPArVV1MMnVSS4DSHJhkingcuCDSQ52Y59gMG1ze5L7GEwD/f7CnIokaZxeb6+sqr3A3pG2nUPL+xlM6Ywb+yngpXOoUZI0Bz4ZK0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY3rFfRJNiU5lGQyyY4x21+b5O4kx5NsGbP9eUm+nuT981G0JKm/GYM+yTLgOuASYANwRZINI90eAq4CPnaS3fwO8JnTL1OSdLr6XNFvBCar6nBVPQ7sBjYPd6iqB6vqXuDJ0cFJXg68CPjkPNQrSZqlPkG/CjgytD7Vtc0oyTOAfw+8dfalSZLmQ5+gz5i26rn/XwX2VtWRU3VKsi3JgSQHpqene+5aktTH8h59poA1Q+urgaM99/9K4DVJfhV4LrAiybGq+pEf6FbVLmAXwMTERN8PEUlSD32Cfj+wPsk64OvAVuCNfXZeVf/4xHKSq4CJ0ZCXJC2sGaduquo4sB3YBzwA3FpVB5NcneQygCQXJpkCLgc+mOTgQhYtSeqvzxU9VbUX2DvStnNoeT+DKZ1T7eNG4MZZVyhJmhOfjJWkxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TG9Qr6JJuSHEoymeTHvvM1yWuT3J3keJItQ+0XJPlckoNJ7k3yj+azeEnSzGYM+iTLgOuAS4ANwBVJNox0ewi4CvjYSPv3gDdV1fnAJuA/Jnn+XIuWJPXX5ztjNwKTVXUYIMluYDNw/4kOVfVgt+3J4YFV9ZWh5aNJvgmsBL4158olSb30mbpZBRwZWp/q2mYlyUZgBfDV2Y6VJJ2+PkGfMW01m4MkeTFwM/DmqnpyzPZtSQ4kOTA9PT2bXUuSZtAn6KeANUPrq4GjfQ+Q5HnAJ4DfqqrPj+tTVbuqaqKqJlauXNl315KkHvoE/X5gfZJ1SVYAW4E9fXbe9f848JGq+oPTL1OSdLpmDPqqOg5sB/YBDwC3VtXBJFcnuQwgyYVJpoDLgQ8mOdgN/4fAa4GrktzT/bpgQc5EkjRWn7tuqKq9wN6Rtp1Dy/sZTOmMjvso8NE51ihJmgOfjJWkxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TG9Qr6JJuSHEoymWTHmO2vTXJ3kuNJtoxsuzLJn3a/rpyvwiVJ/cwY9EmWAdcBlwAbgCuSbBjp9hBwFfCxkbF/CXgH8ApgI/COJC+Ye9mSpL76XNFvBCar6nBVPQ7sBjYPd6iqB6vqXuDJkbFvAD5VVY9U1aPAp4BN81C3JKmnPkG/CjgytD7VtfUxl7GSpHnQJ+gzpq167r/X2CTbkhxIcmB6errnriVJffQJ+ilgzdD6auBoz/33GltVu6pqoqomVq5c2XPXkqQ++gT9fmB9knVJVgBbgT09978PuDjJC7ofwl7ctUmSzpAZg76qjgPbGQT0A8CtVXUwydVJLgNIcmGSKeBy4INJDnZjHwF+h8GHxX7g6q5NknSGLO/Tqar2AntH2nYOLe9nMC0zbuwNwA1zqFGSNAc+GStJjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJalyvoE+yKcmhJJNJdozZflaSW7rtdyZZ27X/RJKbktyX5IEkb5vf8iVJM5kx6JMsA64DLgE2AFck2TDS7S3Ao1V1HnAtcE3XfjlwVlW9BHg58MsnPgQkSWdGnyv6jcBkVR2uqseB3cDmkT6bgZu65duAi5IEKOA5SZYDzwIeB74zL5VLknrpE/SrgCND61Nd29g+3ZeJfxs4h0Hofxf4v8BDwL/zy8El6czqE/QZ01Y9+2wEngB+ClgH/MskP/NjB0i2JTmQ5MD09HSPkiRJffUJ+ilgzdD6auDoyfp00zRnA48AbwT+qKp+UFXfBD4LTIweoKp2VdVEVU2sXLly9mchSTqpPkG/H1ifZF2SFcBWYM9Inz3Ald3yFuCOqioG0zWvz8BzgL8FfHl+Spck9TFj0Hdz7tuBfcADwK1VdTDJ1Uku67pdD5yTZBL4F8CJWzCvA54LfInBB8aHq+reeT4HSdIpLO/Tqar2AntH2nYOLT/G4FbK0XHHxrVLks4cn4yVpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxvUK+iSbkhxKMplkx5jtZyW5pdt+Z5K1Q9temuRzSQ4muS/JM+evfEnSTGYM+iTLGHz36yXABuCKJBtGur0FeLSqzgOuBa7pxi4HPgr8SlWdD/wc8IN5q16SNKM+V/QbgcmqOlxVjwO7gc0jfTYDN3XLtwEXJQlwMXBvVX0RoKr+vKqemJ/SJUl99An6VcCRofWprm1sn6o6DnwbOAf4WaCS7Etyd5J/NfeSJUmzsbxHn4xpq559lgOvBi4EvgfcnuSuqrr9RwYn24BtAOeee26PkiRJffW5op8C1gytrwaOnqxPNy9/NvBI1/6Zqnq4qr4H7AVeNnqAqtpVVRNVNbFy5crZn4Uk6aT6BP1+YH2SdUlWAFuBPSN99gBXdstbgDuqqoB9wEuTPLv7AHgdcP/8lC5J6mPGqZuqOp5kO4PQXgbcUFUHk1wNHKiqPcD1wM1JJhlcyW/txj6a5D8w+LAoYG9VfWKBzkWSNEafOXqqai+DaZfhtp1Dy48Bl59k7EcZ3GIpSVoEPhkrSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMYZ9JLUOINekhpn0EtS4wx6SWqcQS9JjesV9Ek2JTmUZDLJjjHbz0pyS7f9ziRrR7afm+RYkt+Yn7IlSX3NGPRJlgHXAZcAG4ArkmwY6fYW4NGqOg+4FrhmZPu1wP+ce7mSpNnqc0W/EZisqsNV9TiwG9g80mczcFO3fBtwUZIAJPn7wGHg4PyULEmajT5Bvwo4MrQ+1bWN7VNVx4FvA+ckeQ7wm8C75l6qJOl09An6jGmrnn3eBVxbVcdOeYBkW5IDSQ5MT0/3KEmS1NfyHn2mgDVD66uBoyfpM5VkOXA28AjwCmBLkn8LPB94MsljVfX+4cFVtQvYBTAxMTH6ISJJmoM+Qb8fWJ9kHfB1YCvwxpE+e4Argc8BW4A7qqqA15zokOSdwLHRkJckLawZg76qjifZDuwDlgE3VNXBJFcDB6pqD3A9cHOSSQZX8lsXsmhJUn99ruipqr3A3pG2nUPLjwGXz7CPd55GfZKkOfLJWElqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktS4XrdXStKprN3xiTN6vAffe+kZPd5S5xW9JDXOoJekxhn0ktQ4g16SGmfQS1LjDHpJapxBL0mNM+glqXEGvSQ1zqCXpMb1Cvokm5IcSjKZZMeY7WcluaXbfmeStV37301yV5L7ut9fP7/lS5JmMmPQJ1kGXAdcAmwArkiyYaTbW4BHq+o84Frgmq79YeDvVdVLGHx5+M3zVbgkqZ8+V/QbgcmqOlxVjwO7gc0jfTYDN3XLtwEXJUlVfaGqjnbtB4FnJjlrPgqXJPXTJ+hXAUeG1qe6trF9quo48G3gnJE+/wD4QlV9//RKlSSdjj6vKc6YtppNnyTnM5jOuXjsAZJtwDaAc889t0dJkqS++lzRTwFrhtZXA0dP1ifJcuBs4JFufTXwceBNVfXVcQeoql1VNVFVEytXrpzdGUiSTqlP0O8H1idZl2QFsBXYM9JnD4MftgJsAe6oqkryfOATwNuq6rPzVbQkqb8Zp26q6niS7cA+YBlwQ1UdTHI1cKCq9gDXAzcnmWRwJb+1G74dOA/47SS/3bVdXFXfnO8TkZ7K/AYmLaZeXyVYVXuBvSNtO4eWHwMuHzPu3cC751ijJGkOfDJWkhpn0EtS4wx6SWqcQS9JjTPoJalxBr0kNc6gl6TGGfSS1DiDXpIaZ9BLUuMMeklqnEEvSY0z6CWpcQa9JDXOoJekxhn0ktQ4g16SGtcr6JNsSnIoyWSSHWO2n5Xklm77nUnWDm17W9d+KMkb5q90SVIfMwZ9kmXAdcAlwAbgiiQbRrq9BXi0qs4DrgWu6cZuYPD9secDm4APdPuTJJ0hfa7oNwKTVXW4qh4HdgObR/psBm7qlm8DLkqSrn13VX2/qr4GTHb7kySdIX2CfhVwZGh9qmsb26eqjgPfBs7pOVaStICW9+iTMW3Vs0+fsSTZBmzrVo8lOdSjrvnyQuDhM3i8M83zW9pO6/xyzQJUsjBaPr8zfW4/fbINfYJ+ClgztL4aOHqSPlNJlgNnA4/0HEtV7QJ29ahl3iU5UFUTi3HsM8HzW9o8v6XrqXRufaZu9gPrk6xLsoLBD1f3jPTZA1zZLW8B7qiq6tq3dnflrAPWA38yP6VLkvqY8Yq+qo4n2Q7sA5YBN1TVwSRXAweqag9wPXBzkkkGV/Jbu7EHk9wK3A8cB36tqp5YoHORJI3RZ+qGqtoL7B1p2zm0/Bhw+UnGvgd4zxxqXGiLMmV0Bnl+S5vnt3Q9Zc4tgxkWSVKrfAWCJDXOoJekxvWao29Jkr/K4IndVQzu6T8K7KmqBxa1MPXS/f9bBdxZVceG2jdV1R8tXmVzl2QjUFW1v3t9yCbgy93PyJqT5CNV9abFrmMhJHk1g7cAfKmqPrno9Tyd5uiT/CZwBYPXOEx1zasZ3CW0u6reu1i1LbQkb66qDy92HXOR5J8BvwY8AFwA/HpV/bdu291V9bLFrG8ukryDwfuklgOfAl4BfBr4O8C+7qaGJSvJ6C3ZAX4euAOgqi4740XNoyR/UlUbu+VfYvDn9OPAxcB/X+xseboF/VeA86vqByPtK4CDVbV+cSpbeEkeqqpzF7uOuUhyH/DKqjrWvSH1NuDmqvrdJF+oqr+5qAXOQXduFwBnAd8AVlfVd5I8i8G/Xl66qAXOUZK7Gdxm/SF++NT8f+aHt2J/ZvGqm7vhP39J9gO/UFXTSZ4DfL6qXrKY9T3dpm6eBH4K+LOR9hd325a0JPeebBPwojNZywJZdmK6pqoeTPJzwG1Jfprxr9tYSo53z5h8L8lXq+o7AFX1F0mW/J9NYAL4deDtwFur6p4kf7HUA37IM5K8gMHPPVNV0wBV9d0kxxe3tKdf0P9z4PYkf8oPX7Z2LnAesH3Rqpo/LwLeADw60h7g/5z5cubdN5JcUFX3AHRX9r8I3AAs6hXTPHg8ybOr6nvAy080JjmbBi5CqupJ4Nokf9D9/v9oK3/OBu5i8HetkvxkVX0jyXN5ClyEPK2mbgCSPIPBD0lWMfgfMAXsb+GJ3STXAx+uqv89ZtvHquqNi1DWvEmymsGV7zfGbHtVVX12EcqaF0nOqqrvj2l/IfDiqrpvEcpaMEkuBV5VVf96sWtZSEmeDbyoe0374tXxdAt6SXq68T56SWqcQS9JjTPoJalxBr2WvCRvT3Iwyb1J7knyilP0vTHJlgWs5cYkX+vq+HL3INSJbR/qnniVzqiWbm/S01CSVwK/CLysqr7f3aWyYh73v7z7HuTZeGtV3ZbkmcD93aP+X6uqfzpfdUmz4RW9lroXAw+fuDWxqh6uqqNJdibZn+RLSXYl+bF7mU/WJ8mnk/ybJJ8B3t5dof9Et+15SR48sT6DZ3a/f3dovxPd8rEk70nyxSSfT/Kirv3yrp4vJvnjuf7HkcCg19L3SWBNkq8k+UCS13Xt76+qC6vqrwPPYnDVP+pUfZ5fVa+rqncxeOfMpV37VuC/jL5GY8T7ktzD4BmN3VX1zTF9Tjwa/zeAPwZ+qWvfCbyha1/S73/RU4dBryWteyXCy4FtwDRwS5KrgJ9Pcmf3DpnXA+ePGX6qPrcMLX8IeHO3/GZgppfDvbWqLgB+Ergoyd8e0+dx4H90y3cBa7vlzwI3di/GWjbDcaRenKPXktc91fxp4NNdaP8y8FJgoqqOJHknP5xGAaCbP//AKfp8d2j/n02ytvvXwrKq+lLPuo4l+TTwan78FRQ/qB8+rfgE3d/FqvqV7ofJlwL3dK98+PM+x5NOxit6LWlJ/kqS4beOXgAc6pYf7t41Mu4um2f26DPsIwzettj7Vc9JljN43fBXZzHmL1fVnd13Mj8MrOk7VjoZr+i11D0X+L0kzweOA5MMpnG+BdwHPAjsHx1UVd9K8vun6jPiPwHvZhD2M3lfkt9icPfP7cAf9jmRobHrGbyH6Xbgi7MYK43lu26kHrp77zdX1T9Z7Fqk2fKKXppBkt9j8O1Pv7DYtUinwyt66TQkuQ541Ujz7y71r2tUmwx6SWqcd91IUuMMeklqnEEvSY0z6CWpcQa9JDXu/wMqiWkKo8Z/OgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Word 'soft engineer' mostly appear in lower bins\n",
    "table['software engineer'].plot.bar()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#How to use the frequency as features in the model?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 Bin most common words:\n",
      "marketing           28.0\n",
      "analyst             22.0\n",
      "data                19.0\n",
      "manager             15.0\n",
      "specialist           9.0\n",
      "digital              9.0\n",
      "science              6.0\n",
      "business             5.0\n",
      "engineer             4.0\n",
      "sales                4.0\n",
      "data analyst         4.0\n",
      "business analyst     3.0\n",
      "senior               3.0\n",
      "scientist            3.0\n",
      "data scientist       3.0\n",
      "machine              2.0\n",
      "director             1.0\n",
      "learning             1.0\n",
      "systems              1.0\n",
      "machine learning     1.0\n",
      "dtype: float64\n",
      "1 Bin most common words:\n",
      "Salary_Bins         185.0\n",
      "analyst              74.0\n",
      "business             35.0\n",
      "data                 32.0\n",
      "business analyst     30.0\n",
      "manager              20.0\n",
      "marketing            19.0\n",
      "specialist           13.0\n",
      "scientist            12.0\n",
      "data analyst         11.0\n",
      "data scientist       11.0\n",
      "sales                 8.0\n",
      "engineer              8.0\n",
      "senior                7.0\n",
      "director              5.0\n",
      "digital               5.0\n",
      "science               4.0\n",
      "data engineer         2.0\n",
      "analytics             2.0\n",
      "learning              1.0\n",
      "dtype: float64\n",
      "2 Bin most common words:\n",
      "Salary_Bins         300.0\n",
      "analyst              75.0\n",
      "data                 44.0\n",
      "business             31.0\n",
      "engineer             22.0\n",
      "business analyst     21.0\n",
      "data analyst         21.0\n",
      "manager              18.0\n",
      "marketing            14.0\n",
      "digital              13.0\n",
      "senior               13.0\n",
      "sales                10.0\n",
      "scientist             8.0\n",
      "data scientist        8.0\n",
      "systems               7.0\n",
      "learning              6.0\n",
      "machine               6.0\n",
      "machine learning      6.0\n",
      "data engineer         5.0\n",
      "analytics             5.0\n",
      "dtype: float64\n",
      "3 Bin most common words:\n",
      "Salary_Bins         516.0\n",
      "data                 67.0\n",
      "analyst              53.0\n",
      "engineer             49.0\n",
      "business             31.0\n",
      "senior               26.0\n",
      "data engineer        22.0\n",
      "business analyst     17.0\n",
      "systems              15.0\n",
      "data scientist       12.0\n",
      "scientist            12.0\n",
      "manager              12.0\n",
      "data analyst         12.0\n",
      "analytics             8.0\n",
      "marketing             7.0\n",
      "director              7.0\n",
      "digital               7.0\n",
      "learning              6.0\n",
      "machine               6.0\n",
      "software              6.0\n",
      "dtype: float64\n",
      "4 Bin most common words:\n",
      "Salary_Bins         644.0\n",
      "data                 73.0\n",
      "engineer             50.0\n",
      "analyst              42.0\n",
      "business             31.0\n",
      "senior               29.0\n",
      "data engineer        29.0\n",
      "business analyst     20.0\n",
      "scientist            18.0\n",
      "data scientist       17.0\n",
      "software             10.0\n",
      "analytics            10.0\n",
      "manager              10.0\n",
      "science               8.0\n",
      "marketing             8.0\n",
      "senior data           7.0\n",
      "director              7.0\n",
      "machine               6.0\n",
      "sales                 6.0\n",
      "learning              5.0\n",
      "dtype: float64\n",
      "5 Bin most common words:\n",
      "Salary_Bins          685.0\n",
      "engineer              68.0\n",
      "data                  57.0\n",
      "senior                38.0\n",
      "software              24.0\n",
      "software engineer     22.0\n",
      "data engineer         22.0\n",
      "scientist             17.0\n",
      "data scientist        16.0\n",
      "machine learning      15.0\n",
      "learning              15.0\n",
      "machine               15.0\n",
      "analyst               15.0\n",
      "director              14.0\n",
      "senior data           14.0\n",
      "business              11.0\n",
      "science                8.0\n",
      "sales                  6.0\n",
      "business analyst       6.0\n",
      "analytics              6.0\n",
      "dtype: float64\n"
     ]
    }
   ],
   "source": [
    "#Looks \n",
    "for i in range(0,6):\n",
    "    word_count = word_observe[word_observe['Salary_Bins']==i].sum(axis=0)\n",
    "    print(i, \"Bin most common words:\")\n",
    "    cw = word_count.sort_values(ascending=False).head(20)\n",
    "    print(cw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random Forest as base model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "from pprint import pprint\n",
    "from time import time\n",
    "import logging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=None, error_score=nan,\n",
       "             estimator=RandomForestClassifier(bootstrap=True, ccp_alpha=0.0,\n",
       "                                              class_weight=None,\n",
       "                                              criterion='gini', max_depth=None,\n",
       "                                              max_features='auto',\n",
       "                                              max_leaf_nodes=None,\n",
       "                                              max_samples=None,\n",
       "                                              min_impurity_decrease=0.0,\n",
       "                                              min_impurity_split=None,\n",
       "                                              min_samples_leaf=1,\n",
       "                                              min_samples_split=2,\n",
       "                                              min_weight_fraction_leaf=0.0,\n",
       "                                              n_estimators=100, n_jobs=None,\n",
       "                                              oob_score=False,\n",
       "                                              random_state=None, verbose=0,\n",
       "                                              warm_start=False),\n",
       "             iid='deprecated', n_jobs=None,\n",
       "             param_grid={'criterion': ('gini', 'entropy'),\n",
       "                         'max_features': ('auto', 'sqrt', 'log2'),\n",
       "                         'max_samples': (None, 0.5, 0.8, 1),\n",
       "                         'min_samples_split': (2, 6, 8)},\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
       "             scoring=None, verbose=0)"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Inititate RF classifier\n",
    "rf = RandomForestClassifier()\n",
    "parameters = {\n",
    "    'criterion':(\"gini\",\"entropy\"),\n",
    "    'min_samples_split':(2,6,8),\n",
    "    'max_features':(\"auto\",\"sqrt\",\"log2\"),\n",
    "    'max_samples':(None, 0.5,0.8,1) \n",
    "    \n",
    "}\n",
    "\n",
    "clf = GridSearchCV(rf, parameters)\n",
    "clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'bootstrap': True,\n",
       " 'ccp_alpha': 0.0,\n",
       " 'class_weight': None,\n",
       " 'criterion': 'gini',\n",
       " 'max_depth': None,\n",
       " 'max_features': 'auto',\n",
       " 'max_leaf_nodes': None,\n",
       " 'max_samples': 0.8,\n",
       " 'min_impurity_decrease': 0.0,\n",
       " 'min_impurity_split': None,\n",
       " 'min_samples_leaf': 1,\n",
       " 'min_samples_split': 2,\n",
       " 'min_weight_fraction_leaf': 0.0,\n",
       " 'n_estimators': 100,\n",
       " 'n_jobs': None,\n",
       " 'oob_score': False,\n",
       " 'random_state': None,\n",
       " 'verbose': 0,\n",
       " 'warm_start': False}"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.best_estimator_.get_params()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3829787234042553"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.score(X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3928994082840237"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Check feature importance of the best performing model\n",
    "\n",
    "rf = RandomForestClassifier(criterion='gini', max_samples=0.8)\n",
    "rf.fit(X_train, y_train)\n",
    "\n",
    "importances = rf.feature_importances_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0.04536015, 0.02325264, 0.02836942, 0.01211934, 0.05216111,\n",
       "       0.01320237, 0.01966278, 0.01038868, 0.02008677, 0.02811157,\n",
       "       0.04238933, 0.00569282, 0.00647361, 0.00639306, 0.03426544,\n",
       "       0.03645556, 0.03319433, 0.0156367 , 0.01195255, 0.04467765,\n",
       "       0.01079407, 0.0193588 , 0.01437405, 0.01851068, 0.02181357,\n",
       "       0.07510111, 0.35020183])"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "importances"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>analyst</th>\n",
       "      <th>analytics</th>\n",
       "      <th>business</th>\n",
       "      <th>business analyst</th>\n",
       "      <th>data</th>\n",
       "      <th>data analyst</th>\n",
       "      <th>data engineer</th>\n",
       "      <th>data scientist</th>\n",
       "      <th>digital</th>\n",
       "      <th>director</th>\n",
       "      <th>...</th>\n",
       "      <th>science</th>\n",
       "      <th>scientist</th>\n",
       "      <th>senior</th>\n",
       "      <th>senior data</th>\n",
       "      <th>software</th>\n",
       "      <th>software engineer</th>\n",
       "      <th>specialist</th>\n",
       "      <th>systems</th>\n",
       "      <th>City_cat</th>\n",
       "      <th>Cost of Living Plus Rent Index</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>495</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>74.800000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>379</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>59.399889</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>60.940000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>672</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>64.660000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>337</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>63.110000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     analyst  analytics  business  business analyst  data  data analyst  \\\n",
       "495        0          0         0                 0     0             0   \n",
       "379        1          0         0                 0     0             0   \n",
       "67         0          0         0                 0     0             0   \n",
       "672        0          0         0                 0     0             0   \n",
       "337        1          0         0                 0     0             0   \n",
       "\n",
       "     data engineer  data scientist  digital  director  ...  science  \\\n",
       "495              0               0        0         0  ...        0   \n",
       "379              0               0        0         0  ...        0   \n",
       "67               0               0        0         0  ...        0   \n",
       "672              0               0        0         0  ...        0   \n",
       "337              0               0        0         0  ...        0   \n",
       "\n",
       "     scientist  senior  senior data  software  software engineer  specialist  \\\n",
       "495          0       0            0         1                  1           0   \n",
       "379          0       0            0         0                  0           0   \n",
       "67           0       1            0         0                  0           0   \n",
       "672          0       1            0         0                  0           0   \n",
       "337          0       0            0         0                  0           0   \n",
       "\n",
       "     systems  City_cat  Cost of Living Plus Rent Index  \n",
       "495        0         1                       74.800000  \n",
       "379        0         1                       59.399889  \n",
       "67         0         1                       60.940000  \n",
       "672        0         2                       64.660000  \n",
       "337        0         1                       63.110000  \n",
       "\n",
       "[5 rows x 27 columns]"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameter (CV score = 0.388):\n",
      "{'pca__n_components': 25, 'rf__criterion': 'entropy', 'rf__min_samples_split': 2}\n"
     ]
    }
   ],
   "source": [
    "#PCA in the pipeline\n",
    "from sklearn.decomposition import PCA\n",
    "\n",
    "pca = PCA()\n",
    "rf = RandomForestClassifier(criterion='gini', max_samples=0.8)\n",
    "\n",
    "pipe = Pipeline(steps=[('pca', pca), ('rf', rf)])\n",
    "\n",
    "param  = {\n",
    "    'pca__n_components':[3,5,10,15,20,25],\n",
    "    'rf__criterion':(\"gini\",\"entropy\"),\n",
    "    'rf__min_samples_split':(2,6,8),\n",
    "}\n",
    "\n",
    "search = GridSearchCV(pipe, param, n_jobs=-1)\n",
    "search.fit(X_train, y_train)\n",
    "\n",
    "print(\"Best parameter (CV score = %0.3f):\" % search.best_score_)\n",
    "print(search.best_params_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### XGBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import GradientBoostingClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameter (CV score = 0.387):\n",
      "{'pca__n_components': 15, 'xgb__learning_rate': 0.1, 'xgb__loss': 'deviance', 'xgb__subsample': 1.0}\n"
     ]
    }
   ],
   "source": [
    "pca = PCA()\n",
    "XGB = GradientBoostingClassifier(random_state=4)\n",
    "\n",
    "pipe = Pipeline(steps=[('pca', pca), ('xgb', XGB)])\n",
    "\n",
    "param  = {\n",
    "    'pca__n_components':[3,5,10,15,20,25],\n",
    "    'xgb__loss':(\"deviance\",\"exponential\"),\n",
    "    'xgb__learning_rate':(0.1, 0.01, 0.001),\n",
    "    'xgb__subsample':(0.3,0.5,1.0),\n",
    "}\n",
    "\n",
    "search = GridSearchCV(pipe, param, n_jobs=-1)\n",
    "search.fit(X_train, y_train)\n",
    "\n",
    "print(\"Best parameter (CV score = %0.3f):\" % search.best_score_)\n",
    "print(search.best_params_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Conclusion"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Among 7 models we built, Model 2 performs the best with a test accuracy of 0.71. We will use this model for deployment. \n",
    "\n",
    "- Model 1: Word2Vec Embedding + Classification Models (Random Forest/SGD) \n",
    "- Model 2: Count Vectorizer + TF-IDF Transformation + Classification Models (Random Forest/SGD/SVM)\n",
    "- Model 3: Pre-trained GloVe Embedding + Classification Models (Random Forest/SGD)\n",
    "- Model 4: Word Embeddings + Simple Neural Network Model\n",
    "- Model 5: Pre-trained GloVe Embedding + Deep Neural Network Model\n",
    "- Model 6: Mixed Input Network Model with City, Rate Type, and Cost of Living Indexes\n",
    "- Model 7: Just use Job Title, City, Cost of Living Indexes + Classification\n",
    "\n",
    "For next steps:\n",
    "1. Job salary range can change from time to time depends on economic situation and job market. Therefore the model needs to be constantly trained with refreshed data to produce up to date results. Most of the positions were pulled in late May 2020 when COVID19 impacted North America so the salary range during this period of time may not reflect the \"normal\". \n",
    "\n",
    "2. We scrapped the full job description from Indeed as there is no standard way to parse out company detail, skills requirements and benefits so there are noises within the job description text to extract features. \n",
    "\n",
    "3. The size of the salary labeled data is small. In the future, scrape more data with salary to enhance the model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "name": "Salary_Modeling_5.31.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": true,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "265.969px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": "20",
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "position": {
    "height": "208px",
    "left": "797px",
    "right": "20px",
    "top": "-57px",
    "width": "282px"
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}