streetcar_DL_train.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Streetcar Delay Prediction - Deep Learning\n",
    "\n",
    "GOAL: predict streetcar delays using a simple Keras model\n",
    "\n",
    "Source dataset: : https://www.toronto.ca/city-government/data-research-maps/open-data/open-data-catalogue/#e8f359f0-2f47-3058-bf64-6ec488de52da"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Links to key parts of the notebook <a name='linkanchor' />\n",
    "<a href=#ingestdash>Ingest data</a>\n",
    "\n",
    "<a href=#definecategories>Define feature categories</a>\n",
    "\n",
    "<a href=#bookmark>Deal with missing values</a>\n",
    "\n",
    "<a href=#modelfit>Define and fit model</a>\n",
    "\n",
    "<a href=#reload>Reload saved model and weights</a>\n",
    "\n",
    "<a href=#confusionmatrix>Confusion matrix</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: sklearn in /opt/conda/envs/fastai/lib/python3.6/site-packages (0.0)\n",
      "Requirement already satisfied: scikit-learn in /opt/conda/envs/fastai/lib/python3.6/site-packages (from sklearn) (0.20.3)\n",
      "Requirement already satisfied: scipy>=0.13.3 in /opt/conda/envs/fastai/lib/python3.6/site-packages (from scikit-learn->sklearn) (1.2.0)\n",
      "Requirement already satisfied: numpy>=1.8.2 in /opt/conda/envs/fastai/lib/python3.6/site-packages (from scikit-learn->sklearn) (1.15.4)\n",
      "Requirement already satisfied: seaborn in /opt/conda/envs/fastai/lib/python3.6/site-packages (0.9.0)\n",
      "Requirement already satisfied: numpy>=1.9.3 in /opt/conda/envs/fastai/lib/python3.6/site-packages (from seaborn) (1.15.4)\n",
      "Requirement already satisfied: scipy>=0.14.0 in /opt/conda/envs/fastai/lib/python3.6/site-packages (from seaborn) (1.2.0)\n",
      "Requirement already satisfied: pandas>=0.15.2 in /opt/conda/envs/fastai/lib/python3.6/site-packages (from seaborn) (0.23.4)\n",
      "Requirement already satisfied: matplotlib>=1.4.3 in /opt/conda/envs/fastai/lib/python3.6/site-packages (from seaborn) (3.0.2)\n",
      "Requirement already satisfied: python-dateutil>=2.5.0 in /opt/conda/envs/fastai/lib/python3.6/site-packages (from pandas>=0.15.2->seaborn) (2.7.5)\n",
      "Requirement already satisfied: pytz>=2011k in /opt/conda/envs/fastai/lib/python3.6/site-packages (from pandas>=0.15.2->seaborn) (2018.9)\n",
      "Requirement already satisfied: cycler>=0.10 in /opt/conda/envs/fastai/lib/python3.6/site-packages (from matplotlib>=1.4.3->seaborn) (0.10.0)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /opt/conda/envs/fastai/lib/python3.6/site-packages (from matplotlib>=1.4.3->seaborn) (1.0.1)\n",
      "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /opt/conda/envs/fastai/lib/python3.6/site-packages (from matplotlib>=1.4.3->seaborn) (2.3.1)\n",
      "Requirement already satisfied: six>=1.5 in /opt/conda/envs/fastai/lib/python3.6/site-packages (from python-dateutil>=2.5.0->pandas>=0.15.2->seaborn) (1.12.0)\n",
      "Requirement already satisfied: setuptools in /opt/conda/envs/fastai/lib/python3.6/site-packages (from kiwisolver>=1.0.1->matplotlib>=1.4.3->seaborn) (40.6.3)\n",
      "Requirement already satisfied: keras in /opt/conda/envs/fastai/lib/python3.6/site-packages (2.2.4)\n",
      "Requirement already satisfied: six>=1.9.0 in /opt/conda/envs/fastai/lib/python3.6/site-packages (from keras) (1.12.0)\n",
      "Requirement already satisfied: h5py in /opt/conda/envs/fastai/lib/python3.6/site-packages (from keras) (2.9.0)\n",
      "Requirement already satisfied: numpy>=1.9.1 in /opt/conda/envs/fastai/lib/python3.6/site-packages (from keras) (1.15.4)\n",
      "Requirement already satisfied: scipy>=0.14 in /opt/conda/envs/fastai/lib/python3.6/site-packages (from keras) (1.2.0)\n",
      "Requirement already satisfied: pyyaml in /opt/conda/envs/fastai/lib/python3.6/site-packages (from keras) (3.13)\n",
      "Requirement already satisfied: keras-applications>=1.0.6 in /opt/conda/envs/fastai/lib/python3.6/site-packages (from keras) (1.0.7)\n",
      "Requirement already satisfied: keras-preprocessing>=1.0.5 in /opt/conda/envs/fastai/lib/python3.6/site-packages (from keras) (1.0.9)\n",
      "Requirement already satisfied: tensorflow in /opt/conda/envs/fastai/lib/python3.6/site-packages (1.13.1)\n",
      "Requirement already satisfied: termcolor>=1.1.0 in /opt/conda/envs/fastai/lib/python3.6/site-packages (from tensorflow) (1.1.0)\n",
      "Requirement already satisfied: six>=1.10.0 in /opt/conda/envs/fastai/lib/python3.6/site-packages (from tensorflow) (1.12.0)\n",
      "Requirement already satisfied: astor>=0.6.0 in /opt/conda/envs/fastai/lib/python3.6/site-packages (from tensorflow) (0.7.1)\n",
      "Requirement already satisfied: numpy>=1.13.3 in /opt/conda/envs/fastai/lib/python3.6/site-packages (from tensorflow) (1.15.4)\n",
      "Requirement already satisfied: keras-preprocessing>=1.0.5 in /opt/conda/envs/fastai/lib/python3.6/site-packages (from tensorflow) (1.0.9)\n",
      "Requirement already satisfied: tensorflow-estimator<1.14.0rc0,>=1.13.0 in /opt/conda/envs/fastai/lib/python3.6/site-packages (from tensorflow) (1.13.0)\n",
      "Requirement already satisfied: wheel>=0.26 in /opt/conda/envs/fastai/lib/python3.6/site-packages (from tensorflow) (0.32.3)\n",
      "Requirement already satisfied: gast>=0.2.0 in /opt/conda/envs/fastai/lib/python3.6/site-packages (from tensorflow) (0.2.2)\n",
      "Requirement already satisfied: absl-py>=0.1.6 in /opt/conda/envs/fastai/lib/python3.6/site-packages (from tensorflow) (0.7.1)\n",
      "Requirement already satisfied: tensorboard<1.14.0,>=1.13.0 in /opt/conda/envs/fastai/lib/python3.6/site-packages (from tensorflow) (1.13.1)\n",
      "Requirement already satisfied: protobuf>=3.6.1 in /opt/conda/envs/fastai/lib/python3.6/site-packages (from tensorflow) (3.7.0)\n",
      "Requirement already satisfied: keras-applications>=1.0.6 in /opt/conda/envs/fastai/lib/python3.6/site-packages (from tensorflow) (1.0.7)\n",
      "Requirement already satisfied: grpcio>=1.8.6 in /opt/conda/envs/fastai/lib/python3.6/site-packages (from tensorflow) (1.19.0)\n",
      "Requirement already satisfied: mock>=2.0.0 in /opt/conda/envs/fastai/lib/python3.6/site-packages (from tensorflow-estimator<1.14.0rc0,>=1.13.0->tensorflow) (2.0.0)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /opt/conda/envs/fastai/lib/python3.6/site-packages (from tensorboard<1.14.0,>=1.13.0->tensorflow) (3.0.1)\n",
      "Requirement already satisfied: werkzeug>=0.11.15 in /opt/conda/envs/fastai/lib/python3.6/site-packages (from tensorboard<1.14.0,>=1.13.0->tensorflow) (0.15.1)\n",
      "Requirement already satisfied: setuptools in /opt/conda/envs/fastai/lib/python3.6/site-packages (from protobuf>=3.6.1->tensorflow) (40.6.3)\n",
      "Requirement already satisfied: h5py in /opt/conda/envs/fastai/lib/python3.6/site-packages (from keras-applications>=1.0.6->tensorflow) (2.9.0)\n",
      "Requirement already satisfied: pbr>=0.11 in /opt/conda/envs/fastai/lib/python3.6/site-packages (from mock>=2.0.0->tensorflow-estimator<1.14.0rc0,>=1.13.0->tensorflow) (5.1.3)\n",
      "Requirement already satisfied: pydot in /opt/conda/envs/fastai/lib/python3.6/site-packages (1.4.1)\n",
      "Requirement already satisfied: pyparsing>=2.1.4 in /opt/conda/envs/fastai/lib/python3.6/site-packages (from pydot) (2.3.1)\n"
     ]
    }
   ],
   "source": [
    "# libraries that had to explicitly installed for Paperspace Gradient\n",
    "!pip install sklearn\n",
    "!pip install seaborn\n",
    "!pip install keras\n",
    "!pip install tensorflow\n",
    "!pip install pydot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "# common imports\n",
    "import zipfile\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import time\n",
    "import datetime\n",
    "import os\n",
    "from dateutil import relativedelta\n",
    "from io import StringIO\n",
    "import pandas as pd\n",
    "import pickle\n",
    "import pydot\n",
    "from sklearn.base import BaseEstimator\n",
    "from sklearn.base import TransformerMixin\n",
    "\n",
    "# from sklearn.preprocessing import CategoricalEncoder\n",
    "\n",
    "# DSX code to import uploaded documents\n",
    "from io import StringIO\n",
    "import requests\n",
    "import json\n",
    "from sklearn.preprocessing import LabelEncoder, MinMaxScaler, StandardScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline \n",
    "\n",
    "import math\n",
    "from subprocess import check_output\n",
    "import seaborn as sns\n",
    "\n",
    "#model libraries\n",
    "\n",
    "\n",
    "from keras.layers import Input, Dropout, Dense, BatchNormalization, Activation, concatenate, GRU, Embedding, Flatten, BatchNormalization\n",
    "from keras.models import Model\n",
    "from keras.callbacks import ModelCheckpoint, Callback, EarlyStopping\n",
    "from keras import regularizers\n",
    "from keras.layers.normalization import BatchNormalization\n",
    "from keras.regularizers import l2\n",
    "from keras.optimizers import Adam\n",
    "from keras.optimizers import SGD\n",
    "from keras import backend as K\n",
    "from keras.utils.vis_utils import plot_model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set overall parameters\n",
    "\n",
    "dsxmode = False # overall is this being run in DSX?\n",
    "csvmode = True # ingest from CSV\n",
    "dbmode = False # ingest from database\n",
    "pSpace = True # pSpace mode\n",
    "savepath = '/home/paperspace/models/'\n",
    "readpath = '/home/paperspace/data/dmprediction/'\n",
    "\n",
    "testproportion = 0.01 # proportion of data reserved for test set\n",
    "trainproportion = 0.8 # proportion of non-test data dedicated to training (vs. validation)\n",
    "verboseout = True\n",
    "includetext = True # switch to determine whether text fields are included in model\n",
    "presaved = False # switch to determine whether to train model or load saved model\n",
    "savemodel = True # switch to determine whether to save model\n",
    "picklemodel = True # switch for code to allow pickling of Keras model\n",
    "hctextmax = 7000\n",
    "maxwords = 6000\n",
    "\n",
    "# threshold between 0 and 1 target for regression target\n",
    "targetthresh = 6.0\n",
    "targetcontinuous = False # switch to determine whether target is continuous or \n",
    "\n",
    "#time of day thresholds\n",
    "time_of_day = {'overnight':{'start':0,'end':5},'morning_rush':{'start':5,'end':10},\n",
    "              'midday':{'start':10,'end':15},'aft_rush':{'start':15,'end':19},'evening':{'start':19,'end':24}}\n",
    "\n",
    "#time_of_day = {'overnight':[0,5],'morning_rush':[5,10],\n",
    " #             'midday':[10,15],'aft_rush':[15,19],'evening':[19,23]}\n",
    "              \n",
    "\n",
    "\n",
    "emptythresh = 6000\n",
    "# to address imbalance in training data between zero (above targetthresh) and detractor (below targetthresh) specify weight in compile and fit\n",
    "# class_weight = {0 : zero_weight, 1: one_weight}\n",
    "# consider calculating these values from actual skew rather than hard-coding them here\n",
    "zero_weight = 1.0\n",
    "one_weight = 72.8\n",
    "\n",
    "# hyperparameters\n",
    "learning_rate = 0.001\n",
    "dropout_rate = 0.0003 #0.003\n",
    "l2_lambda = 0.0003 #7.5\n",
    "loss_func = \"binary_crossentropy\"\n",
    "if targetcontinuous:\n",
    "    output_activation = \"linear\"\n",
    "else:\n",
    "    output_activation = \"hard_sigmoid\"\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "raw path is /notebooks/manning/notebooks\n"
     ]
    }
   ],
   "source": [
    "# get the directory for that this notebook is in\n",
    "rawpath = os.getcwd()\n",
    "print(\"raw path is\",rawpath)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "path is /notebooks/manning/data/\n"
     ]
    }
   ],
   "source": [
    "# data is in a directory called \"data\" that is a sibling to the directory containing the notebook\n",
    "path = os.path.abspath(os.path.join(rawpath, '..', 'data')) + \"/\"\n",
    "print(\"path is\", path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "# time_of_day = {'overnight':{'start':0,'end':5},'morning_rush':{'start':5,'end':10},\n",
    "#              'midday':{'start':10,'end':15},'aft_rush':{'start':15,'end':19},'evening':{'start':19,'end':23}}\n",
    "              \n",
    "\n",
    "def get_time(hour):\n",
    "    for tod in time_of_day:\n",
    "        if (hour >= time_of_day[tod]['start']) and (hour < time_of_day[tod]['end']):\n",
    "            tod_out = tod\n",
    "    return(tod_out)\n",
    "\n",
    "def weekend_time(day, tod):\n",
    "    if (day=='Saturday') or (day=='Sunday'):\n",
    "        return('w'+tod)\n",
    "    else:\n",
    "        return(tod)\n",
    "            \n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tod on 0\n"
     ]
    }
   ],
   "source": [
    "print(\"tod on\",time_of_day['overnight']['start'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tod  0 overnight\n",
      "tod 5 evening\n"
     ]
    }
   ],
   "source": [
    "print(\"tod  0\",get_time(0))\n",
    "print(\"tod 5\",get_time(23))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "# constants for the streetcar problem\n",
    "# same values saved in data_preparation notebook: pickled_input_dataframe, pickled_output_dataframe\n",
    "pickled_data_file = '2014_2018.pkl'\n",
    "pickled_dataframe = '2014_2018_df.pkl'\n",
    "# pickled_dataframe = '2014_2018_df_direction_uncleaned.pkl'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ingest data <a name='ingestdash' />\n",
    "<a href=#linkanchor>Back to link list</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Report Date</th>\n",
       "      <th>Route</th>\n",
       "      <th>Time</th>\n",
       "      <th>Day</th>\n",
       "      <th>Location</th>\n",
       "      <th>Incident</th>\n",
       "      <th>Min Delay</th>\n",
       "      <th>Min Gap</th>\n",
       "      <th>Direction</th>\n",
       "      <th>Vehicle</th>\n",
       "      <th>Report Date Time</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Report Date Time</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2015-01-01 01:25:00</th>\n",
       "      <td>2015-01-01</td>\n",
       "      <td>504</td>\n",
       "      <td>01:25:00</td>\n",
       "      <td>Thursday</td>\n",
       "      <td>broadview and gerrard</td>\n",
       "      <td>Mechanical</td>\n",
       "      <td>9.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>S/B</td>\n",
       "      <td>4092</td>\n",
       "      <td>2015-01-01 01:25:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-01-01 01:44:00</th>\n",
       "      <td>2015-01-01</td>\n",
       "      <td>504</td>\n",
       "      <td>01:44:00</td>\n",
       "      <td>Thursday</td>\n",
       "      <td>galley and roncesvalles</td>\n",
       "      <td>Held By</td>\n",
       "      <td>14.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>S/B</td>\n",
       "      <td>4030</td>\n",
       "      <td>2015-01-01 01:44:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-01-01 02:04:00</th>\n",
       "      <td>2015-01-01</td>\n",
       "      <td>504</td>\n",
       "      <td>02:04:00</td>\n",
       "      <td>Thursday</td>\n",
       "      <td>king and sherborne</td>\n",
       "      <td>Mechanical</td>\n",
       "      <td>9.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>E/B</td>\n",
       "      <td>4147</td>\n",
       "      <td>2015-01-01 02:04:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-01-01 02:12:00</th>\n",
       "      <td>2015-01-01</td>\n",
       "      <td>306</td>\n",
       "      <td>02:12:00</td>\n",
       "      <td>Thursday</td>\n",
       "      <td>main st. and upper gerard</td>\n",
       "      <td>Investigation</td>\n",
       "      <td>29.0</td>\n",
       "      <td>39.0</td>\n",
       "      <td>S/B</td>\n",
       "      <td>4049</td>\n",
       "      <td>2015-01-01 02:12:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-01-01 05:05:00</th>\n",
       "      <td>2015-01-01</td>\n",
       "      <td>306</td>\n",
       "      <td>05:05:00</td>\n",
       "      <td>Thursday</td>\n",
       "      <td>gerrard and sumach</td>\n",
       "      <td>Mechanical</td>\n",
       "      <td>30.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>W/B</td>\n",
       "      <td>4114</td>\n",
       "      <td>2015-01-01 05:05:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    Report Date Route      Time       Day  \\\n",
       "Report Date Time                                            \n",
       "2015-01-01 01:25:00  2015-01-01   504  01:25:00  Thursday   \n",
       "2015-01-01 01:44:00  2015-01-01   504  01:44:00  Thursday   \n",
       "2015-01-01 02:04:00  2015-01-01   504  02:04:00  Thursday   \n",
       "2015-01-01 02:12:00  2015-01-01   306  02:12:00  Thursday   \n",
       "2015-01-01 05:05:00  2015-01-01   306  05:05:00  Thursday   \n",
       "\n",
       "                                      Location       Incident  Min Delay  \\\n",
       "Report Date Time                                                           \n",
       "2015-01-01 01:25:00      broadview and gerrard     Mechanical        9.0   \n",
       "2015-01-01 01:44:00    galley and roncesvalles        Held By       14.0   \n",
       "2015-01-01 02:04:00         king and sherborne     Mechanical        9.0   \n",
       "2015-01-01 02:12:00  main st. and upper gerard  Investigation       29.0   \n",
       "2015-01-01 05:05:00         gerrard and sumach     Mechanical       30.0   \n",
       "\n",
       "                     Min Gap Direction Vehicle    Report Date Time  \n",
       "Report Date Time                                                    \n",
       "2015-01-01 01:25:00     18.0       S/B    4092 2015-01-01 01:25:00  \n",
       "2015-01-01 01:44:00     23.0       S/B    4030 2015-01-01 01:44:00  \n",
       "2015-01-01 02:04:00     18.0       E/B    4147 2015-01-01 02:04:00  \n",
       "2015-01-01 02:12:00     39.0       S/B    4049 2015-01-01 02:12:00  \n",
       "2015-01-01 05:05:00     60.0       W/B    4114 2015-01-01 05:05:00  "
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "file_name = path + pickled_dataframe\n",
    "merged_data = pd.read_pickle(file_name)\n",
    "merged_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(83365, 11)"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Report Date</th>\n",
       "      <th>Route</th>\n",
       "      <th>Time</th>\n",
       "      <th>Day</th>\n",
       "      <th>Location</th>\n",
       "      <th>Incident</th>\n",
       "      <th>Min Delay</th>\n",
       "      <th>Min Gap</th>\n",
       "      <th>Direction</th>\n",
       "      <th>Vehicle</th>\n",
       "      <th>Report Date Time</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>daym</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Report Date Time</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2015-01-01 01:25:00</th>\n",
       "      <td>2015-01-01</td>\n",
       "      <td>504</td>\n",
       "      <td>01:25:00</td>\n",
       "      <td>Thursday</td>\n",
       "      <td>broadview and gerrard</td>\n",
       "      <td>Mechanical</td>\n",
       "      <td>9.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>S/B</td>\n",
       "      <td>4092</td>\n",
       "      <td>2015-01-01 01:25:00</td>\n",
       "      <td>2015</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-01-01 01:44:00</th>\n",
       "      <td>2015-01-01</td>\n",
       "      <td>504</td>\n",
       "      <td>01:44:00</td>\n",
       "      <td>Thursday</td>\n",
       "      <td>galley and roncesvalles</td>\n",
       "      <td>Held By</td>\n",
       "      <td>14.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>S/B</td>\n",
       "      <td>4030</td>\n",
       "      <td>2015-01-01 01:44:00</td>\n",
       "      <td>2015</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-01-01 02:04:00</th>\n",
       "      <td>2015-01-01</td>\n",
       "      <td>504</td>\n",
       "      <td>02:04:00</td>\n",
       "      <td>Thursday</td>\n",
       "      <td>king and sherborne</td>\n",
       "      <td>Mechanical</td>\n",
       "      <td>9.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>E/B</td>\n",
       "      <td>4147</td>\n",
       "      <td>2015-01-01 02:04:00</td>\n",
       "      <td>2015</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-01-01 02:12:00</th>\n",
       "      <td>2015-01-01</td>\n",
       "      <td>306</td>\n",
       "      <td>02:12:00</td>\n",
       "      <td>Thursday</td>\n",
       "      <td>main st. and upper gerard</td>\n",
       "      <td>Investigation</td>\n",
       "      <td>29.0</td>\n",
       "      <td>39.0</td>\n",
       "      <td>S/B</td>\n",
       "      <td>4049</td>\n",
       "      <td>2015-01-01 02:12:00</td>\n",
       "      <td>2015</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-01-01 05:05:00</th>\n",
       "      <td>2015-01-01</td>\n",
       "      <td>306</td>\n",
       "      <td>05:05:00</td>\n",
       "      <td>Thursday</td>\n",
       "      <td>gerrard and sumach</td>\n",
       "      <td>Mechanical</td>\n",
       "      <td>30.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>W/B</td>\n",
       "      <td>4114</td>\n",
       "      <td>2015-01-01 05:05:00</td>\n",
       "      <td>2015</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    Report Date Route      Time       Day  \\\n",
       "Report Date Time                                            \n",
       "2015-01-01 01:25:00  2015-01-01   504  01:25:00  Thursday   \n",
       "2015-01-01 01:44:00  2015-01-01   504  01:44:00  Thursday   \n",
       "2015-01-01 02:04:00  2015-01-01   504  02:04:00  Thursday   \n",
       "2015-01-01 02:12:00  2015-01-01   306  02:12:00  Thursday   \n",
       "2015-01-01 05:05:00  2015-01-01   306  05:05:00  Thursday   \n",
       "\n",
       "                                      Location       Incident  Min Delay  \\\n",
       "Report Date Time                                                           \n",
       "2015-01-01 01:25:00      broadview and gerrard     Mechanical        9.0   \n",
       "2015-01-01 01:44:00    galley and roncesvalles        Held By       14.0   \n",
       "2015-01-01 02:04:00         king and sherborne     Mechanical        9.0   \n",
       "2015-01-01 02:12:00  main st. and upper gerard  Investigation       29.0   \n",
       "2015-01-01 05:05:00         gerrard and sumach     Mechanical       30.0   \n",
       "\n",
       "                     Min Gap Direction Vehicle    Report Date Time  year  \\\n",
       "Report Date Time                                                           \n",
       "2015-01-01 01:25:00     18.0       S/B    4092 2015-01-01 01:25:00  2015   \n",
       "2015-01-01 01:44:00     23.0       S/B    4030 2015-01-01 01:44:00  2015   \n",
       "2015-01-01 02:04:00     18.0       E/B    4147 2015-01-01 02:04:00  2015   \n",
       "2015-01-01 02:12:00     39.0       S/B    4049 2015-01-01 02:12:00  2015   \n",
       "2015-01-01 05:05:00     60.0       W/B    4114 2015-01-01 05:05:00  2015   \n",
       "\n",
       "                     month  daym  \n",
       "Report Date Time                  \n",
       "2015-01-01 01:25:00      1     1  \n",
       "2015-01-01 01:44:00      1     1  \n",
       "2015-01-01 02:04:00      1     1  \n",
       "2015-01-01 02:12:00      1     1  \n",
       "2015-01-01 05:05:00      1     1  "
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# define cols for year month day\n",
    "# apar_ds[\"year\"] = pd.DatetimeIndex(apar_ds['PMR_Open_Date']).year \n",
    "merged_data['year'] = pd.DatetimeIndex(merged_data['Report Date']).year\n",
    "merged_data['month'] = pd.DatetimeIndex(merged_data['Report Date']).month\n",
    "merged_data['daym'] = pd.DatetimeIndex(merged_data['Report Date']).day\n",
    "merged_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2017    27524\n",
       "2018    15612\n",
       "2016    14021\n",
       "2015    12221\n",
       "2014    11027\n",
       "2019     2960\n",
       "Name: year, dtype: int64"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get the record count by year\n",
    "merged_data[\"year\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "year  month\n",
       "2014  1         319\n",
       "      2         178\n",
       "      3        1278\n",
       "      4        1182\n",
       "      5         964\n",
       "      6        1159\n",
       "      7        1006\n",
       "      8        1031\n",
       "      9        1181\n",
       "      10        993\n",
       "      11        862\n",
       "      12        874\n",
       "2015  1        1241\n",
       "      2        1668\n",
       "      3        1048\n",
       "      4         948\n",
       "      5         869\n",
       "      6         995\n",
       "      7         893\n",
       "      8         972\n",
       "      9        1134\n",
       "      10        931\n",
       "      11        761\n",
       "      12        761\n",
       "2016  1        1183\n",
       "      2        1095\n",
       "      3        1187\n",
       "      4        1119\n",
       "      5        1142\n",
       "      6        1078\n",
       "               ... \n",
       "      9        1036\n",
       "      10       1196\n",
       "      11       1419\n",
       "      12       1520\n",
       "2017  1        2600\n",
       "      2        2222\n",
       "      3        2200\n",
       "      4        2110\n",
       "      5        2524\n",
       "      6        2448\n",
       "      7        1934\n",
       "      8        1786\n",
       "      9        1952\n",
       "      10       2064\n",
       "      11       2288\n",
       "      12       3396\n",
       "2018  1        1622\n",
       "      2        1287\n",
       "      3        1112\n",
       "      4        1289\n",
       "      5        1544\n",
       "      6        1598\n",
       "      7        1234\n",
       "      8        1472\n",
       "      9        1197\n",
       "      10       1160\n",
       "      11       1183\n",
       "      12        914\n",
       "2019  1        1595\n",
       "      2        1365\n",
       "Length: 62, dtype: int64"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get record count by year by month\n",
    "merged_data.groupby([\"year\", \"month\"]).size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Report Date</th>\n",
       "      <th>Route</th>\n",
       "      <th>Time</th>\n",
       "      <th>Day</th>\n",
       "      <th>Location</th>\n",
       "      <th>Incident</th>\n",
       "      <th>Min Delay</th>\n",
       "      <th>Min Gap</th>\n",
       "      <th>Direction</th>\n",
       "      <th>Vehicle</th>\n",
       "      <th>Report Date Time</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>daym</th>\n",
       "      <th>hour</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Report Date Time</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2015-01-01 01:25:00</th>\n",
       "      <td>2015-01-01</td>\n",
       "      <td>504</td>\n",
       "      <td>01:25:00</td>\n",
       "      <td>Thursday</td>\n",
       "      <td>broadview and gerrard</td>\n",
       "      <td>Mechanical</td>\n",
       "      <td>9.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>S/B</td>\n",
       "      <td>4092</td>\n",
       "      <td>2015-01-01 01:25:00</td>\n",
       "      <td>2015</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-01-01 01:44:00</th>\n",
       "      <td>2015-01-01</td>\n",
       "      <td>504</td>\n",
       "      <td>01:44:00</td>\n",
       "      <td>Thursday</td>\n",
       "      <td>galley and roncesvalles</td>\n",
       "      <td>Held By</td>\n",
       "      <td>14.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>S/B</td>\n",
       "      <td>4030</td>\n",
       "      <td>2015-01-01 01:44:00</td>\n",
       "      <td>2015</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-01-01 02:04:00</th>\n",
       "      <td>2015-01-01</td>\n",
       "      <td>504</td>\n",
       "      <td>02:04:00</td>\n",
       "      <td>Thursday</td>\n",
       "      <td>king and sherborne</td>\n",
       "      <td>Mechanical</td>\n",
       "      <td>9.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>E/B</td>\n",
       "      <td>4147</td>\n",
       "      <td>2015-01-01 02:04:00</td>\n",
       "      <td>2015</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-01-01 02:12:00</th>\n",
       "      <td>2015-01-01</td>\n",
       "      <td>306</td>\n",
       "      <td>02:12:00</td>\n",
       "      <td>Thursday</td>\n",
       "      <td>main st. and upper gerard</td>\n",
       "      <td>Investigation</td>\n",
       "      <td>29.0</td>\n",
       "      <td>39.0</td>\n",
       "      <td>S/B</td>\n",
       "      <td>4049</td>\n",
       "      <td>2015-01-01 02:12:00</td>\n",
       "      <td>2015</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-01-01 05:05:00</th>\n",
       "      <td>2015-01-01</td>\n",
       "      <td>306</td>\n",
       "      <td>05:05:00</td>\n",
       "      <td>Thursday</td>\n",
       "      <td>gerrard and sumach</td>\n",
       "      <td>Mechanical</td>\n",
       "      <td>30.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>W/B</td>\n",
       "      <td>4114</td>\n",
       "      <td>2015-01-01 05:05:00</td>\n",
       "      <td>2015</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    Report Date Route      Time       Day  \\\n",
       "Report Date Time                                            \n",
       "2015-01-01 01:25:00  2015-01-01   504  01:25:00  Thursday   \n",
       "2015-01-01 01:44:00  2015-01-01   504  01:44:00  Thursday   \n",
       "2015-01-01 02:04:00  2015-01-01   504  02:04:00  Thursday   \n",
       "2015-01-01 02:12:00  2015-01-01   306  02:12:00  Thursday   \n",
       "2015-01-01 05:05:00  2015-01-01   306  05:05:00  Thursday   \n",
       "\n",
       "                                      Location       Incident  Min Delay  \\\n",
       "Report Date Time                                                           \n",
       "2015-01-01 01:25:00      broadview and gerrard     Mechanical        9.0   \n",
       "2015-01-01 01:44:00    galley and roncesvalles        Held By       14.0   \n",
       "2015-01-01 02:04:00         king and sherborne     Mechanical        9.0   \n",
       "2015-01-01 02:12:00  main st. and upper gerard  Investigation       29.0   \n",
       "2015-01-01 05:05:00         gerrard and sumach     Mechanical       30.0   \n",
       "\n",
       "                     Min Gap Direction Vehicle    Report Date Time  year  \\\n",
       "Report Date Time                                                           \n",
       "2015-01-01 01:25:00     18.0       S/B    4092 2015-01-01 01:25:00  2015   \n",
       "2015-01-01 01:44:00     23.0       S/B    4030 2015-01-01 01:44:00  2015   \n",
       "2015-01-01 02:04:00     18.0       E/B    4147 2015-01-01 02:04:00  2015   \n",
       "2015-01-01 02:12:00     39.0       S/B    4049 2015-01-01 02:12:00  2015   \n",
       "2015-01-01 05:05:00     60.0       W/B    4114 2015-01-01 05:05:00  2015   \n",
       "\n",
       "                     month  daym  hour  \n",
       "Report Date Time                        \n",
       "2015-01-01 01:25:00      1     1     1  \n",
       "2015-01-01 01:44:00      1     1     1  \n",
       "2015-01-01 02:04:00      1     1     2  \n",
       "2015-01-01 02:12:00      1     1     2  \n",
       "2015-01-01 05:05:00      1     1     5  "
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# define cols for time of day\n",
    "merged_data['hour'] = pd.DatetimeIndex(merged_data['Report Date Time']).hour\n",
    "merged_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Report Date</th>\n",
       "      <th>Route</th>\n",
       "      <th>Time</th>\n",
       "      <th>Day</th>\n",
       "      <th>Location</th>\n",
       "      <th>Incident</th>\n",
       "      <th>Min Delay</th>\n",
       "      <th>Min Gap</th>\n",
       "      <th>Direction</th>\n",
       "      <th>Vehicle</th>\n",
       "      <th>Report Date Time</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>daym</th>\n",
       "      <th>hour</th>\n",
       "      <th>time_of_day</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Report Date Time</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2015-01-01 01:25:00</th>\n",
       "      <td>2015-01-01</td>\n",
       "      <td>504</td>\n",
       "      <td>01:25:00</td>\n",
       "      <td>Thursday</td>\n",
       "      <td>broadview and gerrard</td>\n",
       "      <td>Mechanical</td>\n",
       "      <td>9.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>S/B</td>\n",
       "      <td>4092</td>\n",
       "      <td>2015-01-01 01:25:00</td>\n",
       "      <td>2015</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>overnight</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-01-01 01:44:00</th>\n",
       "      <td>2015-01-01</td>\n",
       "      <td>504</td>\n",
       "      <td>01:44:00</td>\n",
       "      <td>Thursday</td>\n",
       "      <td>galley and roncesvalles</td>\n",
       "      <td>Held By</td>\n",
       "      <td>14.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>S/B</td>\n",
       "      <td>4030</td>\n",
       "      <td>2015-01-01 01:44:00</td>\n",
       "      <td>2015</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>overnight</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-01-01 02:04:00</th>\n",
       "      <td>2015-01-01</td>\n",
       "      <td>504</td>\n",
       "      <td>02:04:00</td>\n",
       "      <td>Thursday</td>\n",
       "      <td>king and sherborne</td>\n",
       "      <td>Mechanical</td>\n",
       "      <td>9.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>E/B</td>\n",
       "      <td>4147</td>\n",
       "      <td>2015-01-01 02:04:00</td>\n",
       "      <td>2015</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>overnight</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-01-01 02:12:00</th>\n",
       "      <td>2015-01-01</td>\n",
       "      <td>306</td>\n",
       "      <td>02:12:00</td>\n",
       "      <td>Thursday</td>\n",
       "      <td>main st. and upper gerard</td>\n",
       "      <td>Investigation</td>\n",
       "      <td>29.0</td>\n",
       "      <td>39.0</td>\n",
       "      <td>S/B</td>\n",
       "      <td>4049</td>\n",
       "      <td>2015-01-01 02:12:00</td>\n",
       "      <td>2015</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>overnight</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-01-01 05:05:00</th>\n",
       "      <td>2015-01-01</td>\n",
       "      <td>306</td>\n",
       "      <td>05:05:00</td>\n",
       "      <td>Thursday</td>\n",
       "      <td>gerrard and sumach</td>\n",
       "      <td>Mechanical</td>\n",
       "      <td>30.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>W/B</td>\n",
       "      <td>4114</td>\n",
       "      <td>2015-01-01 05:05:00</td>\n",
       "      <td>2015</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>morning_rush</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    Report Date Route      Time       Day  \\\n",
       "Report Date Time                                            \n",
       "2015-01-01 01:25:00  2015-01-01   504  01:25:00  Thursday   \n",
       "2015-01-01 01:44:00  2015-01-01   504  01:44:00  Thursday   \n",
       "2015-01-01 02:04:00  2015-01-01   504  02:04:00  Thursday   \n",
       "2015-01-01 02:12:00  2015-01-01   306  02:12:00  Thursday   \n",
       "2015-01-01 05:05:00  2015-01-01   306  05:05:00  Thursday   \n",
       "\n",
       "                                      Location       Incident  Min Delay  \\\n",
       "Report Date Time                                                           \n",
       "2015-01-01 01:25:00      broadview and gerrard     Mechanical        9.0   \n",
       "2015-01-01 01:44:00    galley and roncesvalles        Held By       14.0   \n",
       "2015-01-01 02:04:00         king and sherborne     Mechanical        9.0   \n",
       "2015-01-01 02:12:00  main st. and upper gerard  Investigation       29.0   \n",
       "2015-01-01 05:05:00         gerrard and sumach     Mechanical       30.0   \n",
       "\n",
       "                     Min Gap Direction Vehicle    Report Date Time  year  \\\n",
       "Report Date Time                                                           \n",
       "2015-01-01 01:25:00     18.0       S/B    4092 2015-01-01 01:25:00  2015   \n",
       "2015-01-01 01:44:00     23.0       S/B    4030 2015-01-01 01:44:00  2015   \n",
       "2015-01-01 02:04:00     18.0       E/B    4147 2015-01-01 02:04:00  2015   \n",
       "2015-01-01 02:12:00     39.0       S/B    4049 2015-01-01 02:12:00  2015   \n",
       "2015-01-01 05:05:00     60.0       W/B    4114 2015-01-01 05:05:00  2015   \n",
       "\n",
       "                     month  daym  hour   time_of_day  \n",
       "Report Date Time                                      \n",
       "2015-01-01 01:25:00      1     1     1     overnight  \n",
       "2015-01-01 01:44:00      1     1     1     overnight  \n",
       "2015-01-01 02:04:00      1     1     2     overnight  \n",
       "2015-01-01 02:12:00      1     1     2     overnight  \n",
       "2015-01-01 05:05:00      1     1     5  morning_rush  "
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# define time of day column\n",
    "merged_data['time_of_day'] = merged_data['hour'].apply(lambda x:get_time(x))\n",
    "merged_data.head()\n",
    "\n",
    "# add a special timeframe for weekends\n",
    "# df['NewCol'] = df.apply(lambda x: segmentMatch(x['TimeCol'], x['ResponseCol']), axis=1)\n",
    "# weekend_time(day, tod)\n",
    "merged_data['time_of_day'] = merged_data.apply(lambda x: weekend_time(x['Day'], x['time_of_day']), axis=1)\n",
    "merged_data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "morning_rush     25907\n",
       "aft_rush         13655\n",
       "midday           13546\n",
       "evening           8928\n",
       "wmidday           5370\n",
       "wmorning_rush     4210\n",
       "waft_rush         3481\n",
       "overnight         3274\n",
       "wevening          3223\n",
       "wovernight        1771\n",
       "Name: time_of_day, dtype: int64"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_data['time_of_day'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Report Date Time\n",
       "2015-01-01 01:25:00    1\n",
       "2015-01-01 01:44:00    1\n",
       "2015-01-01 02:04:00    1\n",
       "2015-01-01 02:12:00    1\n",
       "2015-01-01 05:05:00    1\n",
       "Name: target, dtype: int64"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# define target column\n",
    "if targetcontinuous:\n",
    "    merged_data = merged_data.rename(columns = {'Min Delay':'target'})\n",
    "else:\n",
    "    # np.where(df['age']>=50, 'yes', 'no')\n",
    "    merged_data['target'] = np.where(merged_data['Min Delay'] >= targetthresh, 1, 0 )\n",
    "merged_data['target'].head()\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7f398477cef0>"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAFKCAYAAAA9n7WvAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xm8ZFV57vHf08xTMygOyKAQhDgBTqByUeDikIgJKkZURIxIxAE0ouIQnKKJ0VwVIiAkiEBUUAPGAWgQUQRBEBAIGBQuaoxRvCooINA894+1i65zuk6dql27hk0/38+nP921T9fbq7vrvHvtNbxLtomIiPZYMu0GRETEcJK4IyJaJok7IqJlkrgjIlomiTsiomWSuCMiWiaJOyKiZZK4IyJaJok7IqJlVh9TXO+1ZN9GAi2793SaijXr8Wa5bbMeb5bb1nS8WW5b0/FmuW1jiqdBfl963BERLZPEHRHRMkncEREtk8QdEdEySdwRES2TxB0R0TJJ3BERLZPEHRHRMkncEREtk8QdEdEySdwRES2TxB0R0TJJ3BERLZPEHRHRMkncEREtk8QdEdEySdwRES2TxB0R0TJJ3BERLZPEHRHRMkncEREtk8QdEdEySdwRES2TxB0R0TJJ3BERLZPEHRHRMkncEREtk8QdEdEySdwRES2TxB0R0TJJ3BERLZPEHRHRMkncEREtk8QdEdEySdwRES2TxB0R0TJJ3BERLZPEHRHRMkncEREtk8QdEdEySdwRES2TxB0R0TJJ3BERLZPEHRHRMkncEREtk8QdEdEySdwRES2TxB0R0TJJ3BERLZPEHRHRMkncEREtk8QdEdEySdwRES2TxB0R0TJJ3BERLZPEHRHRMkncEREtk8QdEdEySdwRES2TxB0R0TJJ3BERLZPEHRHRMkncEREtk8QdEdEySdwRES0zUOKWtKukA6tfbyrpEeNtVkRELGTRxC3pSOCtwBHVpTWAU8bZqIiIWNggPe59gOcBvwew/TNgg3E2KiIiFjZI4r7LtgEDSFpvvE2KiIh+Bkncp0k6DthI0kHAucDx421WREQsZPXFfoPtD0vaC7gV2A74G9vLxt6yiIjoadHEDWB7maRLOr9f0ia2/99YWxYRET0tmrglHQy8F7gDuBcQZbx76/E2LSIiehmkx/1m4NG2bxl3YyIiYnGDTE7+CLh93A2JiIjBDNLjPgK4qBrj/kPnou03jK1VERGxoEES93HA14GrKWPcERExRYMk7ntsv2nsLYmIiIEMMsZ9vqRXS3qopE06P8besoiI6GmQHvdLqp+P6LqW5YAREVMyyM7JlHCNiJghCyZuSXvY/rqk5/f6uu0vjq9ZERGxkH497t0oq0n27vE1A0ncERFT0C9xfx/A9oETaktERAyg36qSd06sFRERMbAcFhwR0TL9hkq2l/T9HtcF2PbjxtSmiIjoo1/ivoneE5MRETFF/RL3XbZvnlhLIiJiIP3GuL89sVZERMTAFkzctl83yYZERMRgsqokIqJlkrgjIlpm0cQtaV1J75J0fPV6W0nPHX/TIiKil0F63CdSjix7SvX6p8D7x9aiiIjoa5DEvY3tDwF3A9i+g7IJJyIipmCQxH2XpHUoFQGRtA1dhwZHRMRkDXICzruBs4AtJJ0KPA1IxcCIiCkZ5ASccyRdDuxCGSI51PYtY29ZRET0NMiqkvNs/8r2V2x/2fYtks6bROMiImJl/Y4uWxtYF3igpI1ZMSG5FNhsAm2LiIge+g2VHAwcRknSl7Micd8K/NOY2xUREQtYMHHb/hjwMUmvt33UBNsUERF9DDI5eZSkxwCPAtbuuv7pcTYsIiJ6WzRxSzoSeAYlcX8VeA5wIZDEHRExBYNswHkhsCfw8+rE9x2AtcbaqoiIWNAgifsO2/cC90haCvwC2Hq8zYqIiIUMsnPyMkkbAcdTVpf8Drh0rK2KiIgFDTI5eUj1y2MlnQUstd3r9PeIiJiAvolb0uqUycjtq0vXUeqWRETElCw4xi1pM+Ba4K8pm3AeBhwOXFt9LSIipqBfj/sDwDG2P9p9UdIbgA8CB4yzYRER0Vu/xL2L7VfMv2j745J+ML4mRUREP/2WA97R52u3N92QiIgYTL8e94aSnt/juigVAiMiYgr6Je4LgL0X+No3x9CWiIgYQL/qgDmeLCJiBg2y5T0iImZIEndERMsMcubkSpUAe12LiIjJGKTHffGA1yIiYgL6HRb8EMo293Uk7cTcw4LXnUDbIiKih37LAZ8FvALYHPgIcw8Lfvt4mxUREQvptxzwJEknA/vZPnWCbYqIiD76jnFXJ98cPKG2RETEAAaZnFwm6c2StpC0SefH2FsWERE9DXJ02Surn1/bdc3k3MmIiKkY5OiyR0yiIRERMZhFE7ekNYDXALtVl74BHGf77jG2KyIiFjDIUMkxwBrAJ6rX+1fXXjWuRkVExMIGSdxPsr1D1+uvS7pqXA2KiIj+BllVslzSNp0XkrYGlo+vSRER0c8gPe7DgfMl3UjZPbkVkFrdERFTMsiqkvMkbQtsR0nc19v+w9hbFhERPQ2yqmRt4BBgV8r67W9JOtb2neNuXERErGyQoZJPA7cBR1Wv9wNOBvYdV6MiImJhgyTu7eatKjk/q0oiIqZnkFUlV0japfNC0s7At8fXpIiI6GeQHvfOwMsl/bh6vSVwnaSrAdt+3NhaFxERKxkkcT977K2IiIiBLTpUYvtmYAtgj+rXvweW2L65eh0RERM0yCnvRwJvBY6oLq0JnDLORkVExMIGmZzcB3gepaeN7Z8BG4yzURERsbBBEvddtk3ZfIOk9cbbpIiI6GeQxH2apOOAjSQdBJwLnDDeZkVExEIGqVXyYUl7AbdS6pX8je1lY29ZRET0NMhyQKpEvQxA0mqSXmr71LG2LCIielpwqETSUklHSDpa0jNVvA64EXjR5JoYERHd+vW4TwZ+DVxMOabscMpSwD+zfeUE2hYRET30S9xb234sgKQTgFuALW3fNpGWRURET/1Wldx3irvt5cBNSdoREdPXr8e9g6Rbq18LWKd6LUpxqaVjb11ERKxkwcRte7VJNiQiIgYzyAaciIiYIUncEREtk8QdEdEySdwRES2TxB0R0TJJ3BERLZPEHRHRMkncEREtk8QdEdEySdwRES2TxB0R0TJJ3BERLZPEHRHRMkncEREtk8QdEdEySdwRES2TxB0R0TJJ3BERLZPEHRHRMkncEREtk8QdEdEySdwRES2TxB0R0TJJ3BERLZPEHRHRMkncEREtk8QdEdEySdwRES2TxB0R0TJJ3BERLZPEHRHRMkncEREtk8QdEdEySdwRES2TxB0R0TJJ3BERLZPEHRHRMkncEREtk8QdEdEySdwRES2TxB0R0TJJ3BERLZPEHRHRMkncEREtk8QdEdEySdwRES2TxB0R0TJJ3BERLZPEHRHRMkncEREtk8QdEdEySdwRES2TxB0R0TJJ3BERLZPEHRHRMkncEREtk8QdEdEySdwRES2TxB0R0TJJ3BERLZPEHRHRMkncEREtk8QdEdEySdwREW1je2o/gFfParxZbtusx5vltuXvOhuxZj3eLLfN9tR73K+e4Xiz3LZZjzfLbWs63iy3rel4s9y2puPNctumnrgjImJISdwRES0z7cT9yRmON8ttm/V4s9y2puPNctuajjfLbWs63iy3DVUD5xER0RLT7nFHRMSQkrgjIlpmoolb0rqS3iXp+Or1tpKeO8k2LEbSetNuQ9tIeq6kRj9Lkp4v6R8lfUTSPk3GnjWSdpV0YPXrTSU9omacJZKe2mzrYhZNusd9IvAH4CnV658C7x8lYFM3A0lPlfQfwHXV6x0kfWLEto0joT1V0kskvbzzY4RYKyWImknjxcANkj4k6Y/rtqerDZ8A/gq4GrgGOFjSP40atymS1pG0XUOxjgTeChxRXVoDOKVOLNv3Ah9pol3dmvzMVfE2lvQ4SY/v/KgZZzVJ547SlnnxNmkw1tMGuVY7/iQnJyVdZvuJkq6wvVN17SrbO4wQ83PA5cDLbT9G0jrAxbZ3HDLOJcALgS91te0a248ZoW2nUG5SXwBOtH1d3VhVvJOBbYArgeXVZdt+Q81437P9+HnXLrf9hBqxlgL7AQcCptykP2P7thqxrgUe4+rDWd38rrb96GFjdcXcFDgIeDiweue67VcOGWdv4MPAmrYfIWlH4L22n1ezXVcCOwHf6/rcfd/242rGew/wfeCLbuCbewyfufcBrwB+RPmcdOLtUTPel4D9bf+2zvvnxbqB8vc8EfjaKP9+C3xvrXStrtUX/y2NuqtKrJ1vyG0oPfBRbGP7LyTtB2D7DkmqE8j2T+a9dflCv3fAeC/rSmgnShopoQFPBB416jekpO2BRwMbSnp+15eWAmvXiWn7VklfANYBDgP2AQ6X9HHbRw0Z7gfAlsDN1estKMloFGcC3wLOZbT/13cDTwa+AWD7SkkPHyHeXbZdfTaaGKp7E7AecI+kOwGVZnppzXiNfOa6vIjyPXtXQ/HuBK6WtAz4fedizRvLI4H/DbwSOKrqFH7K9n8OGkDSU4CnAptKelPXl5YCq9VoU0+TTtzvBs4CtpB0KvA0Sg9tFE3dDH5SjQ9a0prAG6iGTUbRcEK7BngI8N8jNms74LnARsDeXddvo/RKh1L1Ql9J6ZmdDDzZ9i8krUv5Nxz27/kA4DpJl1avnwRcXPWuqNm7Xdf2W2u8b757bP+2Zt+gl9MkHQdsJOkgyr/j8XWD2d6gqYZVmvrMdcfbCPhFQ/G+Uv0YWXVzWgYsk7Q7ZcjqEElXAW+zffEAYdYE1qfk1u7/i1spT/SNmPg6bkkPAHah9AS+Y/uWEeM9E3gH8CjgHKqbge3zh4zzQOBjlDuuqliH2v7VCG2bn9BO6k5otrcaMt75wI7ApXTdnEZ4TH/KgB/GxeJ8GjjB9jd7fG1P2+cNGe/p/b5u+4Ihm4ik9wMX2f7qsO+dF+efgfOAtwEvoNzg17D9VyPE3At4JuVzd7btZSO2cWNgW7qennr93ywS498pnaENaPYz90TK0881TcRrUpWbXgbsD/wP8M/Alyh//9NtDzz/I2kr2zdLWs/27xd/x5BtnfAY93m291zsWo24jd4MmjKphFYnkVXxGhn3bQNJt1GGEO4C7q4uDz2EUN1030FXogXeZ/vOEdu3lLn/B/+vZpxXAYcCm1PGa3ehzPkMNYY8jptnFfda4DjKxPO9DcS7iRVj5d3t27pGrP+kdLBOtP3TeV97q+2/HyLWUyiJf33bW0raATjY9iHDtqtn/EkkbklrA+sC5wPPoHzgoYz7fM127ZUITd0MqtUUr2flJDb1nkA3SQ+mDB0AXGq79iOnpIso476X0zXua/sLQ8bZhTIc8seUR8XVgN/XHVdtOt4sk3Qw8F7gDkoi64xJD514qnhXUz4f37G9YzWf8R7bf1Ez3t/PH2LqdW2IeBfY7ntTGDLeA7perg3sC2xi+29qxFJTY/njWOzQbVJj3AdTxnc3oySJTuK+Fai1zKvrZvDA6tGw+2awWY2QZ1DukP9OV09gFGNIaC8C/oEyMSbKBMrhtj9fs4lNjfseTVkSeDplMuvlwB81HG/bEduIpOcBu1Uvv2H7y0O8tzN00NMIN/g3A49u8CnxTtt3SkLSWrav12hLF/eiLFfs9pwe1wZ1uaQPUoYguodKvlcnWI+hzI9KuhAYOnFTcslbKBP33cNMtVa8NL3YodtEErftjwEfk/T6GhNyC+m+GXT/p9e9Gdxp++NNNKxL0wntHcCTOr3saqjjXKBu4v6ypD8ZddwXwPYPJa1mezllBc1FsxRP0t9ReqKnVpcOlbSr7bcNGOLDo/z5ffwIuL3BeD+VtBGlI7JM0q+Bnw0bRNJrgEOArSV1r+jZABjl/2Kn6udduq4ZqLscsHt53RLK91ndCdpTgc9RJu7/CjgA+GXNWGNZ7NAxjcnJx1AmErvvaJ8eIV4jNwNJL6H06s6hgZ5AFbOzbv2+dbmSLrJda3ebpKttP7br9RLgqu5rQ8brHve9i5pLxyR9kzKpewLwc8oKhFe45vr8puNVMb8P7OiySQVJqwFXeMj10tVyvTvmxVnLdq3kK2knyhLRS5j7uau1Tnpe7KcDGwJnDbv8TtKGwMbABykTsR231R1/r+J2bsaNqCbsO+4B/i/wYds/qBHrcttPmPf9WmtoZxyLHbpNdDmgyi6xZ1AS91cpj1wXArUTN/Bb9djJVeNm8FjKbPIerBgqqd0TqNxe3W2vlPQhSgIaZZ3uWZLOBj5Tvf4Lyr9jLQ0uHdufMgz0OuCNlHXXLxgx3pIG43VsBHSSzoY1Y5xH+Wb8XfV6Hco3Zd2t5scBX2feZN0oJO0KbGv7xOqp7GHATcPEcNnQ8ltgv+rm9GBKvlhf0vq2f1yzeT+U9HngXzzihrSqnbuPGqNLZ9L6vyX9KeVJZfM6gaqhr5c21bD5Jr2q5GpgB0pPZ4dqou0E23sv8tZ+Mbt722sDe1J2oQ21ZlLS9cDjhu2ZLBJzK8qyojUpCWhD4BO2fzhCzBdQljwK+KbtfxshligfrkfYfp+kLYCH2r50kbe2jsoGrb+jTJCLMtZ9hO3PDhnnSs/bldvr2hDxaj+BLRDvSMpwwXa2HylpM8pStlrbrSW9jrL/4n/o6tAM+6TSFW8DyvDhgZSb878An7V9a814GwJHsmLu4gLKTtahd1KqlMr4FqWjcBRlvuw9tr9UI1avYdffApfZPnPYeCvFn3DivtT2kyVdDuxO2fBxjUfYytzjz9gQOHnYySKVXVKvH2WVxgJxNwWwXXesbGwkHUP5ZtzD9h9Xk7zn2H7SIm/tvP9q+k/YDTsM8WfA5rb/qXp9CbBp9eW3jDAJ24n/UMo4t4BLbP+8RoxvUz4n36tePwE42vZT+r9zwXh/S9kh+u/MHSqpuxyw6S30PwR2buoRf17s3ShPjxtR5mneN2ynRmVz2zXASdWl/YEdbD9/4XeNn6RPAttT5regPDFeS7kp3Gj7sFHiT3rn5GXVxMnxlNUlv6Ms7G/S7dRbgfBg4HpJ32XEjQFVT/ZIyqO+gCWS7gGOsv3eGvEutL1rNSbdnShH3c68s+3HS7qCEujX1dDOoJqu7PgWSm+sYy1Kol2PMg48dOKWtH21sqIzidVZn7uZpM1qzGEcBpwuqTPh91DKkFVdL6l+PqLrmoFaywFpfgv9Tyg9xUZUwy5/SulxP5xSFOtU4H9Rhv0eOWTIbWx3D6O9p7p5DdOmo+jfAakz3/BHlA7RPdWfcQxlSG0vyrDYSCaauL1i8fmxks4CltoeqQbFvGVaq1GW3p1WI9SRo7RjnsMowxlPsn0TgKStgWMkvdH2/xkmmO1dq5+b3s58d/WN1Pkm35Qhxlltd2qJdIaFtrV9rkoJgjqfrTVt/6Tr9YVVT+9XIySgN1FO2O5VNW/oOQzb31VZG70d5cZ5ve27F3lbv3i1Srj20egWeuBG4BuSvsLcDs0/1ox3A2W46h9sd69O+XzVAx/WHdXqoAsBVCrw3TFkjMuqn59GmX/7XPV6X0oHs46HUTocnZveesBmtpdLGrU+0+SGSiStTpmM3L66dB1ltvueEeN2z/jeA9zsebueJq3qwe41f21ulRjP6TzC1oh7su39F7s2RLyXUnqLj6c8ar4QeKft0/u+ceU4B1GS4ya2t5G0LXCsh98E9UPbPZdLSvqR7W2GiTfv/Wt73u7GXtf6vH8P21/X3KJc97H9xSHb02i8rrivp6zEeTKMvoW+GjPv1b731Iy3vu3fLf47B463I+Wz25ls/jVwQJ0OYbVC5ZmdG7GkNSjfr0NPgEr6S+CdrNhzsRvwAcrQ0LttHz5szG4T6XFXEyTnU1ZVXEH5izwX+Iik3W0Pvc60w/YFmrub8IYh2zaOYYg15iftqq2/rD4Mdc2ZC6huhkOXYO1qz6nVfMOelL/vn9ec6X8tJVFcUsW9QdKDasS5RNJBtuf0EFV2F446pHYR5Qa12LWFPJ2y+qPXRLqBYRPtbg3H63gwZcv79ygTfyPVq+4k6GpS0Q0k3XskvZaVN7nULbNwHfAhSj2gjSg93D+nXjXJzShrwDvzC+tTYzNfNVR6DmXop3MDfXtXnhspacPkhko+ABxj+6PdFyW9gbJO9IC6gTX6bsL1oPFhiH4rU4ZetSLpCODtwDqSOrPvqmINfXq05haM/wUrlhciaZMaE2N/sH2Xql1i1Q2lzqPcG4EzVNbUd8aen0AZ6/7zGvGQ9BDKY+s6Kmumu3fYrjtoHNtHVj+PWs2y4/sNx6OK905J76LUUjkQOFrSacA/2/7RsPFU9l2cDGxSvb6FUvv+2ppNPBm4HngWZav/SxltY8qZwG8on5f/GiEOlFVHV2jF2vCnU1bUDKWaYzjDpa79yCtIeplUrZLrbW+/wNd+YLv2llyVkot7ed5uQg+4WUMNFjfvirmcrtrA3V8C1rZdq9ct6YO2j1j8dy4ap1OYR5S617+ufr0R8ONhx11V1qj/hrIz9PWUHXf/YfsdNdu3ByueLq61/fU6capYB1AK9z8R+C7MKbdwUo0hjrUoKwQeztyaNkNNOo/jczcv/g6UxP1sytPuLsAy228ZMs5FwDtcVduU9AzgA66/iewK2zt1VrpUT6Bnu/5BCo3V/6jiPQTYuXo5Z+WRpEcPesNSObHpU7a/21Tbuk2qx91vsmDU7b5L5i3h+xXDHcn2IM0teD5HnUkY240VTJ8X9wg1ULKzk5glHUspgvPV6vVzKJtLhvU24C8ps+UHUx4RT6gRp9O+r1OGEUZm+yTgJEkv8JDFsxZwJuVx/HJGPwSkcdVT7AHALZT/g8Nt362yy/YGysqdYaznrhLJtr8x4kqVzkTub6re/M8pN8G6LpL0WNsjr9QAqBL1Qr3kkxl8aG13ypF7N1M6cZ2h11rLMuebVOKef9JKhyiPrKPotZvwa0O8fzXKWFZjlfHHRQuU7KT+7s4nuauOtO2vqRwtNRTb90o6AzjDM7hevfIElaqRvwGoboB/bfudQ8bZ3PazG2jP9ppbA6Rj1G/wBwLP717xA/f9H9VZvnljNfRycvX6ZQy5C3OeT1b/9u+kFJpaH3jXsEG0Yg/B6sCBkm6k3EgbTZDz/9ghfu9zxvDn32dSQyUn9vv6qON81U1hVxh+N+G4H1mbpOZLdp5N2Sl2CuWb4GXAbrafNeD7569XF6UCWq316uOkrnNOu64N/X+vsrHiqFF7eCp1qf9koa/PT7zTUiXZ99C1W5eyKuI3Q8bp9VTbSYQe9slWZfnpgsbx71fz8/Ig5j4d1y0VMMekqgM2OgHTI/4XqWbhVU5+fqntUxd5W8fM97S7NF2ycz9K4u3c6L5ZXRtUo+vVx2y16t/sDwAqa83XGvTNY+jh3TUryXkR21B2+y2h/N33pDzhDfv37Uz+b0fpfHS2ke9N+dwNZdb/7VRKCH+EsirlF8BWlEnYRnaJT3rnZGNUTgx5LWXFwJcoZ8W9lrLU5kpWlO9czEin70xYIyU7O6rVI4eO0J6XM2+9uu0bJb2MshyqVuKunqD+HngQK3rydZdmdpwCnFc9/ZmyMeWk/m+Zo+ldot9uON64nEqpGX4NIxTB6lpWeA7weFeHZUt6Nyu2hU+FpKfZ/nb3jX0Bw6wIex9lKPPcajJ2d4brFPU18bKuTZF0JmU1xMWU5LsxpZjTobaH2vLaRhqtZOdHbR+mBQ4H8IDb/PvN6I8y269SH2PvmmvK+8V9Nl1lNm2fXTPO/Op763eeOO5vVO1zaDDe9ZRaIp0nn7UopYl7rjqbBK0o59rYsKlWlHS+CtipmmO41PaTm4g/6bKuK93RBrjLLWRrV3WoJZ1AmUXfsnMnvz9SOVHnWtu3uWw82oBSUOiSIUN1JppGPRyg0fXqXf6n6aRduY5ySvu5ktaVtMGwnxd1Vd+j1E9Zg9Kbr1V9rwWOrL6/zmPulve6G4ROBi6V9G+UTsM+DPfkMw53V09iD1OPqn6uV6vkN5LWpwwDnSrpF6xYUTOySQ+VXMzKy2l6XRvEff8ILvv/b7o/J+3KMcz9t/p9j2uLst2pv3AZPQ4FGCLUDlqxIaib6JqQqeEylWqNZ9BMspizLZ8ybvsw4FiGHyrbh6r6XtWmn1U30PurAyllKtZgbp36Wv8Xtv9W0tcoRaUADrR9xcitHM1zKU9ie1C/Nsl8V1GWOr+RssloQ8oKmkZMast7I7vX5ulOGmLFrsImxkNn1ZzDTKvHr1H+D0c6FGBc69Upn4vbKbv/7vvjqL8NHJrblt9o9T2VU+P/mvK0eJBKnZftPMR5mGO2g2uesLQQl4qMtU+Walo1R/NZSdfZvqqhsLtXHaJ7qZ4oFlj+WcuketzPouxe25wy09q9e+3tdQKOMWnMshurDRbHVK8PoVRvq2ttd9WesP27KpFM1ZhWITW1Lb/p6nsnUnp5nXreP6VM1s1K4v6OpEfZ/o9pN2QCflUN4TyN8tm4kDJnNnDROq04q3MbrXxWZ2MT0pOsDrgE2G+IZXoxT9VD/DgrNtycCxzmmoc/qOFDAUYl6S22P6QF6iPXHGvsxB5pW76ko4F/tX2RpL0oTwNNVN/rTGLdt85c0lUe4XzNJkm6jjK0dBPj3+AyVZKWAf/K3M1GL7W91xAxxnJW53wTG+OuHusPZvBlejFPlaBfvOhvHFzThwKMqjMheVnf31XPqNvyb6BUs3wopV7zqQ2tXrqrWlPeGXrZhtnaSt/ELtG2eJDt7s2Cn5I01Ek17jqrs9GWzTPpo8veRalb8jm6ijA1eSe6P5O0OeUsvNqPcj1irkFDhwLMqmrS9STbL2sg1laUm+eLKROwn6GcmfifNeM9E3gHpYD/OZT/2wPdVR8kJkPSucCnWFE+Yz/K/8XM7fWYdOLutdbVtuse07RKaeJRrkfMp7JypbtPDxmjkQ0zkvoeyjro+vIFYp9NWRve5GHQO1FqXj9ulDkXSQ+gbNYQpZzBSrXcY/wkbQkcTZlvMKVe+6GzuEuztRtwVkVq/oTxkynjl1dSaoxASbhDjSU3tWFG0i8pZxx+hrL6Y045Ats9p2RLAAAIAElEQVQXjBD7OMqyyS8x92lv2BoZa1CGD15MWUp4AfAZ22fUbNd583t0va7F9Ek6wvYHp90OmPwGnDWA11BO/4By+MFx98fH8zG5pdpO3v0oN8rp208EHuXR795NbZh5COUw1f0oh+h+hZIU6xbt7/az6scSVtTNGFg1Ibkf5aDbS4HPAq+23avu+iDx1qYshX2gSiGn7iWyQ5+6EhOxL2XSceomPVRyAmUhf2en1P7ActuvmlgjWmyBR7k3uGbFMUmnV+//7xHb9TFK0m1yw8xalET5D8B7bR81ShtHpXIqyr8CX2hiTkbSoZTJ4c0oJ7d0L5E93vbRo/4Z0Sz1qDA5LZNO3Cstc5qlpU9tJOkwzzsSboj3ng/sSOlBdifcocaS1btsr13jHMEqYf8pJWk/nDK08S+2ax1L1VRdlnGR9Ppp35RiME3WMhnVpBP394B9XZ19p1IC9POz8o/RRpJ+bHvLmu99eq/ro4wlj0LSScBjKAdhfNb2NQ3EfILty2ft79pN5SSYRzG3bvNQE8Qxfqtyj3tPyk6xGymPhluRpU8jkfQT21tM6c9udMOMpHtZMXHYHa+RMgYqlfzwDJ3SUxWtegYlcX+VcnLKhbZfOM12xcokvd32B6bdDpjw5KTt8zq1GFixbniWNhu00dB3Xkm3LfC+YRNkoxtmbA9zVuhApJVO6Vki6R5m55SeFwI7AFfYPlDSgxnhvM6oT9IjKeUkHmz7MZIeBzzP9vsBZiVpw+R73GtTthrvSkkc3wKOtX3nxBrRQosk2nVst/ZAjHGT9EbKEWGv9rxTeii1zKd6So+qGs2SLqccMHsbcI3tRk5KicFJuoByEMtxXeUHGj1FvimT/ob/NOWD2ZmM2Y+ymWTfCbejVWzPZNnQcW6YadBYTulp0GUqpxodTyk29TvKZHFM3rq2L+0UIqvcM63G9DPpxL3dvBUk56ucEBHt9BT6bJiZEWv02olo+5fVvoKpsn1I9ctjJZ0FLLXdWPnPGMotVa2YTt2YFwIjLZUdl0kn7isk7WL7OwCSdqY9Z+/Fysa5YaYp4zqlZ2RVadnnUA4qgDJncNb0WrTKey3wSWB7Sf9FqYg4cn2bcZj0GPd1lInJzoaRLSkf1nu5n5aKXFXM2oaZDknL6dri3v0lSj3yqfS6JW0GnE/p0V1RtWcnys1wd9u1D4GO0agcjrHEM3yi1qQT91b9vj6LxVyiv6Y3zKwqJH0KuHL+5imVgzKeYPuAqTRsFVbNNbyclYuu1a4DPy4TLzKluSdkPxDYwPfTE7Lv78axYWZVIel6L3CyuaQf2N5u0m1a1Um6CPgOpWZ753xNbE/7MOOVTLrHfd8J2bYfWT0unm77/npC9v3auDfM3J/124U3Szv0ViWztKV9MZOenFzVTsi+XxvHhplVyIZVHfP5RKkQGJN3sso5ol9mbu2emTvoZdKJu9ETsiNa7AJg7wW+9s1JNiTucxdlcv0drHiCNDBzB71MeqjkzcC2lCVkH6SckP0Z2x+fWCMiInqQ9CNg5zacQDSNycnGTsiOiGhKtRP4xbZvn3ZbFjPVo8uqQ1xfbDsnv0fEVEn6N+DRlPX13WPcM7cccCJj3JKWUnYlPYyyzndZ9fpwynmHSdyxSpK01vwKmb2uxUScUf2YeRPpcUs6E/g1cDHlgNWNgTUpJyhfOfYGRMyoXkvQ2rQsLaZjUqtKtrb9WLjv3MlbgC1neUtpxDhJegjlCXQdSTsx97DgdafWsFWQpNNsv0jS1axcPtmzeLTipBL3fae4214u6aYk7VjFPQt4BbA58BHmHhb89im1aVV1aPXzdZTh2w4BH5p8cxY3qaGS7kI/AtYBbic77GIVJmkJsF8m52fDAsNW35/F4ncT6XHbXm0Sf05Em9i+V9LBZHJ+qiS9hnIy19aSumuhb8CMlp2e6nLAiFWdpHcBdwCfo6v87Cxus76/krQhZcHEB4G3dX3ptln9f0jijpgiSb0qY9r2zG2zjtmRxB0R0TI5HTxiiqpzL18D7FZd+gbllPG7F3xTrPLS446YompfwxpAp1j//sBy26+aXqti1iVxR0yRpKvmb/DodS2iWwrhR0zXcknbdF5I2hpYPsX2RAtkjDtiug4Hzpd0I2VD2lbAgdNtUsy6DJVETJmktYDtKIn7+lQGjMUkcUdMkaS1Kbv2dqUUOPoWcKztO6fasJhpSdwRUyTpNOA24JTq0n7Axrb3nV6rYtYlcUdMUVaVRB1ZVRIxXVdI2qXzQtLOzGhho5gd6XFHTJGk6ygTkz+uLm1JqQt9L6VmycyVFI3pS+KOmCJJW/X7uu2bJ9WWaI8MlURMUZWYtwD2qH79e2CJ7ZuTtGMh6XFHTJGkI4EnAtvZfqSkzYDTbT9tyk2LGZYed8R07QM8j+oQBds/o5y8ErGgJO6I6brL5bHXAJLWm3J7ogWSuCOm6zRJxwEbSToIOBc4YcptihmXMe6IKZO0F/BMSq2Ss20vm3KTYsYlcUfMEEmrAS+2nZPfY0EZKomYAklLJR0h6WhJz1TxOuBG4EXTbl/MtvS4I6ZA0pnAr4GLgT2BjYE1gUNtXznNtsXsS+KOmAJJV9t+bPXr1YBbgC1t3zbdlkUbZKgkYjruO8Xd9nLgpiTtGFR63BFTIGk51aYbymqSdYDbq1/b9tJptS1mXxJ3RETLZKgkIqJlkrgjIlomiTsiomWSuCMiWiaJOyKiZZK4IyJa5v8Du7CgC7EIQAkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.heatmap(merged_data.isnull(), yticklabels=False, cbar=False, cmap='viridis')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define test / training sets;  encode categorical values; process text field"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Through train test split. Test proportion:\n",
      "0.01\n"
     ]
    }
   ],
   "source": [
    "# get training and test data set\n",
    "\n",
    "train, test = train_test_split(merged_data, test_size = testproportion)\n",
    "print(\"Through train test split. Test proportion:\")\n",
    "print(testproportion)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define feature categories <a name='definecategories' />\n",
    "<a href=#linkanchor>Back to link list</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "all cols ['Report Date', 'Route', 'Time', 'Day', 'Location', 'Incident', 'Min Delay', 'Min Gap', 'Direction', 'Vehicle', 'Report Date Time', 'year', 'month', 'daym', 'hour', 'time_of_day', 'target']\n"
     ]
    }
   ],
   "source": [
    "allcols = list(merged_data)\n",
    "print(\"all cols\",allcols)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "allcols 17\n",
      "excludefromcolist 8\n",
      "['Report Date', 'Time', 'Report Date Time', 'Min Gap', 'Min Delay', 'target', 'time_of_day', 'Vehicle']\n",
      "textcols 1\n",
      "['Incident']\n",
      "continuouscols 0\n",
      "[]\n",
      "collist 8\n",
      "['Day', 'hour', 'Direction', 'Location', 'daym', 'year', 'month', 'Route']\n"
     ]
    }
   ],
   "source": [
    "# define the required column lists\n",
    "\n",
    "textcols = ['Incident'] # columns to deal with as text - replace entries with multiple IDs and use embeddings, RNN\n",
    "continuouscols = [] # columns to deal with as continuous values - no embeddings\n",
    "if targetcontinuous:\n",
    "    excludefromcolist = ['Report Date','Time','Report Date Time','Min Gap','target','time_of_day','Vehicle'] # columns to exclude completely from the model\n",
    "else:\n",
    "    # if target column is not renamed Min Delay put Min Delay in exclusion list\n",
    "    excludefromcolist = ['Report Date','Time','Report Date Time','Min Gap','Min Delay','target','time_of_day','Vehicle'] # columns to exclude completely from the model\n",
    "# list(set(temp1) - set(temp2))\n",
    "nontextcols = list(set(allcols) - set(textcols))\n",
    "collist = list(set(nontextcols) - set(excludefromcolist) - set(continuouscols))\n",
    "\n",
    "# print column list lengths and contents:\n",
    "print(\"allcols\",len(allcols))\n",
    "print(\"excludefromcolist\",len(excludefromcolist))\n",
    "print(excludefromcolist)\n",
    "print(\"textcols\",len(textcols))\n",
    "print(textcols)\n",
    "print(\"continuouscols\",len(continuouscols))\n",
    "print(continuouscols)\n",
    "print(\"collist\",len(collist))\n",
    "print(collist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ensure continuous columns have numeric type\n",
    "for col in continuouscols:\n",
    "    print(\"col is\",col)\n",
    "    merged_data[col] = merged_data[col].astype(float)\n",
    "    print(\"got through one\")\n",
    "    superset_data[col] = superset_data[col].astype(float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Report Date         datetime64[ns]\n",
       "Route                       object\n",
       "Time                        object\n",
       "Day                         object\n",
       "Location                    object\n",
       "Incident                    object\n",
       "Min Delay                  float64\n",
       "Min Gap                    float64\n",
       "Direction                   object\n",
       "Vehicle                     object\n",
       "Report Date Time    datetime64[ns]\n",
       "year                         int64\n",
       "month                        int64\n",
       "daym                         int64\n",
       "hour                         int64\n",
       "time_of_day                 object\n",
       "target                       int64\n",
       "dtype: object"
      ]
     },
     "execution_count": 73,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_data.dtypes"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Deal with missing values <a name='bookmark' />\n",
    "<a href=#linkanchor>Back to link list</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fill missing values\n",
    "\n",
    "def fill_missing(dataset):\n",
    "    for col in collist:\n",
    "        dataset[col].fillna(value=\"missing\", inplace=True)\n",
    "    for col in continuouscols:\n",
    "        dataset[col].fillna(value=0.0,inplace=True)\n",
    "    for col in textcols:\n",
    "        dataset[col].fillna(value=\"missing\", inplace=True)\n",
    "    return (dataset)\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing  Day\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/envs/fastai/lib/python3.6/site-packages/ipykernel_launcher.py:15: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  from ipykernel import kernelapp as app\n",
      "/opt/conda/envs/fastai/lib/python3.6/site-packages/ipykernel_launcher.py:16: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  app.launch_new_instance()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing  hour\n",
      "processing  Direction\n",
      "processing  Location\n",
      "processing  daym\n",
      "processing  year\n",
      "processing  month\n",
      "processing  Route\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Report Date</th>\n",
       "      <th>Route</th>\n",
       "      <th>Time</th>\n",
       "      <th>Day</th>\n",
       "      <th>Location</th>\n",
       "      <th>Incident</th>\n",
       "      <th>Min Delay</th>\n",
       "      <th>Min Gap</th>\n",
       "      <th>Direction</th>\n",
       "      <th>Vehicle</th>\n",
       "      <th>Report Date Time</th>\n",
       "      <th>year</th>\n",
       "      <th>month</th>\n",
       "      <th>daym</th>\n",
       "      <th>hour</th>\n",
       "      <th>time_of_day</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Report Date Time</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2018-09-10 06:10:00</th>\n",
       "      <td>2018-09-10</td>\n",
       "      <td>4</td>\n",
       "      <td>06:10:00</td>\n",
       "      <td>1</td>\n",
       "      <td>3469</td>\n",
       "      <td>Late Leaving Garage</td>\n",
       "      <td>5.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>54</td>\n",
       "      <td>4236</td>\n",
       "      <td>2018-09-10 06:10:00</td>\n",
       "      <td>4</td>\n",
       "      <td>8</td>\n",
       "      <td>9</td>\n",
       "      <td>6</td>\n",
       "      <td>morning_rush</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016-10-02 08:25:00</th>\n",
       "      <td>2016-10-02</td>\n",
       "      <td>4</td>\n",
       "      <td>08:25:00</td>\n",
       "      <td>3</td>\n",
       "      <td>3469</td>\n",
       "      <td>Mechanical</td>\n",
       "      <td>7.0</td>\n",
       "      <td>14.0</td>\n",
       "      <td>35</td>\n",
       "      <td>4201</td>\n",
       "      <td>2016-10-02 08:25:00</td>\n",
       "      <td>2</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>wmorning_rush</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-01-12 07:26:00</th>\n",
       "      <td>2015-01-12</td>\n",
       "      <td>14</td>\n",
       "      <td>07:26:00</td>\n",
       "      <td>1</td>\n",
       "      <td>7072</td>\n",
       "      <td>Mechanical</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>35</td>\n",
       "      <td>4089</td>\n",
       "      <td>2015-01-12 07:26:00</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>7</td>\n",
       "      <td>morning_rush</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2014-11-27 15:04:00</th>\n",
       "      <td>2014-11-27</td>\n",
       "      <td>4</td>\n",
       "      <td>15:04:00</td>\n",
       "      <td>4</td>\n",
       "      <td>7953</td>\n",
       "      <td>Mechanical</td>\n",
       "      <td>6.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>35</td>\n",
       "      <td>4228</td>\n",
       "      <td>2014-11-27 15:04:00</td>\n",
       "      <td>0</td>\n",
       "      <td>10</td>\n",
       "      <td>26</td>\n",
       "      <td>15</td>\n",
       "      <td>aft_rush</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2018-03-05 09:37:00</th>\n",
       "      <td>2018-03-05</td>\n",
       "      <td>13</td>\n",
       "      <td>09:37:00</td>\n",
       "      <td>1</td>\n",
       "      <td>9677</td>\n",
       "      <td>Investigation</td>\n",
       "      <td>11.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>35</td>\n",
       "      <td>4103</td>\n",
       "      <td>2018-03-05 09:37:00</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>9</td>\n",
       "      <td>morning_rush</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    Report Date  Route      Time  Day  Location  \\\n",
       "Report Date Time                                                  \n",
       "2018-09-10 06:10:00  2018-09-10      4  06:10:00    1      3469   \n",
       "2016-10-02 08:25:00  2016-10-02      4  08:25:00    3      3469   \n",
       "2015-01-12 07:26:00  2015-01-12     14  07:26:00    1      7072   \n",
       "2014-11-27 15:04:00  2014-11-27      4  15:04:00    4      7953   \n",
       "2018-03-05 09:37:00  2018-03-05     13  09:37:00    1      9677   \n",
       "\n",
       "                                Incident  Min Delay  Min Gap  Direction  \\\n",
       "Report Date Time                                                          \n",
       "2018-09-10 06:10:00  Late Leaving Garage        5.0      8.0         54   \n",
       "2016-10-02 08:25:00           Mechanical        7.0     14.0         35   \n",
       "2015-01-12 07:26:00           Mechanical       13.0      0.0         35   \n",
       "2014-11-27 15:04:00           Mechanical        6.0     12.0         35   \n",
       "2018-03-05 09:37:00        Investigation       11.0      6.0         35   \n",
       "\n",
       "                    Vehicle    Report Date Time  year  month  daym  hour  \\\n",
       "Report Date Time                                                           \n",
       "2018-09-10 06:10:00    4236 2018-09-10 06:10:00     4      8     9     6   \n",
       "2016-10-02 08:25:00    4201 2016-10-02 08:25:00     2      9     1     8   \n",
       "2015-01-12 07:26:00    4089 2015-01-12 07:26:00     1      0    11     7   \n",
       "2014-11-27 15:04:00    4228 2014-11-27 15:04:00     0     10    26    15   \n",
       "2018-03-05 09:37:00    4103 2018-03-05 09:37:00     4      2     4     9   \n",
       "\n",
       "                       time_of_day  target  \n",
       "Report Date Time                            \n",
       "2018-09-10 06:10:00   morning_rush       0  \n",
       "2016-10-02 08:25:00  wmorning_rush       1  \n",
       "2015-01-12 07:26:00   morning_rush       1  \n",
       "2014-11-27 15:04:00       aft_rush       1  \n",
       "2018-03-05 09:37:00   morning_rush       1  "
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# process categorical data\n",
    "# replace values with category IDs in the categorical columns\n",
    "# \n",
    "\n",
    "\n",
    "lelist = {}\n",
    "\n",
    "for col in collist:\n",
    "    le = LabelEncoder()\n",
    "    if verboseout:\n",
    "        print(\"processing \",col)\n",
    "    # fit on superset to ensure encoding includes future data but only model on past data\n",
    "    # np.hstack([superset_data[col]])\n",
    "    le.fit(np.hstack([merged_data[col]]))\n",
    "    train[col] = le.transform(train[col])\n",
    "    test[col] = le.transform(test[col])\n",
    "    lelist[col] = le\n",
    "    del le\n",
    "    \n",
    "    \n",
    "train.head(5)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Report Date Time\n",
       "2018-09-10 06:10:00    Late Leaving Garage\n",
       "2016-10-02 08:25:00             Mechanical\n",
       "2015-01-12 07:26:00             Mechanical\n",
       "2014-11-27 15:04:00             Mechanical\n",
       "2018-03-05 09:37:00          Investigation\n",
       "2017-08-03 21:20:00             Mechanical\n",
       "2018-07-11 04:25:00             Mechanical\n",
       "2017-10-19 18:00:00              Diversion\n",
       "2016-06-16 08:27:00          General Delay\n",
       "2014-10-22 13:49:00          Investigation\n",
       "Name: Incident, dtype: object"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['Incident'].head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "processing text col Incident\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/envs/fastai/lib/python3.6/site-packages/ipykernel_launcher.py:17: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "/opt/conda/envs/fastai/lib/python3.6/site-packages/ipykernel_launcher.py:18: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
     ]
    }
   ],
   "source": [
    "# part 1 of text processing\n",
    "# tokenize list of text columns (made up of multiple strings)\n",
    "\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "\n",
    "# text columns that we care about\n",
    "# textcols = ['Other_Reason_for_Cancellation', 'Reason_for_Reopening', 'Resolution_Description', 'Subject']\n",
    "# textcols = ['Resolution_Description', 'Subject']\n",
    "\n",
    "for col in textcols:\n",
    "    if verboseout:\n",
    "        print(\"processing text col\",col)\n",
    "    \n",
    "    # Tokenizer lower cases and removes punctuation by default\n",
    "    tok_raw = Tokenizer(num_words=maxwords,lower=True)\n",
    "    tok_raw.fit_on_texts(train[col])\n",
    "    train[col] = tok_raw.texts_to_sequences(train[col])\n",
    "    test[col] = tok_raw.texts_to_sequences(test[col])\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Report Date Time\n",
       "2018-09-10 06:10:00    [5, 6, 7]\n",
       "2016-10-02 08:25:00          [1]\n",
       "2015-01-12 07:26:00          [1]\n",
       "2014-11-27 15:04:00          [1]\n",
       "2018-03-05 09:37:00          [2]\n",
       "2017-08-03 21:20:00          [1]\n",
       "2018-07-11 04:25:00          [1]\n",
       "2017-10-19 18:00:00         [15]\n",
       "2016-06-16 08:27:00       [8, 9]\n",
       "2014-10-22 13:49:00          [2]\n",
       "Name: Incident, dtype: object"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['Incident'].head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "maxtrain.max() 17\n",
      "maxtest .max() 17\n",
      "textmax 7000\n",
      "max_dict {'Day': 7, 'hour': 24, 'Direction': 95, 'Location': 10074, 'daym': 31, 'year': 6, 'month': 12, 'Route': 15, 'Incident': 17}\n"
     ]
    }
   ],
   "source": [
    "# define max values for embeddings\n",
    "\n",
    "\n",
    "max_dict = {}\n",
    "textmax = 50\n",
    "\n",
    "for col in collist:\n",
    "    max_dict[col] = np.max([train[col].max(), test[col].max()])+1\n",
    "    \n",
    "# np.max([np.max(train['Resolution_Description'].max()), np.max(train['Subject'].max()),np.max(train['Other_Reason_for_Cancellation'].max()),np.max(train['Reason_for_Reopening'].max())])) \n",
    "for cols in textcols:\n",
    "    maxtrain = np.max(train[(train[cols].map(len) != 0)][cols].map(max))\n",
    "    maxtest = np.max(test[(test[cols].map(len) != 0)][cols].map(max))\n",
    "    if verboseout:\n",
    "        print(\"maxtrain.max()\",maxtrain)\n",
    "        print(\"maxtest .max()\",maxtest)\n",
    "    max_dict[cols] = max(maxtrain,maxtest)\n",
    "    # max_dict[cols] = np.max([np.max(train[cols].max()), np.max(test[cols].max())])+20\n",
    "    # np.max(df[(df['seq_item_description'].map(len) != 0)].seq_item_description.map(max))\n",
    "    if max_dict[cols] > textmax:\n",
    "        textmax = max_dict[cols]\n",
    "\n",
    "if textmax < hctextmax:\n",
    "    textmax = hctextmax\n",
    "        \n",
    "print(\"textmax\",textmax)\n",
    "                             \n",
    "if verboseout:\n",
    "    print(\"max_dict\",max_dict)\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Split training set into train / validate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(66024, 17)\n",
      "(16507, 17)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/envs/fastai/lib/python3.6/site-packages/sklearn/model_selection/_split.py:2179: FutureWarning: From version 0.21, test_size will always complement train_size unless both are specified.\n",
      "  FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "# define and scale target and get validation sets\n",
    "\n",
    "dtrain, dvalid = train_test_split(train, random_state=123, train_size=trainproportion)\n",
    "\n",
    "print(dtrain.shape)\n",
    "print(dvalid.shape)\n",
    "if verboseout:\n",
    "    train[\"target\"].head()\n",
    "\n",
    "    \n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cat col is Day\n",
      "cat col is hour\n",
      "cat col is Direction\n",
      "cat col is Location\n",
      "cat col is daym\n",
      "cat col is year\n",
      "cat col is month\n",
      "cat col is Route\n",
      "text col is Incident\n",
      "cat col is Day\n",
      "cat col is hour\n",
      "cat col is Direction\n",
      "cat col is Location\n",
      "cat col is daym\n",
      "cat col is year\n",
      "cat col is month\n",
      "cat col is Route\n",
      "text col is Incident\n",
      "cat col is Day\n",
      "cat col is hour\n",
      "cat col is Direction\n",
      "cat col is Location\n",
      "cat col is daym\n",
      "cat col is year\n",
      "cat col is month\n",
      "cat col is Route\n",
      "text col is Incident\n",
      "keras variables defined\n",
      "X_train_list [array([4, 5, 2, ..., 5, 6, 1]), array([ 7,  8,  8, ..., 20,  5,  9]), array([35, 35, 43, ..., 54, 54, 35]), array([2618, 4196, 2340, ..., 8400, 1544, 2476]), array([26,  4, 12, ...,  3,  0, 17]), array([3, 2, 2, ..., 0, 0, 2]), array([ 3,  6,  1, ..., 10,  9,  0]), array([ 9,  7,  8, ...,  4, 13,  8]), array([[ 0,  0,  0, ...,  0,  3,  4],\n",
      "       [ 0,  0,  0, ...,  0,  0,  1],\n",
      "       [ 0,  0,  0, ...,  0,  0,  2],\n",
      "       ...,\n",
      "       [ 0,  0,  0, ..., 10, 11, 12],\n",
      "       [ 0,  0,  0, ...,  5,  6,  7],\n",
      "       [ 0,  0,  0, ...,  0,  3,  4]], dtype=int32)]\n"
     ]
    }
   ],
   "source": [
    "# define keras variables\n",
    "from keras.preprocessing.sequence import pad_sequences\n",
    "\n",
    "# X for the features used\n",
    "\n",
    "def get_keras_vars(dataset):\n",
    "    X = {}\n",
    "    dictlist = []\n",
    "    for col in collist:\n",
    "        if verboseout:\n",
    "            print(\"cat col is\",col)\n",
    "        X[col] = np.array(dataset[col])\n",
    "        dictlist.append(np.array(dataset[col]))\n",
    "       \n",
    "    for col in textcols:\n",
    "        if verboseout:\n",
    "            print(\"text col is\",col)\n",
    "        X[col] = pad_sequences(dataset[col], maxlen=max_dict[col])\n",
    "        dictlist.append(pad_sequences(dataset[col], maxlen=max_dict[col]))\n",
    "        \n",
    "    for col in continuouscols:\n",
    "        if verboseout:\n",
    "            print(\"cont col is\",col)\n",
    "        X[col] = np.array(dataset[col])\n",
    "        dictlist.append(np.array(dataset[col]))\n",
    "        \n",
    "    return X, dictlist\n",
    "\n",
    "def get_keras_list_only(X_in):\n",
    "    dictlist = []\n",
    "    for key, value in X_in.items():\n",
    "        print(\"X def loop key\",key)\n",
    "        print(\"value shape\",value.shape)\n",
    "        temp = [key,value]\n",
    "        dictlist.append(value)\n",
    "    return dictlist\n",
    "\n",
    "def get_keras_np(X_in):\n",
    "    return np.array(list(X_in.items()),dtype=object)\n",
    "# np.array(list(result.items()), dtype=dtype)\n",
    "\n",
    "# the deployment API for Watson Studio can only take a list/array, not a dictionary, so define list-only version for input\n",
    "X_train, X_train_list = get_keras_vars(dtrain)\n",
    "X_valid, X_valid_list = get_keras_vars(dvalid)\n",
    "X_test,X_test_list = get_keras_vars(test)\n",
    "print(\"keras variables defined\")\n",
    "print(\"X_train_list\",X_train_list)\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Define and fit model <a name='modelfit' />\n",
    "<a href=#linkanchor>Back to link list</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "about to define embeddings\n",
      "textmax is 7000\n",
      "col Incident\n",
      "text input shape 17\n",
      "max in the midst 27\n",
      "through loops for cols\n",
      "through definition of non-text parts of main_l\n",
      "main_l Tensor(\"concatenate_24/concat:0\", shape=(?, ?), dtype=float32)\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "Day (InputLayer)                (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "hour (InputLayer)               (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_19 (Embedding)        (None, 1, 10)        70          Day[0][0]                        \n",
      "__________________________________________________________________________________________________\n",
      "embedding_20 (Embedding)        (None, 1, 10)        240         hour[0][0]                       \n",
      "__________________________________________________________________________________________________\n",
      "Direction (InputLayer)          (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_19 (BatchNo (None, 1, 10)        40          embedding_19[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_20 (BatchNo (None, 1, 10)        40          embedding_20[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "embedding_21 (Embedding)        (None, 1, 10)        950         Direction[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "Location (InputLayer)           (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "flatten_17 (Flatten)            (None, 10)           0           batch_normalization_19[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "flatten_18 (Flatten)            (None, 10)           0           batch_normalization_20[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_21 (BatchNo (None, 1, 10)        40          embedding_21[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "embedding_22 (Embedding)        (None, 1, 10)        100740      Location[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "daym (InputLayer)               (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_20 (Dropout)            (None, 10)           0           flatten_17[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dropout_21 (Dropout)            (None, 10)           0           flatten_18[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_19 (Flatten)            (None, 10)           0           batch_normalization_21[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_22 (BatchNo (None, 1, 10)        40          embedding_22[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "embedding_23 (Embedding)        (None, 1, 10)        310         daym[0][0]                       \n",
      "__________________________________________________________________________________________________\n",
      "year (InputLayer)               (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_17 (Concatenate)    (None, 20)           0           dropout_20[0][0]                 \n",
      "                                                                 dropout_21[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dropout_22 (Dropout)            (None, 10)           0           flatten_19[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_20 (Flatten)            (None, 10)           0           batch_normalization_22[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_23 (BatchNo (None, 1, 10)        40          embedding_23[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "embedding_24 (Embedding)        (None, 1, 10)        60          year[0][0]                       \n",
      "__________________________________________________________________________________________________\n",
      "month (InputLayer)              (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_18 (Concatenate)    (None, 30)           0           concatenate_17[0][0]             \n",
      "                                                                 dropout_22[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dropout_23 (Dropout)            (None, 10)           0           flatten_20[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_21 (Flatten)            (None, 10)           0           batch_normalization_23[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_24 (BatchNo (None, 1, 10)        40          embedding_24[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "embedding_25 (Embedding)        (None, 1, 10)        120         month[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "Route (InputLayer)              (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_19 (Concatenate)    (None, 40)           0           concatenate_18[0][0]             \n",
      "                                                                 dropout_23[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dropout_24 (Dropout)            (None, 10)           0           flatten_21[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_22 (Flatten)            (None, 10)           0           batch_normalization_24[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_25 (BatchNo (None, 1, 10)        40          embedding_25[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "embedding_26 (Embedding)        (None, 1, 10)        150         Route[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "Incident (InputLayer)           (None, 17)           0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_20 (Concatenate)    (None, 50)           0           concatenate_19[0][0]             \n",
      "                                                                 dropout_24[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dropout_25 (Dropout)            (None, 10)           0           flatten_22[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_23 (Flatten)            (None, 10)           0           batch_normalization_25[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_26 (BatchNo (None, 1, 10)        40          embedding_26[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "embedding_27 (Embedding)        (None, 17, 50)       350000      Incident[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_21 (Concatenate)    (None, 60)           0           concatenate_20[0][0]             \n",
      "                                                                 dropout_25[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dropout_26 (Dropout)            (None, 10)           0           flatten_23[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_24 (Flatten)            (None, 10)           0           batch_normalization_26[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_27 (BatchNo (None, 17, 50)       200         embedding_27[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_22 (Concatenate)    (None, 70)           0           concatenate_21[0][0]             \n",
      "                                                                 dropout_26[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dropout_27 (Dropout)            (None, 10)           0           flatten_24[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "gru_3 (GRU)                     (None, 16)           3216        batch_normalization_27[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_23 (Concatenate)    (None, 80)           0           concatenate_22[0][0]             \n",
      "                                                                 dropout_27[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dropout_19 (Dropout)            (None, 16)           0           gru_3[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_24 (Concatenate)    (None, 96)           0           concatenate_23[0][0]             \n",
      "                                                                 dropout_19[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dense_3 (Dense)                 (None, 1)            97          concatenate_24[0][0]             \n",
      "==================================================================================================\n",
      "Total params: 456,473\n",
      "Trainable params: 456,213\n",
      "Non-trainable params: 260\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# define model in Keras\n",
    "\n",
    "def get_model():\n",
    "    \n",
    "          \n",
    "    catinputs = {} # list of categorical inputs\n",
    "    textinputs = {} # list of text inputs\n",
    "    continputs = {} # list of continuous inputs\n",
    "    embeddings = {}\n",
    "    textembeddings = {}\n",
    "    catemb = 10 # size of categorical embeddings\n",
    "    textemb = 50 # size of text embeddings\n",
    "    \n",
    "      \n",
    "    print(\"about to define embeddings\")\n",
    "    collistfix = []\n",
    "    textlayerlist = []\n",
    "    inputlayerlist = []\n",
    "    i = 0\n",
    "    print(\"textmax is\",textmax)\n",
    "    # define layers for categorical columns\n",
    "    for col in collist:\n",
    "        catinputs[col] = Input(shape=[1],name=col)\n",
    "        inputlayerlist.append(catinputs[col])\n",
    "        #print(\"inputname\",inputname)\n",
    "        embeddings[col] = (Embedding(max_dict[col],catemb) (catinputs[col]))\n",
    "        # batchnorm all \n",
    "        embeddings[col] = (BatchNormalization() (embeddings[col]))\n",
    "        collistfix.append(embeddings[col])\n",
    "    \n",
    "    \n",
    "    \n",
    "    # define layers for text columns\n",
    "    if includetext:    \n",
    "        for col in textcols:\n",
    "            print(\"col\",col)\n",
    "            textinputs[col] = Input(shape=[X_train[col].shape[1]], name=col)\n",
    "            print(\"text input shape\",X_train[col].shape[1])\n",
    "            inputlayerlist.append(textinputs[col])\n",
    "            textembeddings[col] = (Embedding(textmax,textemb) (textinputs[col]))\n",
    "            textembeddings[col] = (BatchNormalization() (textembeddings[col])) \n",
    "            textembeddings[col] = Dropout(dropout_rate) ( GRU(16,kernel_regularizer=l2(l2_lambda)) (textembeddings[col]))\n",
    "            collistfix.append(textembeddings[col])\n",
    "            print(\"max in the midst\",np.max([np.max(train[col].max()), np.max(test[col].max())])+10)\n",
    "        print(\"through loops for cols\")\n",
    "        \n",
    "    # define layers for continuous columns\n",
    "    for col in continuouscols: \n",
    "        continputs[col] = Input(shape=[1],name=col)\n",
    "        inputlayerlist.append(continputs[col])\n",
    "    \n",
    "    \n",
    "              \n",
    "    # build up layers\n",
    "    # main_l = concatenate([Dropout(dropout_rate) (Flatten() (embeddings['Vehicle']) ),Dropout(dropout_rate) (Flatten() (embeddings['Direction']) )])\n",
    "    main_l = concatenate([Dropout(dropout_rate) (Flatten() (embeddings[collist[0]]) ),Dropout(dropout_rate) (Flatten() (embeddings[collist[1]]) )])\n",
    "    for cols in collist:\n",
    "        if (cols != collist[0]) & (cols != collist[1]):\n",
    "            main_l = concatenate([main_l,Dropout(dropout_rate) (Flatten() (embeddings[cols]) )])\n",
    "    \n",
    "    print(\"through definition of non-text parts of main_l\")\n",
    "    if includetext:\n",
    "        for col in textcols:\n",
    "            main_l = concatenate([main_l,textembeddings[col]])\n",
    "                                                 \n",
    "    for col in continuouscols:\n",
    "        main_l = concatenate([main_l,continputs[col]])\n",
    "    \n",
    "    print(\"main_l\", main_l)                                            \n",
    "    \n",
    "       \n",
    "    \n",
    "    # define output layer\n",
    "    output = Dense(1, activation=output_activation) (main_l)\n",
    "   \n",
    "    # define model        \n",
    "                                                  \n",
    "    model = Model(inputlayerlist, output)\n",
    "    \n",
    "       \n",
    "    # define optimizer\n",
    "    optimizer = SGD(lr=learning_rate)\n",
    "   \n",
    "    # compile model\n",
    "    model.compile(loss=loss_func, optimizer=optimizer, metrics=[\"accuracy\"], weighted_metrics=[\"accuracy\"])\n",
    "         \n",
    "    \n",
    "    return model\n",
    "\n",
    "    \n",
    "model = get_model()\n",
    "\n",
    "# FIX THIS\n",
    "# plot_model(model, to_file='/home/paperspace/visualizations/streetcarmodel_dec16.png', show_shapes=True, show_layer_names=True)\n",
    "\n",
    "# output model summary\n",
    "\n",
    "model.summary()\n",
    "    \n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "text cols ['Incident']\n",
      "dropout  0.0003\n",
      "L2 lambda  0.0003\n",
      "batch size  1000\n",
      "epochs 50\n",
      "learning_rate 0.001\n",
      "loss function binary_crossentropy\n",
      "output activation function hard_sigmoid\n",
      "about to define embeddings\n",
      "textmax is 7000\n",
      "col Incident\n",
      "text input shape 17\n",
      "max in the midst 27\n",
      "through loops for cols\n",
      "through definition of non-text parts of main_l\n",
      "main_l Tensor(\"concatenate_32/concat:0\", shape=(?, ?), dtype=float32)\n",
      "Train on 66024 samples, validate on 16507 samples\n",
      "Epoch 1/50\n",
      "66024/66024 [==============================] - 7s 112us/step - loss: 0.7508 - acc: 0.5341 - weighted_acc: 0.5341 - val_loss: 0.7206 - val_acc: 0.5446 - val_weighted_acc: 0.5446\n",
      "Epoch 2/50\n",
      "66024/66024 [==============================] - 3s 43us/step - loss: 0.7066 - acc: 0.5750 - weighted_acc: 0.5750 - val_loss: 0.6933 - val_acc: 0.5810 - val_weighted_acc: 0.5810\n",
      "Epoch 3/50\n",
      "66024/66024 [==============================] - 3s 42us/step - loss: 0.6822 - acc: 0.6058 - weighted_acc: 0.6058 - val_loss: 0.6735 - val_acc: 0.6076 - val_weighted_acc: 0.6076\n",
      "Epoch 4/50\n",
      "66024/66024 [==============================] - 3s 38us/step - loss: 0.6655 - acc: 0.6278 - weighted_acc: 0.6278 - val_loss: 0.6610 - val_acc: 0.6294 - val_weighted_acc: 0.6294\n",
      "Epoch 5/50\n",
      "66024/66024 [==============================] - 3s 43us/step - loss: 0.6535 - acc: 0.6439 - weighted_acc: 0.6439 - val_loss: 0.6493 - val_acc: 0.6468 - val_weighted_acc: 0.6468\n",
      "Epoch 6/50\n",
      "66024/66024 [==============================] - 3s 43us/step - loss: 0.6448 - acc: 0.6559 - weighted_acc: 0.6559 - val_loss: 0.6403 - val_acc: 0.6594 - val_weighted_acc: 0.6594\n",
      "Epoch 7/50\n",
      "66024/66024 [==============================] - 3s 40us/step - loss: 0.6378 - acc: 0.6639 - weighted_acc: 0.6639 - val_loss: 0.6333 - val_acc: 0.6664 - val_weighted_acc: 0.6664\n",
      "Epoch 8/50\n",
      "66024/66024 [==============================] - 3s 41us/step - loss: 0.6324 - acc: 0.6697 - weighted_acc: 0.6697 - val_loss: 0.6277 - val_acc: 0.6726 - val_weighted_acc: 0.6726\n",
      "Epoch 9/50\n",
      "66024/66024 [==============================] - 3s 42us/step - loss: 0.6283 - acc: 0.6732 - weighted_acc: 0.6732 - val_loss: 0.6236 - val_acc: 0.6766 - val_weighted_acc: 0.6766\n",
      "Epoch 10/50\n",
      "66024/66024 [==============================] - 3s 39us/step - loss: 0.6241 - acc: 0.6775 - weighted_acc: 0.6775 - val_loss: 0.6201 - val_acc: 0.6803 - val_weighted_acc: 0.6803\n",
      "Epoch 11/50\n",
      "66024/66024 [==============================] - 3s 40us/step - loss: 0.6208 - acc: 0.6816 - weighted_acc: 0.6816 - val_loss: 0.6171 - val_acc: 0.6841 - val_weighted_acc: 0.6841\n",
      "Epoch 12/50\n",
      "66024/66024 [==============================] - 3s 42us/step - loss: 0.6179 - acc: 0.6852 - weighted_acc: 0.6852 - val_loss: 0.6160 - val_acc: 0.6866 - val_weighted_acc: 0.6866\n",
      "Epoch 13/50\n",
      "66024/66024 [==============================] - 3s 39us/step - loss: 0.6158 - acc: 0.6866 - weighted_acc: 0.6866 - val_loss: 0.6142 - val_acc: 0.6890 - val_weighted_acc: 0.6890\n",
      "Epoch 14/50\n",
      "66024/66024 [==============================] - 3s 42us/step - loss: 0.6139 - acc: 0.6891 - weighted_acc: 0.6891 - val_loss: 0.6123 - val_acc: 0.6905 - val_weighted_acc: 0.6905\n",
      "Epoch 15/50\n",
      "66024/66024 [==============================] - 3s 42us/step - loss: 0.6122 - acc: 0.6907 - weighted_acc: 0.6907 - val_loss: 0.6107 - val_acc: 0.6933 - val_weighted_acc: 0.6933\n",
      "Epoch 16/50\n",
      "66024/66024 [==============================] - 3s 40us/step - loss: 0.6108 - acc: 0.6918 - weighted_acc: 0.6918 - val_loss: 0.6098 - val_acc: 0.6948 - val_weighted_acc: 0.6948\n",
      "Epoch 17/50\n",
      "66024/66024 [==============================] - 3s 42us/step - loss: 0.6095 - acc: 0.6931 - weighted_acc: 0.6931 - val_loss: 0.6094 - val_acc: 0.6959 - val_weighted_acc: 0.6959\n",
      "Epoch 18/50\n",
      "66024/66024 [==============================] - 3s 44us/step - loss: 0.6087 - acc: 0.6939 - weighted_acc: 0.6939 - val_loss: 0.6084 - val_acc: 0.6972 - val_weighted_acc: 0.6972\n",
      "Epoch 19/50\n",
      "66024/66024 [==============================] - 3s 46us/step - loss: 0.6073 - acc: 0.6949 - weighted_acc: 0.6949 - val_loss: 0.6078 - val_acc: 0.6991 - val_weighted_acc: 0.6991\n",
      "Epoch 20/50\n",
      "66024/66024 [==============================] - 3s 42us/step - loss: 0.6061 - acc: 0.6953 - weighted_acc: 0.6953 - val_loss: 0.6059 - val_acc: 0.6987 - val_weighted_acc: 0.6987\n",
      "Epoch 21/50\n",
      "66024/66024 [==============================] - 3s 44us/step - loss: 0.6049 - acc: 0.6967 - weighted_acc: 0.6967 - val_loss: 0.6064 - val_acc: 0.6994 - val_weighted_acc: 0.6994\n",
      "Epoch 22/50\n",
      "66024/66024 [==============================] - 3s 42us/step - loss: 0.6042 - acc: 0.6974 - weighted_acc: 0.6974 - val_loss: 0.6059 - val_acc: 0.7015 - val_weighted_acc: 0.7015\n",
      "Epoch 23/50\n",
      "66024/66024 [==============================] - 3s 42us/step - loss: 0.6035 - acc: 0.6973 - weighted_acc: 0.6973 - val_loss: 0.6076 - val_acc: 0.7018 - val_weighted_acc: 0.7018\n",
      "Epoch 24/50\n",
      "66024/66024 [==============================] - 3s 41us/step - loss: 0.6038 - acc: 0.6986 - weighted_acc: 0.6986 - val_loss: 0.6072 - val_acc: 0.7038 - val_weighted_acc: 0.7038\n",
      "Epoch 25/50\n",
      "66024/66024 [==============================] - 3s 42us/step - loss: 0.6025 - acc: 0.6986 - weighted_acc: 0.6986 - val_loss: 0.6053 - val_acc: 0.7040 - val_weighted_acc: 0.7040\n",
      "Epoch 26/50\n",
      "66024/66024 [==============================] - 3s 40us/step - loss: 0.6015 - acc: 0.6997 - weighted_acc: 0.6997 - val_loss: 0.6044 - val_acc: 0.7040 - val_weighted_acc: 0.7040\n",
      "Epoch 27/50\n",
      "66024/66024 [==============================] - 3s 40us/step - loss: 0.6008 - acc: 0.7001 - weighted_acc: 0.7001 - val_loss: 0.6035 - val_acc: 0.7032 - val_weighted_acc: 0.7032\n",
      "Epoch 28/50\n",
      "66024/66024 [==============================] - 3s 42us/step - loss: 0.6001 - acc: 0.7014 - weighted_acc: 0.7014 - val_loss: 0.6035 - val_acc: 0.7059 - val_weighted_acc: 0.7059\n",
      "Epoch 29/50\n",
      "66024/66024 [==============================] - 3s 42us/step - loss: 0.6005 - acc: 0.7015 - weighted_acc: 0.7015 - val_loss: 0.6030 - val_acc: 0.7047 - val_weighted_acc: 0.7047\n",
      "Epoch 30/50\n",
      "66024/66024 [==============================] - 3s 40us/step - loss: 0.5996 - acc: 0.7016 - weighted_acc: 0.7016 - val_loss: 0.6019 - val_acc: 0.7048 - val_weighted_acc: 0.7048\n",
      "Epoch 31/50\n",
      "66024/66024 [==============================] - 3s 42us/step - loss: 0.5985 - acc: 0.7022 - weighted_acc: 0.7022 - val_loss: 0.6015 - val_acc: 0.7062 - val_weighted_acc: 0.7062\n",
      "Epoch 32/50\n",
      "66024/66024 [==============================] - 3s 43us/step - loss: 0.5982 - acc: 0.7016 - weighted_acc: 0.7016 - val_loss: 0.6009 - val_acc: 0.7068 - val_weighted_acc: 0.7068\n",
      "Epoch 33/50\n",
      "66024/66024 [==============================] - 3s 40us/step - loss: 0.5975 - acc: 0.7021 - weighted_acc: 0.7021 - val_loss: 0.6003 - val_acc: 0.7079 - val_weighted_acc: 0.7079\n",
      "Epoch 34/50\n",
      "66024/66024 [==============================] - 3s 42us/step - loss: 0.5972 - acc: 0.7030 - weighted_acc: 0.7030 - val_loss: 0.6002 - val_acc: 0.7080 - val_weighted_acc: 0.7080\n",
      "Epoch 35/50\n",
      "66024/66024 [==============================] - 3s 42us/step - loss: 0.5980 - acc: 0.7020 - weighted_acc: 0.7020 - val_loss: 0.6044 - val_acc: 0.7045 - val_weighted_acc: 0.7045\n",
      "Epoch 36/50\n",
      "66024/66024 [==============================] - 3s 40us/step - loss: 0.5992 - acc: 0.7012 - weighted_acc: 0.7012 - val_loss: 0.6015 - val_acc: 0.7064 - val_weighted_acc: 0.7064\n",
      "Epoch 37/50\n",
      "66024/66024 [==============================] - 3s 43us/step - loss: 0.5977 - acc: 0.7032 - weighted_acc: 0.7032 - val_loss: 0.5999 - val_acc: 0.7082 - val_weighted_acc: 0.7082\n",
      "Epoch 38/50\n",
      "66024/66024 [==============================] - 3s 43us/step - loss: 0.5967 - acc: 0.7040 - weighted_acc: 0.7040 - val_loss: 0.5989 - val_acc: 0.7095 - val_weighted_acc: 0.7095\n",
      "Epoch 39/50\n",
      "66024/66024 [==============================] - 3s 39us/step - loss: 0.5964 - acc: 0.7044 - weighted_acc: 0.7044 - val_loss: 0.5988 - val_acc: 0.7106 - val_weighted_acc: 0.7106\n",
      "Epoch 40/50\n",
      "66024/66024 [==============================] - 3s 42us/step - loss: 0.5956 - acc: 0.7047 - weighted_acc: 0.7047 - val_loss: 0.5987 - val_acc: 0.7103 - val_weighted_acc: 0.7103\n",
      "Epoch 41/50\n",
      "66024/66024 [==============================] - 3s 42us/step - loss: 0.5956 - acc: 0.7043 - weighted_acc: 0.7043 - val_loss: 0.5980 - val_acc: 0.7090 - val_weighted_acc: 0.7090\n",
      "Epoch 42/50\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "66024/66024 [==============================] - 3s 41us/step - loss: 0.5954 - acc: 0.7047 - weighted_acc: 0.7047 - val_loss: 0.5982 - val_acc: 0.7104 - val_weighted_acc: 0.7104\n",
      "Epoch 43/50\n",
      "66024/66024 [==============================] - 3s 41us/step - loss: 0.5950 - acc: 0.7049 - weighted_acc: 0.7049 - val_loss: 0.5973 - val_acc: 0.7105 - val_weighted_acc: 0.7105\n",
      "Epoch 44/50\n",
      "66024/66024 [==============================] - 3s 43us/step - loss: 0.5940 - acc: 0.7051 - weighted_acc: 0.7051 - val_loss: 0.5981 - val_acc: 0.7105 - val_weighted_acc: 0.7105\n",
      "Epoch 45/50\n",
      "66024/66024 [==============================] - 3s 42us/step - loss: 0.5941 - acc: 0.7049 - weighted_acc: 0.7049 - val_loss: 0.5965 - val_acc: 0.7104 - val_weighted_acc: 0.7104\n",
      "Epoch 46/50\n",
      "66024/66024 [==============================] - 3s 42us/step - loss: 0.5991 - acc: 0.7042 - weighted_acc: 0.7042 - val_loss: 0.9950 - val_acc: 0.6192 - val_weighted_acc: 0.6192\n",
      "Epoch 47/50\n",
      "66024/66024 [==============================] - 3s 41us/step - loss: 0.7684 - acc: 0.6145 - weighted_acc: 0.6145 - val_loss: 0.7144 - val_acc: 0.6330 - val_weighted_acc: 0.6330\n",
      "Epoch 48/50\n",
      "66024/66024 [==============================] - 3s 42us/step - loss: 0.6721 - acc: 0.6356 - weighted_acc: 0.6356 - val_loss: 0.6561 - val_acc: 0.6473 - val_weighted_acc: 0.6473\n",
      "Epoch 49/50\n",
      "66024/66024 [==============================] - 3s 40us/step - loss: 0.6472 - acc: 0.6514 - weighted_acc: 0.6514 - val_loss: 0.6397 - val_acc: 0.6575 - val_weighted_acc: 0.6575\n",
      "Epoch 50/50\n",
      "66024/66024 [==============================] - 3s 43us/step - loss: 0.6338 - acc: 0.6635 - weighted_acc: 0.6635 - val_loss: 0.6299 - val_acc: 0.6683 - val_weighted_acc: 0.6683\n"
     ]
    }
   ],
   "source": [
    "BATCH_SIZE = 1000\n",
    "# 200\n",
    "epochs = 50\n",
    "print(\"text cols\",textcols)\n",
    "print(\"dropout \",dropout_rate)\n",
    "print(\"L2 lambda \",l2_lambda)\n",
    "print(\"batch size \",BATCH_SIZE)\n",
    "print(\"epochs\",epochs)\n",
    "print(\"learning_rate\",learning_rate)\n",
    "print(\"loss function\",loss_func)\n",
    "print(\"output activation function\",output_activation)\n",
    "\n",
    "\n",
    "model = get_model()\n",
    "modelfit = model.fit(X_train_list, dtrain.target, epochs=epochs, batch_size=BATCH_SIZE\n",
    "         , validation_data=(X_valid_list, dvalid.target), verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved model, weights, tokenizer and label encoder to disk\n",
      "Saved model, weights, tokenizer and label encoder to disk\n"
     ]
    }
   ],
   "source": [
    "modifier = \"mar23\"\n",
    "savepath = path\n",
    "if savemodel == True:\n",
    "    model_json = model.to_json()\n",
    "    with open(savepath+'model'+modifier+'.json', \"w\") as json_file:\n",
    "        json_file.write(model_json)\n",
    "    # serialize weights to HDF5\n",
    "    model.save_weights(savepath+'dmweights'+modifier+'.h5')\n",
    "    \n",
    "    # save tokenizer tok_raw for text and label encoder for categorical cols\n",
    "    with open(savepath+'tokenizer'+modifier+'.pickle', 'wb') as handle:\n",
    "        pickle.dump(tok_raw, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "    with open(savepath+'le'+modifier+'.pickle', 'wb') as handle2:\n",
    "        pickle.dump(lelist, handle2, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "    print(\"Saved model, weights, tokenizer and label encoder to disk\")\n",
    "      \n",
    "    model.save(savepath+'dmmodel'+modifier+'.h5')\n",
    "    print(\"Saved model, weights, tokenizer and label encoder to disk\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load saved model and weights <a name='reload' />\n",
    "<a href=#linkanchor>Back to link list</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import load_model\n",
    "new_model = load_model(savepath+'dmmodel'+modifier+'.h5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "if presaved == True:\n",
    "    BATCH_SIZE = 200\n",
    "    epochs = 1\n",
    "    modelfit2 = new_model.fit(X_train, dtrain.target, epochs=epochs, batch_size=BATCH_SIZE\n",
    "         , validation_data=(X_valid, dvalid.target),class_weight = {0 : zero_weight, 1: one_weight}, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "ACCOUNT_NAME (InputLayer)       (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "CASE_OWNER_ALIAS (InputLayer)   (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "embedding_22 (Embedding)        (None, 1, 10)        22150       ACCOUNT_NAME[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "embedding_13 (Embedding)        (None, 1, 10)        2690        CASE_OWNER_ALIAS[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "SUPPORT_MISSION (InputLayer)    (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_22 (BatchNo (None, 1, 10)        40          embedding_22[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_13 (BatchNo (None, 1, 10)        40          embedding_13[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "embedding_14 (Embedding)        (None, 1, 10)        10          SUPPORT_MISSION[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "SEVERITY_LEVEL_NUMBER_FORMULA ( (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "flatten_12 (Flatten)            (None, 10)           0           batch_normalization_22[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "flatten_13 (Flatten)            (None, 10)           0           batch_normalization_13[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_14 (BatchNo (None, 1, 10)        40          embedding_14[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "embedding_15 (Embedding)        (None, 1, 10)        40          SEVERITY_LEVEL_NUMBER_FORMULA[0][\n",
      "__________________________________________________________________________________________________\n",
      "ACCOUNT_PRIORITY (InputLayer)   (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_14 (Dropout)            (None, 10)           0           flatten_12[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dropout_15 (Dropout)            (None, 10)           0           flatten_13[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_14 (Flatten)            (None, 10)           0           batch_normalization_14[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_15 (BatchNo (None, 1, 10)        40          embedding_15[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "embedding_16 (Embedding)        (None, 1, 10)        190         ACCOUNT_PRIORITY[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "BLUE_DIAMOND_ACCOUNT (InputLaye (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_13 (Concatenate)    (None, 20)           0           dropout_14[0][0]                 \n",
      "                                                                 dropout_15[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dropout_16 (Dropout)            (None, 10)           0           flatten_14[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_15 (Flatten)            (None, 10)           0           batch_normalization_15[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_16 (BatchNo (None, 1, 10)        40          embedding_16[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "embedding_17 (Embedding)        (None, 1, 10)        20          BLUE_DIAMOND_ACCOUNT[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "OPERATING_SYSTEM (InputLayer)   (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_14 (Concatenate)    (None, 30)           0           concatenate_13[0][0]             \n",
      "                                                                 dropout_16[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dropout_17 (Dropout)            (None, 10)           0           flatten_15[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_16 (Flatten)            (None, 10)           0           batch_normalization_16[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_17 (BatchNo (None, 1, 10)        40          embedding_17[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "embedding_18 (Embedding)        (None, 1, 10)        60          OPERATING_SYSTEM[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "LEGACY_PROBLEM_NUMBER (InputLay (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_15 (Concatenate)    (None, 40)           0           concatenate_14[0][0]             \n",
      "                                                                 dropout_17[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dropout_18 (Dropout)            (None, 10)           0           flatten_16[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_17 (Flatten)            (None, 10)           0           batch_normalization_17[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_18 (BatchNo (None, 1, 10)        40          embedding_18[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "embedding_19 (Embedding)        (None, 1, 10)        20          LEGACY_PROBLEM_NUMBER[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "PRODUCT_VERSION (InputLayer)    (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_16 (Concatenate)    (None, 50)           0           concatenate_15[0][0]             \n",
      "                                                                 dropout_18[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dropout_19 (Dropout)            (None, 10)           0           flatten_17[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_18 (Flatten)            (None, 10)           0           batch_normalization_18[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_19 (BatchNo (None, 1, 10)        40          embedding_19[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "embedding_20 (Embedding)        (None, 1, 10)        430         PRODUCT_VERSION[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "CREATED_BY (InputLayer)         (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_17 (Concatenate)    (None, 60)           0           concatenate_16[0][0]             \n",
      "                                                                 dropout_19[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dropout_20 (Dropout)            (None, 10)           0           flatten_18[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_19 (Flatten)            (None, 10)           0           batch_normalization_19[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_20 (BatchNo (None, 1, 10)        40          embedding_20[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "embedding_21 (Embedding)        (None, 1, 10)        22760       CREATED_BY[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "PRODUCT_NAME (InputLayer)       (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_18 (Concatenate)    (None, 70)           0           concatenate_17[0][0]             \n",
      "                                                                 dropout_20[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dropout_21 (Dropout)            (None, 10)           0           flatten_19[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_20 (Flatten)            (None, 10)           0           batch_normalization_20[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_21 (BatchNo (None, 1, 10)        40          embedding_21[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "embedding_23 (Embedding)        (None, 1, 10)        280         PRODUCT_NAME[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "SUBJECT (InputLayer)            (None, 5999)         0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_19 (Concatenate)    (None, 80)           0           concatenate_18[0][0]             \n",
      "                                                                 dropout_21[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dropout_22 (Dropout)            (None, 10)           0           flatten_20[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_21 (Flatten)            (None, 10)           0           batch_normalization_21[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_23 (BatchNo (None, 1, 10)        40          embedding_23[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "embedding_24 (Embedding)        (None, 5999, 50)     350000      SUBJECT[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_20 (Concatenate)    (None, 90)           0           concatenate_19[0][0]             \n",
      "                                                                 dropout_22[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dropout_23 (Dropout)            (None, 10)           0           flatten_21[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "flatten_22 (Flatten)            (None, 10)           0           batch_normalization_23[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_24 (BatchNo (None, 5999, 50)     200         embedding_24[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_21 (Concatenate)    (None, 100)          0           concatenate_20[0][0]             \n",
      "                                                                 dropout_23[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dropout_24 (Dropout)            (None, 10)           0           flatten_22[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "gru_2 (GRU)                     (None, 16)           3216        batch_normalization_24[0][0]     \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_22 (Concatenate)    (None, 110)          0           concatenate_21[0][0]             \n",
      "                                                                 dropout_24[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "dropout_13 (Dropout)            (None, 16)           0           gru_2[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_23 (Concatenate)    (None, 126)          0           concatenate_22[0][0]             \n",
      "                                                                 dropout_13[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "OWNERSHIP_CHANGES (InputLayer)  (None, 1)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_24 (Concatenate)    (None, 127)          0           concatenate_23[0][0]             \n",
      "                                                                 OWNERSHIP_CHANGES[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 1)            128         concatenate_24[0][0]             \n",
      "==================================================================================================\n",
      "Total params: 402,634\n",
      "Trainable params: 402,314\n",
      "Non-trainable params: 320\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "new_model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Predictions and renderings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "# functions to parse and manipulate dates in the style of the input CSVs\n",
    "import datetime\n",
    "from datetime import date\n",
    "def create_date(year,month):\n",
    "    outdate = datetime.date(year,month,15)\n",
    "    return(outdate)\n",
    "\n",
    "def parse_bic_date(bic_date_in):\n",
    "    year = int(bic_date_in[0:4])\n",
    "    month = int(bic_date_in[-2:])\n",
    "    return(year,month)\n",
    "\n",
    "def create_date_from_bic(bic_date_in):\n",
    "    yr,mth = parse_bic_date(bic_date_in)\n",
    "    retdate = create_date(yr,mth)\n",
    "    return retdate\n",
    "\n",
    "def get_datecomp_from_csvdate (csv_date):\n",
    "    day = int(csv_date[0:2])\n",
    "    # month_number = datetime.datetime.strptime(month_name, '%b').month\n",
    "    month = datetime.datetime.strptime(csv_date[3:6], '%b').month\n",
    "    year = int('20'+csv_date[-2:])\n",
    "    # year = int(csv_date[-2:])\n",
    "    return (year,month,day)\n",
    "\n",
    "def get_date_from_csvdate (csv_date):\n",
    "    day = int(csv_date[0:2])\n",
    "    # month_number = datetime.datetime.strptime(month_name, '%b').month\n",
    "    month = datetime.datetime.strptime(csv_date[3:6], '%b',coerce=True).month\n",
    "    year = int('20'+csv_date[-2:])\n",
    "    # year = int(csv_date[-2:])\n",
    "    return (date(year,month,day))\n",
    "\n",
    "def get_year_from_csvdate (csv_date):\n",
    "    year = int('20'+csv_date[-2:])\n",
    "    return (year)\n",
    "\n",
    "def get_month_from_csvdate (csv_date):\n",
    "    month = datetime.datetime.strptime(csv_date[3:6], '%b',coerce=True).month\n",
    "    return (month)\n",
    "\n",
    "def get_day_from_csvdate (csv_date):\n",
    "    day = int(csv_date[0:2])\n",
    "    return (day)\n",
    "\n",
    "def get_weekday (date):\n",
    "    return(date.weekday())\n",
    "\n",
    "# pd.to_datetime(x, coerce=True)\n",
    "\n",
    "def validatedate(csv_text):\n",
    "    try:\n",
    "        datetime.datetime.strptime(csv_date[3:6], '%b')\n",
    "    except ValueError:\n",
    "        raise ValueError(\"Incorrect data format, should be YYYY-MM-DD\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "# predictions on training set\n",
    "\n",
    "preds = model.predict(X_train, batch_size=BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "11"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(lelist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5950, 1)"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.47658885],\n",
       "       [ 0.48568058],\n",
       "       [ 0.51618594],\n",
       "       [ 0.54550904],\n",
       "       [ 0.48846439],\n",
       "       [ 0.44110146],\n",
       "       [ 0.35933176],\n",
       "       [ 0.46696281],\n",
       "       [ 0.46008301],\n",
       "       [ 0.52748609],\n",
       "       [ 0.45919886],\n",
       "       [ 0.41886404],\n",
       "       [ 0.39999038],\n",
       "       [ 0.43974134],\n",
       "       [ 0.41980836],\n",
       "       [ 0.47716776],\n",
       "       [ 0.43132985],\n",
       "       [ 0.46688703],\n",
       "       [ 0.51388305],\n",
       "       [ 0.44410166],\n",
       "       [ 0.45134065],\n",
       "       [ 0.41294196],\n",
       "       [ 0.40390512],\n",
       "       [ 0.48919168],\n",
       "       [ 0.4630186 ],\n",
       "       [ 0.54636103],\n",
       "       [ 0.47538921],\n",
       "       [ 0.41760489],\n",
       "       [ 0.44419828],\n",
       "       [ 0.35426328],\n",
       "       [ 0.35809675],\n",
       "       [ 0.49904078],\n",
       "       [ 0.4187327 ],\n",
       "       [ 0.49405068],\n",
       "       [ 0.4368012 ],\n",
       "       [ 0.58154649],\n",
       "       [ 0.40500203],\n",
       "       [ 0.49542972],\n",
       "       [ 0.45565513],\n",
       "       [ 0.4841933 ],\n",
       "       [ 0.47077447],\n",
       "       [ 0.49711803],\n",
       "       [ 0.44285139],\n",
       "       [ 0.38223869],\n",
       "       [ 0.38384244],\n",
       "       [ 0.46742117],\n",
       "       [ 0.4218891 ],\n",
       "       [ 0.44025958],\n",
       "       [ 0.50178283],\n",
       "       [ 0.45705149]], dtype=float32)"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds[:50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/paperspace/anaconda3/envs/fastai/lib/python3.6/site-packages/ipykernel_launcher.py:1: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYEAAAD8CAYAAACRkhiPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAFlhJREFUeJzt3X2QXXV9x/H3p6BMJILY6G3cYBOmQUtYjc2WMvVhbgYtqQ8NWEdDGSHisFqo1ZlMa7APOmUyg63RDqXgrMIAo2WbEZGUBxWRK7XTiAkT2SSQGiDKrjEZZAZcyqQufvvH/cUc1r27N/fh3N39fV4zd/bc33n4/b45Bz57zzl7jyICMzPL02/0egBmZtY7DgEzs4w5BMzMMuYQMDPLmEPAzCxjDgEzs4w5BMzMMuYQMDPLmEPAzCxjx/d6ADNZtGhRLF26tOH8Z599lhNPPLG8Ac0SudYN+daea93g2lupfceOHU9GxCtmXDAipn0BpwL3AXuA3cBHU/vLgXuAH6afpxTWuQLYB+wFzi20rwJG0ryrAc3U/6pVq2I6991337Tz56tc647It/Zc645w7a0AtscM/3+NiKZOB00AGyLiDOBs4HJJZwAbgXsjYjlwb3pPmrcOWAGsAa6VdFza1nXApcDy9FrTRP9mZtYlM4ZARByIiAfT9M+Bh4E+YC1wU1rsJuC8NL0WGI6IwxHxOPXf+s+StBg4KSK2pZS6ubCOmZn1wDFdGJa0FHgD8D2gEhEH0qyfApU03Qc8UVhtNLX1penJ7WZm1iNNXxiWtBC4FfhYRDwj6VfzIiIkdew7qSUNAoMAlUqFWq3WcNnx8fFp589XudYN+daea93g2rtZe1MhIOlF1APgyxHx1dR8UNLiiDiQTvUcSu1j1C8mH7EktY2l6cntvyYihoAhgIGBgahWqw3HVqvVmG7+fJVr3ZBv7bnWDa69m7XPeDpI9V/5rwcejojPFmZtBS5O0xcDtxfa10k6QdIy6heAH0injp6RdHba5kWFdczMrAea+STwRuD9wIiknantE8BVwBZJHwR+BLwXICJ2S9pC/ZbSCeDyiHg+rXcZcCOwALg7vczMrEdmDIGI+C6gBrPPabDOJmDTFO3bgTOPZYBmZtY9/toIM7OMzfqvjbC5YenGO0vra0P/BOsL/e2/6h2l9W023/iTgJlZxhwCZmYZcwiYmWXMIWBmljGHgJlZxhwCZmYZcwiYmWXMIWBmljGHgJlZxhwCZmYZcwiYmWXMIWBmljGHgJlZxvwtovNMmd/maWZznz8JmJllzCFgZpaxZh40f4OkQ5J2Fdr+XdLO9Np/5NnDkpZKeq4w7/OFdVZJGpG0T9LV6WHzZmbWQ81cE7gRuAa4+UhDRLzvyLSkzcDTheUfjYiVU2znOuBS4HvAXcAa/KB5M7OemvGTQETcDzw11bz02/x7gVum24akxcBJEbEtIoJ6oJx37MM1M7NOavfuoDcDByPih4W2Zen00NPA30bEfwJ9wGhhmdHUNiVJg8AgQKVSoVarNRzA+Pj4tPPnq0Z1b+ifKH8wJasseGGduez/XI91cO3drL3dELiAF34KOAC8OiJ+JmkV8DVJK451oxExBAwBDAwMRLVabbhsrVZjuvnzVaO612dwi+iG/gk2jxw9dPdfWO3dYEqU67EOrr2btbccApKOB94NrDrSFhGHgcNpeoekR4HTgTFgSWH1JanNzMx6qJ1bRN8KPBIRvzrNI+kVko5L06cBy4HHIuIA8Iyks9N1hIuA29vo28zMOqCZW0RvAf4beI2kUUkfTLPW8esXhN8CPJSuCXwF+HBEHLmofBnwRWAf8Ci+M8jMrOdmPB0UERc0aF8/RdutwK0Nlt8OnHmM4zMzsy7yXwybmWXMIWBmljGHgJlZxhwCZmYZcwiYmWXMIWBmljGHgJlZxhwCZmYZcwiYmWXMIWBmljGHgJlZxhwCZmYZcwiYmWXMIWBmljGHgJlZxhwCZmYZcwiYmWWsmcdL3iDpkKRdhbZPSRqTtDO93l6Yd4WkfZL2Sjq30L5K0kiad3V61rCZmfVQM58EbgTWTNH+uYhYmV53AUg6g/qzh1ekda498uB54DrgUuoPn1/eYJtmZlaiGUMgIu4HnpppuWQtMBwRhyPiceoPlT9L0mLgpIjYFhEB3Ayc1+qgzcysM9q5JvARSQ+l00WnpLY+4InCMqOprS9NT243M7MeOr7F9a4DrgQi/dwMXNKpQUkaBAYBKpUKtVqt4bLj4+PTzp+vGtW9oX+i/MGUrLLghXXmsv9zPdbBtXez9pZCICIOHpmW9AXgjvR2DDi1sOiS1DaWpie3N9r+EDAEMDAwENVqteFYarUa082frxrVvX7jneUPpmQb+ifYPHL00N1/YbV3gylRrsc6uPZu1t7S6aB0jv+I84Ejdw5tBdZJOkHSMuoXgB+IiAPAM5LOTncFXQTc3sa4zcysA2b8JCDpFqAKLJI0CnwSqEpaSf100H7gQwARsVvSFmAPMAFcHhHPp01dRv1OowXA3ellZmY9NGMIRMQFUzRfP83ym4BNU7RvB848ptGZmVlX+S+Gzcwy5hAwM8uYQ8DMLGMOATOzjDkEzMwy5hAwM8uYQ8DMLGMOATOzjDkEzMwy5hAwM8uYQ8DMLGMOATOzjDkEzMwy5hAwM8uYQ8DMLGMOATOzjDkEzMwy5hAwM8vYjCEg6QZJhyTtKrT9k6RHJD0k6TZJL0vtSyU9J2lnen2+sM4qSSOS9km6Oj1w3szMeqiZTwI3Amsmtd0DnBkRrwP+B7iiMO/RiFiZXh8utF8HXAosT6/J2zQzs5LNGAIRcT/w1KS2b0bERHq7DVgy3TYkLQZOiohtERHAzcB5rQ3ZzMw6pRPXBC4B7i68X5ZOBX1H0ptTWx8wWlhmNLWZmVkPqf6L+QwLSUuBOyLizEntfwMMAO+OiJB0ArAwIn4maRXwNWAFcDpwVUS8Na33ZuDjEfHOBv0NAoMAlUpl1fDwcMOxjY+Ps3DhwhlrmG8a1T0y9nQPRlOuygI4+NzR9/19J/duMCXK9VgH195K7atXr94REQMzLXd8S6MCJK0H3gmck07xEBGHgcNpeoekR6kHwBgvPGW0JLVNKSKGgCGAgYGBqFarDcdRq9WYbv581aju9RvvLH8wJdvQP8HmkaOH7v4Lq70bTIlyPdbBtXez9pZOB0laA/w18CcR8b+F9ldIOi5Nn0b9AvBjEXEAeEbS2emuoIuA29sevZmZtWXGTwKSbgGqwCJJo8Anqd8NdAJwT7rTc1u6E+gtwD9I+gXwS+DDEXHkovJl1O80WkD9GkLxOoKZmfXAjCEQERdM0Xx9g2VvBW5tMG87cOZU88zMrDf8F8NmZhlzCJiZZcwhYGaWMYeAmVnGHAJmZhlzCJiZZcwhYGaWMYeAmVnGHAJmZhlzCJiZZcwhYGaWMYeAmVnGHAJmZhlzCJiZZcwhYGaWMYeAmVnGHAJmZhlzCJiZZWzGEJB0g6RDknYV2l4u6R5JP0w/TynMu0LSPkl7JZ1baF8laSTNuzo9cN7MzHqomU8CNwJrJrVtBO6NiOXAvek9ks4A1gEr0jrXSjourXMdcCmwPL0mb9PMzEo2YwhExP3AU5Oa1wI3pembgPMK7cMRcTgiHgf2AWdJWgycFBHbIiKAmwvrmJlZjxzf4nqViDiQpn8KVNJ0H7CtsNxoavtFmp7cPiVJg8AgQKVSoVarNRzI+Pj4tPPnq0Z1b+ifKH8wJasseGGduez/XI91cO3drL3VEPiViAhJ0YnBFLY5BAwBDAwMRLVabbhsrVZjuvnzVaO612+8s/zBlGxD/wSbR44euvsvrPZuMCXK9VgH197N2lu9O+hgOsVD+nkotY8BpxaWW5LaxtL05HYzM+uhVkNgK3Bxmr4YuL3Qvk7SCZKWUb8A/EA6dfSMpLPTXUEXFdYxM7MemfF0kKRbgCqwSNIo8EngKmCLpA8CPwLeCxARuyVtAfYAE8DlEfF82tRl1O80WgDcnV7z0tISTsls6J/I4tSPmXXXjCEQERc0mHVOg+U3AZumaN8OnHlMozMzs67yXwybmWXMIWBmljGHgJlZxhwCZmYZcwiYmWWs7b8YNuu1Mm7Jncr+q97Rk37NOsmfBMzMMuYQMDPLmEPAzCxjDgEzs4w5BMzMMuYQMDPLmEPAzCxjDgEzs4w5BMzMMuYQMDPLmEPAzCxjLYeApNdI2ll4PSPpY5I+JWms0P72wjpXSNonaa+kcztTgpmZtarlL5CLiL3ASgBJxwFjwG3AB4DPRcRnistLOgNYB6wAXgV8S9LphWcQm5lZyTp1Ougc4NGI+NE0y6wFhiPicEQ8DuwDzupQ/2Zm1oJOhcA64JbC+49IekjSDZJOSW19wBOFZUZTm5mZ9Ygior0NSC8GfgKsiIiDkirAk0AAVwKLI+ISSdcA2yLiS2m964G7I+IrU2xzEBgEqFQqq4aHhxv2Pz4+zsKFC9uqodNGxp7ueh+VBXDwua53MyvNltr7+04utb/ZeKyXxbUfe+2rV6/eEREDMy3XiYfK/DHwYEQcBDjyE0DSF4A70tsx4NTCektS26+JiCFgCGBgYCCq1WrDzmu1GtPN74X1JTzkZEP/BJtH8nwm0Gypff+F1VL7m43Hellce7Vr2+/E6aALKJwKkrS4MO98YFea3gqsk3SCpGXAcuCBDvRvZmYtauvXKUknAm8DPlRo/kdJK6mfDtp/ZF5E7Ja0BdgDTACX+84gM7PeaisEIuJZ4Dcntb1/muU3AZva6dPMzDrHfzFsZpYxh4CZWcYcAmZmGXMImJllzCFgZpYxh4CZWcYcAmZmGXMImJllzCFgZpYxh4CZWcYcAmZmGXMImJllzCFgZpYxh4CZWcYcAmZmGXMImJllzCFgZpYxh4CZWcbaCgFJ+yWNSNopaXtqe7mkeyT9MP08pbD8FZL2Sdor6dx2B29mZu3pxCeB1RGxMiIG0vuNwL0RsRy4N71H0hnAOmAFsAa4VtJxHejfzMxa1I3TQWuBm9L0TcB5hfbhiDgcEY8D+4CzutC/mZk1qd0QCOBbknZIGkxtlYg4kKZ/ClTSdB/wRGHd0dRmZmY9cnyb678pIsYkvRK4R9IjxZkREZLiWDeaAmUQoFKpUKvVGi47Pj4+7fxe2NA/0fU+KgvK6Wc2mi21l33czcZjvSyuvda17bcVAhExln4eknQb9dM7ByUtjogDkhYDh9LiY8CphdWXpLaptjsEDAEMDAxEtVptOIZarcZ083th/cY7u97Hhv4JNo+0m+Fz02ypff+F1VL7m43Hellce7Vr22/5dJCkEyW99Mg08EfALmArcHFa7GLg9jS9FVgn6QRJy4DlwAOt9m9mZu1r59epCnCbpCPb+beI+Lqk7wNbJH0Q+BHwXoCI2C1pC7AHmAAuj4jn2xq9mZm1peUQiIjHgNdP0f4z4JwG62wCNrXap5mZdZb/YtjMLGMOATOzjDkEzMwy5hAwM8uYQ8DMLGMOATOzjDkEzMwy5hAwM8uYQ8DMLGMOATOzjDkEzMwy5hAwM8uYQ8DMLGMOATOzjDkEzMwy5hAwM8uYQ8DMLGPtPGP4VEn3Sdojabekj6b2T0kak7Qzvd5eWOcKSfsk7ZV0bicKMDOz1rXzjOEJYENEPJgeOL9D0j1p3uci4jPFhSWdAawDVgCvAr4l6XQ/Z9jMrHda/iQQEQci4sE0/XPgYaBvmlXWAsMRcTgiHgf2AWe12r+ZmbWvI9cEJC0F3gB8LzV9RNJDkm6QdEpq6wOeKKw2yvShYWZmXaaIaG8D0kLgO8CmiPiqpArwJBDAlcDiiLhE0jXAtoj4UlrveuDuiPjKFNscBAYBKpXKquHh4Yb9j4+Ps3DhwrZq6LSRsae73kdlARx8ruvdzEqzpfb+vpNL7W82Hutlce3HXvvq1at3RMTATMu1c00ASS8CbgW+HBFfBYiIg4X5XwDuSG/HgFMLqy9Jbb8mIoaAIYCBgYGoVqsNx1Cr1Zhufi+s33hn1/vY0D/B5pG2dt+cNVtq339htdT+ZuOxXhbXXu3a9tu5O0jA9cDDEfHZQvviwmLnA7vS9FZgnaQTJC0DlgMPtNq/mZm1r51fp94IvB8YkbQztX0CuEDSSuqng/YDHwKIiN2StgB7qN9ZdLnvDDIz662WQyAivgtoill3TbPOJmBTq32amVln+S+Gzcwy5hAwM8uYQ8DMLGO9v8/ObI5aWsKtwEUb+id+dfvx/qveUWrfNn/5k4CZWcYcAmZmGXMImJllzCFgZpYxh4CZWcbm9d1BZd+9YWY21/iTgJlZxhwCZmYZcwiYmWXMIWBmljGHgJlZxhwCZmYZcwiYmWXMIWBmlrHSQ0DSGkl7Je2TtLHs/s3M7KhS/2JY0nHAvwJvA0aB70vaGhF7yhyH2VzXq7+G93MM5p+yPwmcBeyLiMci4v+AYWBtyWMwM7Ok7O8O6gOeKLwfBf6g5DGYWYt69QnkxjUn9qTfHMzKL5CTNAgMprfjkvZOs/gi4Mnuj2p2+ctM64Z8a8+1boDVn863dlrf77/dzEJlh8AYcGrh/ZLU9gIRMQQMNbNBSdsjYqAzw5s7cq0b8q0917rBtXez9rKvCXwfWC5pmaQXA+uArSWPwczMklI/CUTEhKS/AL4BHAfcEBG7yxyDmZkdVfo1gYi4C7irg5ts6rTRPJRr3ZBv7bnWDa69axQR3dy+mZnNYv7aCDOzjM2JEJjpqyYkrZX0kKSdkrZLelMvxtkNzX7NhqTflzQh6T1ljq9bmtjnVUlPp32+U9Lf92Kc3dDMPk/175S0W9J3yh5jtzSx3/+qsM93SXpe0st7MdZOaqLukyX9h6QfpH3+gY51HhGz+kX9AvKjwGnAi4EfAGdMWmYhR09tvQ54pNfjLqv2wnLfpn6t5T29HndJ+7wK3NHrsfao9pcBe4BXp/ev7PW4y6p90vLvAr7d63GXtM8/AXw6Tb8CeAp4cSf6nwufBGb8qomIGI/0rwOcCMyXCx3Nfs3GR4BbgUNlDq6Lcv56kWZq/zPgqxHxY4CIyHW/XwDcUsrIuquZugN4qSRR/6X3KWCiE53PhRCY6qsm+iYvJOl8SY8AdwKXlDS2bpuxdkl9wPnAdSWOq9ua2ufAH6bTgHdLWlHO0LqumdpPB06RVJO0Q9JFpY2uu5rd70h6CbCG+i8/c10zdV8D/C7wE2AE+GhE/LITnc+FEGhKRNwWEa8FzgOu7PV4SvTPwMc7dUDMIQ9SPx3yOuBfgK/1eDxlOh5YBbwDOBf4O0mn93ZIpXsX8F8R8VSvB1KSc4GdwKuAlcA1kk7qxIbnQgg09VUTR0TE/cBpkhZ1e2AlaKb2AWBY0n7gPcC1ks4rZ3hdM2PdEfFMRIyn6buAF2W0z0eBb0TEsxHxJHA/8PqSxtdNx/Lf+jrmx6kgaK7uD1A/BRgRsQ94HHhtR3rv9UWRJi6aHA88Bizj6EWTFZOW+R2OXhj+vfQPqF6PvYzaJy1/I/PjwnAz+/y3Cvv8LODHuexz6qcF7k3LvgTYBZzZ67GXUXta7mTq58RP7PWYS9zn1wGfStOV9P+4RZ3of1Z+i2hRNPiqCUkfTvM/D/wpcJGkXwDPAe+L9K81lzVZ+7zTZN3vAf5c0gT1fb4ul30eEQ9L+jrwEPBL4IsRsat3o+6MYzjezwe+GRHP9mioHdVk3VcCN0oaAUT9FHBHvlXVfzFsZpaxuXBNwMzMusQhYGaWMYeAmVnGHAJmZhlzCJiZZcwhYGaWMYeAmVnGHAJmZhn7fzlhvedlaf8oAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f2e0d182c18>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dtrain[\"predict\"] = preds\n",
    "dtrain.predict[:5]\n",
    "if verboseout:\n",
    "    dtrain.predict.hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYEAAAD8CAYAAACRkhiPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAE4ZJREFUeJzt3H+s3fV93/Hnq3ZCXQgJiPTOM2wQzW1mcEnLHWJr/7gZ2vCadRAtYs7YMBnC22BZJlnbTKVtnSprVApTy1rYrLbDaF2R1TbDCyEZc3IXbRqh0JEY80N4wRR7Bi+JNnbRxHLpe3/cD+PM+N57DPecc30/z4d0dD7n8/18v9/Pm+/XvO75fs85qSokSX36gUlPQJI0OYaAJHXMEJCkjhkCktQxQ0CSOmYISFLHDAFJ6thQIZDkQ0l+K8lzSZ5N8ieTXJjk0SQvtOcLBsbfmeRIkueTXDfQf1WSQ23ZPUkyiqIkScMZ9p3ALwFfrqqPAlcCzwK7gYNVtRk42F6TZAuwHbgc2Abcm2Rd2859wG3A5vbYtkJ1SJLehSz3jeEkHwSeAj5SA4OTPA/MVNWJJBuB2ar60SR3AlTVP2njvgL8HHAU+FoLEpJ8uq3/15fa/0UXXVSXXnrpostff/11zj333GXKXHt6rRv6rb3XusHa303tTz755Heq6sPLjVs/xLYuA/478C+TXAk8CXwOmKqqE23MK8BUa28CHhtY/1jr+35rn9r/Dkl2AjsBpqam+PznP7/o5Obm5jjvvPOGKGNt6bVu6Lf2XusGa383tX/84x9/aZhxw4TAeuAngM9W1TeS/BLt0s9bqqqSrNiPEFXVXmAvwPT0dM3MzCw6dnZ2lqWWr1W91g391t5r3WDto6x9mHsCx4BjVfWN9vq3WAiFV9tlINrzybb8OHDJwPoXt77jrX1qvyRpQpYNgap6BXg5yY+2rmuBZ4ADwI7WtwN4qLUPANuTnJPkMhZuAD/eLh29luSa9qmgmwfWkSRNwDCXgwA+C/xGkvcD3wY+w0KA7E9yK/AScCNAVR1Osp+FoJgH7qiqN9t2bgfuBzYAj7SHJGlChgqBqnoKmD7NomsXGb8H2HOa/ieAK85kgpKk0fEbw5LUMUNAkjpmCEhSxwwBSerYsJ8OOitduvvhSU9hZHZtneeWNVzfUlZL7Ufv+sSkpyC9Z74TkKSOGQKS1DFDQJI6ZghIUscMAUnqmCEgSR0zBCSpY4aAJHXMEJCkjhkCktQxQ0CSOmYISFLHDAFJ6pghIEkdMwQkqWOGgCR1zBCQpI4ZApLUMUNAkjpmCEhSxwwBSeqYISBJHRsqBJIcTXIoyVNJnmh9FyZ5NMkL7fmCgfF3JjmS5Pkk1w30X9W2cyTJPUmy8iVJkoZ1Ju8EPl5VH6uq6fZ6N3CwqjYDB9trkmwBtgOXA9uAe5Osa+vcB9wGbG6Pbe+9BEnSu/VeLgddD+xr7X3ADQP9D1bVG1X1InAEuDrJRuD8qnqsqgp4YGAdSdIErB9yXAH/PsmbwL+oqr3AVFWdaMtfAaZaexPw2MC6x1rf91v71P53SLIT2AkwNTXF7OzsohObm5tbdPmurfNL1XRWm9qwtutbymqpfanzchSWOtfXOmufHdn2hw2Bn6qq40l+GHg0yXODC6uqktRKTaqFzF6A6enpmpmZWXTs7Owsiy2/ZffDKzWlVWfX1nnuPjTs4VtbVkvtR2+aGev+ljrX1zprnxnZ9oe6HFRVx9vzSeALwNXAq+0SD+35ZBt+HLhkYPWLW9/x1j61X5I0IcuGQJJzk3zgrTbwZ4GngQPAjjZsB/BQax8Atic5J8llLNwAfrxdOnotyTXtU0E3D6wjSZqAYd5TTwFfaJ/mXA/866r6cpLfBfYnuRV4CbgRoKoOJ9kPPAPMA3dU1ZttW7cD9wMbgEfaQ5I0IcuGQFV9G7jyNP3fBa5dZJ09wJ7T9D8BXHHm05QkjYLfGJakjhkCktQxQ0CSOmYISFLHDAFJ6pghIEkdMwQkqWOGgCR1zBCQpI4ZApLUMUNAkjpmCEhSxwwBSeqYISBJHTMEJKljhoAkdcwQkKSOGQKS1DFDQJI6ZghIUscMAUnqmCEgSR0zBCSpY4aAJHXMEJCkjhkCktQxQ0CSOjZ0CCRZl+S/JPlie31hkkeTvNCeLxgYe2eSI0meT3LdQP9VSQ61ZfckycqWI0k6E2fyTuBzwLMDr3cDB6tqM3CwvSbJFmA7cDmwDbg3ybq2zn3AbcDm9tj2nmYvSXpPhgqBJBcDnwB+daD7emBfa+8Dbhjof7Cq3qiqF4EjwNVJNgLnV9VjVVXAAwPrSJImYP2Q434R+HvABwb6pqrqRGu/Aky19ibgsYFxx1rf91v71P53SLIT2AkwNTXF7OzsohObm5tbdPmurfOLrne2m9qwtutbymqpfanzchSWOtfXOmufHdn2lw2BJH8eOFlVTyaZOd2YqqoktVKTqqq9wF6A6enpmpk57W6BhX+Iiy2/ZffDKzWlVWfX1nnuPjRshq8tq6X2ozfNjHV/S53ra521z4xs+8P8S/pJ4C8k+WngB4Hzk/wr4NUkG6vqRLvUc7KNPw5cMrD+xa3veGuf2i9JmpBl7wlU1Z1VdXFVXcrCDd+vVtVfAQ4AO9qwHcBDrX0A2J7knCSXsXAD+PF26ei1JNe0TwXdPLCOJGkC3st76ruA/UluBV4CbgSoqsNJ9gPPAPPAHVX1ZlvnduB+YAPwSHtIkibkjEKgqmaB2db+LnDtIuP2AHtO0/8EcMWZTlKSNBp+Y1iSOmYISFLHDAFJ6pghIEkdMwQkqWOGgCR1zBCQpI4ZApLUMUNAkjpmCEhSxwwBSeqYISBJHTMEJKljhoAkdcwQkKSOGQKS1DFDQJI6ZghIUscMAUnqmCEgSR0zBCSpY4aAJHXMEJCkjhkCktQxQ0CSOmYISFLHDAFJ6tiyIZDkB5M8nuSbSQ4n+cet/8IkjyZ5oT1fMLDOnUmOJHk+yXUD/VclOdSW3ZMkoylLkjSMYd4JvAH86aq6EvgYsC3JNcBu4GBVbQYOttck2QJsBy4HtgH3JlnXtnUfcBuwuT22rWAtkqQztGwI1IK59vJ97VHA9cC+1r8PuKG1rwcerKo3qupF4AhwdZKNwPlV9VhVFfDAwDqSpAkY6p5AknVJngJOAo9W1TeAqao60Ya8Aky19ibg5YHVj7W+Ta19ar8kaULWDzOoqt4EPpbkQ8AXklxxyvJKUis1qSQ7gZ0AU1NTzM7OLjp2bm5u0eW7ts6v1JRWnakNa7u+payW2pc6L0dhqXN9rbP22ZFtf6gQeEtV/Y8kX2PhWv6rSTZW1Yl2qedkG3YcuGRgtYtb3/HWPrX/dPvZC+wFmJ6erpmZmUXnNDs7y2LLb9n98PJFnaV2bZ3n7kNndPjWjNVS+9GbZsa6v6XO9bXO2mdGtv1hPh304fYOgCQbgD8DPAccAHa0YTuAh1r7ALA9yTlJLmPhBvDj7dLRa0muaZ8KunlgHUnSBAzz59RGYF/7hM8PAPur6otJ/jOwP8mtwEvAjQBVdTjJfuAZYB64o11OArgduB/YADzSHpKkCVk2BKrqW8CPn6b/u8C1i6yzB9hzmv4ngCveuYYkaRL8xrAkdcwQkKSOGQKS1DFDQJI6ZghIUscMAUnqmCEgSR0zBCSpY4aAJHXMEJCkjhkCktQxQ0CSOmYISFLHDAFJ6pghIEkdMwQkqWOGgCR1zBCQpI4ZApLUMUNAkjpmCEhSxwwBSeqYISBJHTMEJKljhoAkdcwQkKSOGQKS1LFlQyDJJUm+luSZJIeTfK71X5jk0SQvtOcLBta5M8mRJM8nuW6g/6okh9qye5JkNGVJkoYxzDuBeWBXVW0BrgHuSLIF2A0crKrNwMH2mrZsO3A5sA24N8m6tq37gNuAze2xbQVrkSSdoWVDoKpOVNXvtfb/Ap4FNgHXA/vasH3ADa19PfBgVb1RVS8CR4Crk2wEzq+qx6qqgAcG1pEkTcAZ3RNIcinw48A3gKmqOtEWvQJMtfYm4OWB1Y61vk2tfWq/JGlC1g87MMl5wG8Df6eqXhu8nF9VlaRWalJJdgI7AaamppidnV107Nzc3KLLd22dX6kprTpTG9Z2fUtZLbUvdV6OwlLn+lpn7bMj2/5QIZDkfSwEwG9U1e+07leTbKyqE+1Sz8nWfxy4ZGD1i1vf8dY+tf8dqmovsBdgenq6ZmZmFp3b7Owsiy2/ZffDS9Z1Ntu1dZ67Dw2d4WvKaqn96E0zY93fUuf6WmftMyPb/jCfDgrwa8CzVfVPBxYdAHa09g7goYH+7UnOSXIZCzeAH2+Xjl5Lck3b5s0D60iSJmCYP6d+EvirwKEkT7W+nwXuAvYnuRV4CbgRoKoOJ9kPPMPCJ4vuqKo323q3A/cDG4BH2kOSNCHLhkBV/Udgsc/zX7vIOnuAPafpfwK44kwmKEkaHb8xLEkdMwQkqWOGgCR1zBCQpI4ZApLUMUNAkjpmCEhSxwwBSeqYISBJHTMEJKljhoAkdcwQkKSOGQKS1DFDQJI6ZghIUscMAUnqmCEgSR0zBCSpY4aAJHXMEJCkjhkCktQxQ0CSOmYISFLHDAFJ6pghIEkdMwQkqWOGgCR1bNkQSPLrSU4meXqg78IkjyZ5oT1fMLDsziRHkjyf5LqB/quSHGrL7kmSlS9HknQmhnkncD+w7ZS+3cDBqtoMHGyvSbIF2A5c3ta5N8m6ts59wG3A5vY4dZuSpDFbNgSq6uvA907pvh7Y19r7gBsG+h+sqjeq6kXgCHB1ko3A+VX1WFUV8MDAOpKkCXm39wSmqupEa78CTLX2JuDlgXHHWt+m1j61X5I0Qevf6waqqpLUSkzmLUl2AjsBpqammJ2dXXTs3Nzcost3bZ1fyWmtKlMb1nZ9S1kttS91Xo7CUuf6WmftsyPb/rsNgVeTbKyqE+1Sz8nWfxy4ZGDcxa3veGuf2n9aVbUX2AswPT1dMzMzi05kdnaWxZbfsvvh5eo4a+3aOs/dh95zhp+VVkvtR2+aGev+ljrX1zprnxnZ9t/t5aADwI7W3gE8NNC/Pck5SS5j4Qbw4+3S0WtJrmmfCrp5YB1J0oQs++dUkt8EZoCLkhwD/hFwF7A/ya3AS8CNAFV1OMl+4BlgHrijqt5sm7qdhU8abQAeaQ9J0gQtGwJV9elFFl27yPg9wJ7T9D8BXHFGs5MkjZTfGJakjhkCktQxQ0CSOmYISFLHDAFJ6pghIEkdMwQkqWOGgCR1zBCQpI4ZApLUMUNAkjpmCEhSxwwBSeqYISBJHTMEJKljhoAkdcwQkKSOGQKS1DFDQJI6ZghIUscMAUnqmCEgSR0zBCSpY+snPQHpbHXp7ofHur9dW+e5Zcz7XC16rP3oXZ8Yy358JyBJHTMEJKljhoAkdWzsIZBkW5LnkxxJsnvc+5ckvW2sIZBkHfArwJ8DtgCfTrJlnHOQJL1t3O8ErgaOVNW3q+r/AA8C1495DpKkZtwhsAl4eeD1sdYnSZqAVfk9gSQ7gZ3t5VyS55cYfhHwndHPanX5253WDf3W3mvd0Gft+YX/13y3tf/RYQaNOwSOA5cMvL649f1/qmovsHeYDSZ5oqqmV2Z6Z49e64Z+a++1brD2UdY+7stBvwtsTnJZkvcD24EDY56DJKkZ6zuBqppP8reArwDrgF+vqsPjnIMk6W1jvydQVV8CvrSCmxzqstEa1Gvd0G/tvdYN1j4yqapRbl+StIr5sxGS1LGzIgSW+6mJJNcn+VaSp5I8keSnJjHPURj2ZzaS/Ikk80k+Nc75jcoQx3wmyf9sx/ypJP9wEvMchWGOeav/qSSHk/yHcc9xVIY47n934Jg/neTNJBdOYq4raYi6P5jk3yb5Zjvmn1mxnVfVqn6wcAP5vwIfAd4PfBPYcsqY83j70taPAc9Net7jqn1g3FdZuNfyqUnPe0zHfAb44qTnOqHaPwQ8A/yR9vqHJz3vcdV+yvifAb466XmP6Zj/LPALrf1h4HvA+1di/2fDO4Flf2qiquaq/dcBzgXWyo2OYX9m47PAbwMnxzm5Eer550WGqf0vA79TVb8PUFW9HvdPA785lpmN1jB1F/CBJGHhj97vAfMrsfOzIQSG+qmJJJ9M8hzwMPDXxjS3UVu29iSbgE8C941xXqM27M+L/Kl2GfCRJJePZ2ojN0ztPwJckGQ2yZNJbh7b7EZr6J+VSfJDwDYW/vg52w1T9y8Dfxz4b8Ah4HNV9QcrsfOzIQSGUlVfqKqPAjcAPz/p+YzRLwJ/f6VOiLPI77FwOeTHgH8G/JsJz2ec1gNXAZ8ArgP+QZIfmeyUxu5ngP9UVd+b9ETG5DrgKeAPAx8DfjnJ+Sux4bMhBIb6qYm3VNXXgY8kuWjUExuDYWqfBh5MchT4FHBvkhvGM72RWbbuqnqtquZa+0vA+zo65seAr1TV61X1HeDrwJVjmt8oncm/9e2sjUtBMFzdn2HhEmBV1RHgReCjK7L3Sd8UGeKmyXrg28BlvH3T5PJTxvwx3r4x/BPtP2AmPfdx1H7K+PtZGzeGhznmf2jgmF8N/H4vx5yFywIH29gfAp4Grpj03MdRexv3QRauiZ876TmP8ZjfB/xca0+1/8ddtBL7X5W/IjqoFvmpiSR/oy3/58BfBG5O8n3gfwN/qdp/rbPZkLWvOUPW/SngbyaZZ+GYb+/lmFfVs0m+DHwL+APgV6vq6cnNemWcwfn+SeDfVdXrE5rqihqy7p8H7k9yCAgLl4BX5FdV/cawJHXsbLgnIEkaEUNAkjpmCEhSxwwBSeqYISBJHTMEJKljhoAkdcwQkKSO/V9NvwzMSfUoLQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f2dcdb82860>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "if verboseout:\n",
    "    dtrain.predict.hist(bins=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'numpy.ndarray'>\n",
      "(5950, 1)\n",
      "<class 'pandas.core.series.Series'>\n"
     ]
    }
   ],
   "source": [
    "# print(type(apar_ds['Time_to_relief'].iloc[0]))\n",
    "print(type(preds))\n",
    "print(preds.shape)\n",
    "print(type(dtrain.target))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/paperspace/anaconda3/envs/fastai/lib/python3.6/site-packages/ipykernel_launcher.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "810     0\n",
       "7110    0\n",
       "3967    1\n",
       "3800    1\n",
       "627     0\n",
       "Name: predround, dtype: int64"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get rounded predictions \n",
    "dtrain[\"predround\"] = preds.round().astype(int)\n",
    "dtrain.predround[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1273.0\n",
      "percentage correct train\n",
      "0.786050420168\n"
     ]
    }
   ],
   "source": [
    "# get delta between predictions on training set and actual training target values\n",
    "# hand calculate accuracy on training set as ratio of (total training samples - wrong training predictions)/total training samples\n",
    "\n",
    "deltatr = abs(dtrain.target[:100000] - dtrain.predround[:100000])\n",
    "deltatr[:50]\n",
    "print(deltatr.sum())\n",
    "print(\"percentage correct train\")\n",
    "print((len(deltatr) - deltatr.sum())/len(deltatr))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/paperspace/anaconda3/envs/fastai/lib/python3.6/site-packages/ipykernel_launcher.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  after removing the cwd from sys.path.\n",
      "/home/paperspace/anaconda3/envs/fastai/lib/python3.6/site-packages/ipykernel_launcher.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \"\"\"\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "6508    0.488539\n",
       "3460    0.523025\n",
       "2945    0.388055\n",
       "6663    0.397696\n",
       "4871    0.365404\n",
       "Name: predict, dtype: float32"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# predict values for validation X values\n",
    "# X_valid, dvalid.target\n",
    "predval = model.predict(X_valid, batch_size=BATCH_SIZE)\n",
    "dvalid[\"predround\"] = predval.round().astype(int)\n",
    "dvalid[\"predict\"] = predval\n",
    "#print(type(deltaval))\n",
    "#print(len(deltaval))\n",
    "dvalid.predict[:5]\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYEAAAD8CAYAAACRkhiPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAEkFJREFUeJzt3X+MXPdZ7/H3B5sGN1abRIHFsnOx0TVcnBgQWUL5IbRRkOLbAk51q1yXQB0IWEAoBVmAAxL9A1mKdAkCBCmySFWjVrVMaIlFGm4jlyECkYakhDpOGmKaXzZpDBRaNlSBTR/+2AMZjMPOnlnP2Pt9v6TVnvme75nzPDuz/uw5Z2acqkKS1KYvmXYBkqTpMQQkqWGGgCQ1zBCQpIYZApLUMENAkhpmCEhSwwwBSWqYISBJDVs77QKWcvnll9fmzZvP+X5eeuklLr744nO+n/NVy/233DvY/2rt/5FHHvm7qvrypead9yGwefNmHn744XO+n8FgwNzc3Dnfz/mq5f5b7h3sf7X2n+TZUeZ5OkiSGmYISFLDDAFJapghIEkNMwQkqWGGgCQ1zBCQpIYZApLUMENAkhp23r9jeByb99078ty92xe4eRnzV5uW++/b+zO3v+UcVCNNlkcCktQwQ0CSGmYISFLDDAFJapghIEkNMwQkqWGGgCQ1zBCQpIYZApLUMENAkhpmCEhSwwwBSWqYISBJDVsyBJK8N8npJI8Njf2/JJ9K8skkH05yydC625KcSPJkkuuHxq9Ocqxb9+tJsvLtSJKWY5QjgfcBO84Yux+4qqq+Hvgr4DaAJNuAXcCV3TZ3JlnTbfMe4EeArd3XmfcpSZqwJUOgqh4APnvG2EeraqG7+SCwqVveCRyqqper6mngBHBNkg3AG6rqwaoq4HeAG1aqCUlSPytxTeCHgPu65Y3A80PrTnZjG7vlM8clSVM01v8sluQXgAXgAytTzn/c7x5gD8DMzAyDwaDX/ezdvrD0pM7MuuXNX21a7r9v732fl+eb+fn5VdNLH6333zsEktwMfDdwXXeKB+AUcMXQtE3d2ClePWU0PH5WVXUAOAAwOztbc3NzvWpczn8ZuHf7AnccW9X/2+Z/q+X++/b+zE1zK1/MFAwGA/r+jq0Grfff63RQkh3AzwLfW1X/PLTqCLAryUVJtrB4AfihqnoB+HySN3WvCnoHcM+YtUuSxrTknz9JPgjMAZcnOQm8m8VXA10E3N+90vPBqvrRqjqe5DDwOIuniW6tqle6u/pxFl9ptI7Fawj3IUmaqiVDoKrefpbhu/6b+fuB/WcZfxi4alnVSZLOKd8xLEkNMwQkqWGGgCQ1zBCQpIYZApLUMENAkhpmCEhSwwwBSWqYISBJDTMEJKlhhoAkNcwQkKSGGQKS1DBDQJIaZghIUsMMAUlqmCEgSQ0zBCSpYYaAJDXMEJCkhhkCktQwQ0CSGmYISFLDlgyBJO9NcjrJY0NjlyW5P8lT3fdLh9bdluREkieTXD80fnWSY926X0+SlW9HkrQcoxwJvA/YccbYPuBoVW0Fjna3SbIN2AVc2W1zZ5I13TbvAX4E2Np9nXmfkqQJWzIEquoB4LNnDO8EDnbLB4EbhsYPVdXLVfU0cAK4JskG4A1V9WBVFfA7Q9tIkqZkbc/tZqrqhW75M8BMt7wReHBo3slu7F+75TPHzyrJHmAPwMzMDIPBoFeRe7cvjDx3Zt3y5q82Lffft/e+z8vzzfz8/KrppY/W++8bAv+hqipJrUQxQ/d5ADgAMDs7W3Nzc73u5+Z99448d+/2Be44NvaP44LVcv99e3/mprmVL2YKBoMBfX/HVoPW++/76qAXu1M8dN9Pd+OngCuG5m3qxk51y2eOS5KmqG8IHAF2d8u7gXuGxncluSjJFhYvAD/UnTr6fJI3da8KesfQNpKkKVnyGDjJB4E54PIkJ4F3A7cDh5PcAjwL3AhQVceTHAYeBxaAW6vqle6ufpzFVxqtA+7rviRJU7RkCFTV219j1XWvMX8/sP8s4w8DVy2rOknSOeU7hiWpYYaAJDXMEJCkhhkCktQwQ0CSGmYISFLDDAFJapghIEkNMwQkqWGGgCQ1zBCQpIYZApLUMENAkhpmCEhSwwwBSWqYISBJDTMEJKlhhoAkNcwQkKSGGQKS1DBDQJIaZghIUsPGCoEkP53keJLHknwwyZcluSzJ/Ume6r5fOjT/tiQnkjyZ5Prxy5ckjaN3CCTZCPwkMFtVVwFrgF3APuBoVW0Fjna3SbKtW38lsAO4M8ma8cqXJI1j3NNBa4F1SdYCrwf+BtgJHOzWHwRu6JZ3Aoeq6uWqeho4AVwz5v4lSWPoHQJVdQr4ZeA54AXgc1X1UWCmql7opn0GmOmWNwLPD93FyW5MkjQla/tu2J3r3wlsAf4R+N0k3z88p6oqSfW47z3AHoCZmRkGg0GvGvduXxh57sy65c1fbVruv2/vfZ+X55v5+flV00sfrfffOwSA7wKerqq/BUjyIeDbgBeTbKiqF5JsAE53808BVwxtv6kb+y+q6gBwAGB2drbm5uZ6FXjzvntHnrt3+wJ3HBvnx3Fha7n/vr0/c9PcyhczBYPBgL6/Y6tB6/2Pc03gOeBNSV6fJMB1wBPAEWB3N2c3cE+3fATYleSiJFuArcBDY+xfkjSm3n/6VdXHk9wNfAJYAP6Cxb/e1wOHk9wCPAvc2M0/nuQw8Hg3/9aqemXM+iVJYxjr+L+q3g28+4zhl1k8Kjjb/P3A/nH2KUlaOb5jWJIaZghIUsMMAUlqmCEgSQ0zBCSpYYaAJDXMEJCkhhkCktQwQ0CSGmYISFLDDAFJapghIEkNMwQkqWGGgCQ1zBCQpIYZApLUMENAkhpmCEhSwwwBSWqYISBJDTMEJKlhhoAkNcwQkKSGjRUCSS5JcneSTyV5Ism3Jrksyf1Jnuq+Xzo0/7YkJ5I8meT68cuXJI1j3COBXwP+sKr+F/ANwBPAPuBoVW0Fjna3SbIN2AVcCewA7kyyZsz9S5LG0DsEkrwR+E7gLoCq+peq+kdgJ3Cwm3YQuKFb3gkcqqqXq+pp4ARwTd/9S5LGl6rqt2HyjcAB4HEWjwIeAd4FnKqqS7o5Af6hqi5J8hvAg1X1/m7dXcB9VXX3We57D7AHYGZm5upDhw71qvHYqc+NPHdmHbz4hV67WRVa7r9v79s3vnHli5mC+fl51q9fP+0ypma19n/ttdc+UlWzS81bO8Y+1gLfBLyzqj6e5NfoTv38u6qqJMtOmao6wGLAMDs7W3Nzc70KvHnfvSPP3bt9gTuOjfPjuLC13H/f3p+5aW7li5mCwWBA39+x1aD1/se5JnASOFlVH+9u381iKLyYZANA9/10t/4UcMXQ9pu6MUnSlPQOgar6DPB8kq/thq5j8dTQEWB3N7YbuKdbPgLsSnJRki3AVuChvvuXJI1v3OP/dwIfSPI64NPAD7IYLIeT3AI8C9wIUFXHkxxmMSgWgFur6pUx9y9JGsNYIVBVjwJnu/Bw3WvM3w/sH2efkqSV4zuGJalhhoAkNcwQkKSGGQKS1DBDQJIaZghIUsMMAUlqmCEgSQ0zBCSpYYaAJDXMEJCkhhkCktQwQ0CSGmYISFLDDAFJapghIEkNMwQkqWGGgCQ1zBCQpIYZApLUMENAkhpmCEhSw8YOgSRrkvxFkj/obl+W5P4kT3XfLx2ae1uSE0meTHL9uPuWJI1nJY4E3gU8MXR7H3C0qrYCR7vbJNkG7AKuBHYAdyZZswL7lyT1NFYIJNkEvAX47aHhncDBbvkgcMPQ+KGqermqngZOANeMs39J0njGPRL4VeBngS8Ojc1U1Qvd8meAmW55I/D80LyT3ZgkaUrW9t0wyXcDp6vqkSRzZ5tTVZWketz3HmAPwMzMDIPBoFeNe7cvjDx3Zt3y5q82Lffft/e+z8vzzfz8/KrppY/W++8dAsC3A9+b5M3AlwFvSPJ+4MUkG6rqhSQbgNPd/FPAFUPbb+rG/ouqOgAcAJidna25ubleBd68796R5+7dvsAdx8b5cVzYWu6/b+/P3DS38sVMwWAwoO/v2GrQev+9TwdV1W1VtamqNrN4wfdjVfX9wBFgdzdtN3BPt3wE2JXkoiRbgK3AQ70rlySN7Vz86Xc7cDjJLcCzwI0AVXU8yWHgcWABuLWqXjkH+5ckjWhFQqCqBsCgW/574LrXmLcf2L8S+5Qkjc93DEtSwwwBSWqYISBJDTMEJKlhhoAkNcwQkKSGGQKS1DBDQJIaZghIUsMMAUlqmCEgSQ0zBCSpYYaAJDXMEJCkhhkCktQwQ0CSGmYISFLDDAFJapghIEkNMwQkqWGGgCQ1zBCQpIYZApLUsN4hkOSKJH+U5PEkx5O8qxu/LMn9SZ7qvl86tM1tSU4keTLJ9SvRgCSpv3GOBBaAvVW1DXgTcGuSbcA+4GhVbQWOdrfp1u0CrgR2AHcmWTNO8ZKk8fQOgap6oao+0S3/E/AEsBHYCRzsph0EbuiWdwKHqurlqnoaOAFc03f/kqTxparGv5NkM/AAcBXwXFVd0o0H+IequiTJbwAPVtX7u3V3AfdV1d1nub89wB6AmZmZqw8dOtSrrmOnPjfy3Jl18OIXeu1mVWi5/769b9/4xpUvZgrm5+dZv379tMuYmtXa/7XXXvtIVc0uNW/tuDtKsh74PeCnqurzi//uL6qqSrLslKmqA8ABgNnZ2Zqbm+tV28377h157t7tC9xxbOwfxwWr5f779v7MTXMrX8wUDAYD+v6OrQat9z/Wq4OSfCmLAfCBqvpQN/xikg3d+g3A6W78FHDF0OabujFJ0pSM8+qgAHcBT1TVrwytOgLs7pZ3A/cMje9KclGSLcBW4KG++5ckjW+c4/9vB34AOJbk0W7s54HbgcNJbgGeBW4EqKrjSQ4Dj7P4yqJbq+qVMfYvSRpT7xCoqj8B8hqrr3uNbfYD+/vuUzqfbF7GNafz2d7tC8u6frbanK/9P3P7WyayH98xLEkNMwQkqWGGgCQ1zBCQpIYZApLUMENAkhpmCEhSwwwBSWqYISBJDTMEJKlhhoAkNcwQkKSGGQKS1DBDQJIaZghIUsMMAUlqmCEgSQ0zBCSpYYaAJDXMEJCkhhkCktQwQ0CSGjbxEEiyI8mTSU4k2Tfp/UuSXjXREEiyBvhN4H8D24C3J9k2yRokSa+a9JHANcCJqvp0Vf0LcAjYOeEaJEmdSYfARuD5odsnuzFJ0hSkqia3s+RtwI6q+uHu9g8A31JVP3HGvD3Anu7m1wJPTqC8y4G/m8B+zlct999y72D/q7X/r6qqL19q0tpJVDLkFHDF0O1N3dh/UlUHgAOTKgogycNVNTvJfZ5PWu6/5d7B/lvvf9Kng/4c2JpkS5LXAbuAIxOuQZLUmeiRQFUtJPkJ4P8Da4D3VtXxSdYgSXrVpE8HUVUfAT4y6f2OYKKnn85DLfffcu9g/033P9ELw5Kk84sfGyFJDWsuBJb62IokO5N8MsmjSR5O8h3TqPNcGPUjO5J8c5KF7iW9q8YIj/1cks91j/2jSX5xGnWeK6M8/t3P4NEkx5P88aRrPFdGeOx/ZuhxfyzJK0kum0atE1dVzXyxeDH6r4GvBl4H/CWw7Yw563n1NNnXA5+adt2T6n1o3sdYvG7ztmnXPeHHfg74g2nXOsX+LwEeB/5Hd/srpl33pHo/Y/73AB+bdt2T+mrtSGDJj62oqvnqngnAxcBquWgy6kd2vBP4PeD0JIubgNY/smSU/r8P+FBVPQdQVavlObDcx/7twAcnUtl5oLUQGOljK5K8NcmngHuBH5pQbefakr0n2Qi8FXjPBOualFE/suTbutOB9yW5cjKlTcQo/X8NcGmSQZJHkrxjYtWdWyN/XE2S1wM7WPxDqAkTf4nohaCqPgx8OMl3Ar8EfNeUS5qUXwV+rqq+mGTatUzDJ1g8FTKf5M3A7wNbp1zTJK0FrgauA9YBf5bkwar6q+mWNVHfA/xpVX122oVMSmshMNLHVvy7qnogyVcnubyqLvTPFhml91ngUBcAlwNvTrJQVb8/mRLPqSX7r6rPDy1/JMmdq+Sxh9Ee/5PA31fVS8BLSR4AvgG40ENgOb/3u2joVBDQ3IXhtcCngS28eoHoyjPm/E9evTD8TSw+WTLt2ifR+xnz38fqujA8ymP/lUOP/TXAc6vhsV9G/18HHO3mvh54DLhq2rVPovdu3huBzwIXT7vmSX41dSRQr/GxFUl+tFv/W8D/Ad6R5F+BLwD/t7pnyIVsxN5XrRH7fxvwY0kWWHzsd62Gxx5G67+qnkjyh8AngS8Cv11Vj02v6pWxjOf+W4GP1uKRUDN8x7AkNay1VwdJkoYYApLUMENAkhpmCEhSwwwBSWqYISBJDTMEJKlhhoAkNezfALhHb74YFj5UAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f2dcdcc6be0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "if verboseout:\n",
    "    dvalid.predict.hist(bins=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6508    0.0\n",
      "3460    1.0\n",
      "2945    0.0\n",
      "6663    0.0\n",
      "4871    0.0\n",
      "4380    1.0\n",
      "110     1.0\n",
      "1582    1.0\n",
      "6088    1.0\n",
      "1823    1.0\n",
      "Name: deltaval, dtype: float64\n",
      "337.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/paperspace/anaconda3/envs/fastai/lib/python3.6/site-packages/ipykernel_launcher.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n"
     ]
    }
   ],
   "source": [
    "# hand calculation of proportion correct guesses in validation set\n",
    "\n",
    "dvalid[\"deltaval\"] = abs(dvalid.target - dvalid.predround)\n",
    "print(dvalid[\"deltaval\"][:10])\n",
    "print(dvalid[\"deltaval\"].sum())\n",
    "# print(\"percentage correct\")\n",
    "# print((len(deltaval) - deltaval.sum())/len(deltaval))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CASE_NUMBER</th>\n",
       "      <th>CASE_OWNER_ALIAS</th>\n",
       "      <th>OPENED_DATE</th>\n",
       "      <th>SUPPORT_MISSION</th>\n",
       "      <th>PRODUCT_NAME</th>\n",
       "      <th>ACCOUNT_PRIORITY</th>\n",
       "      <th>OWNERSHIP_CHANGES</th>\n",
       "      <th>SEVERITY_LEVEL_NUMBER_FORMULA</th>\n",
       "      <th>ACCOUNT_NAME</th>\n",
       "      <th>BLUE_DIAMOND_ACCOUNT</th>\n",
       "      <th>PRODUCT_VERSION</th>\n",
       "      <th>OPERATING_SYSTEM</th>\n",
       "      <th>SUBJECT</th>\n",
       "      <th>CREATED_BY</th>\n",
       "      <th>LEGACY_PROBLEM_NUMBER</th>\n",
       "      <th>target</th>\n",
       "      <th>predround</th>\n",
       "      <th>predict</th>\n",
       "      <th>deltaval</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3460</th>\n",
       "      <td>TS000149858</td>\n",
       "      <td>247</td>\n",
       "      <td>3/19/2018</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1268</td>\n",
       "      <td>1</td>\n",
       "      <td>40</td>\n",
       "      <td>5</td>\n",
       "      <td>[3, 3386, 1694, 99, 36, 2, 876, 99, 20, 738, 293]</td>\n",
       "      <td>1819</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.523025</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4380</th>\n",
       "      <td>TS000162980</td>\n",
       "      <td>103</td>\n",
       "      <td>3/29/2018</td>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>988</td>\n",
       "      <td>1</td>\n",
       "      <td>40</td>\n",
       "      <td>5</td>\n",
       "      <td>[22, 499, 5, 642, 159, 7, 349]</td>\n",
       "      <td>1578</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.519243</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>110</th>\n",
       "      <td>TS000130911</td>\n",
       "      <td>112</td>\n",
       "      <td>03/05/2018</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>216</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>[10, 14, 105, 147]</td>\n",
       "      <td>439</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.525033</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1582</th>\n",
       "      <td>TS000117826</td>\n",
       "      <td>188</td>\n",
       "      <td>2/18/2018</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>812</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>[188, 64, 52, 2, 33, 320, 298, 1363, 4, 1, 60,...</td>\n",
       "      <td>1470</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.521524</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6088</th>\n",
       "      <td>TS000841054</td>\n",
       "      <td>212</td>\n",
       "      <td>4/27/2018</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>629</td>\n",
       "      <td>1</td>\n",
       "      <td>40</td>\n",
       "      <td>5</td>\n",
       "      <td>[745, 52, 2, 1774, 2292, 284, 867, 2, 90, 21, ...</td>\n",
       "      <td>412</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.504811</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1823</th>\n",
       "      <td>TS000121491</td>\n",
       "      <td>73</td>\n",
       "      <td>2/22/2018</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1171</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>[4660, 1]</td>\n",
       "      <td>1819</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.529827</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1512</th>\n",
       "      <td>TS000116911</td>\n",
       "      <td>63</td>\n",
       "      <td>2/15/2018</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>18</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>92</td>\n",
       "      <td>1</td>\n",
       "      <td>30</td>\n",
       "      <td>5</td>\n",
       "      <td>[]</td>\n",
       "      <td>1819</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.507433</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1680</th>\n",
       "      <td>TS000119274</td>\n",
       "      <td>1</td>\n",
       "      <td>2/19/2018</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>385</td>\n",
       "      <td>1</td>\n",
       "      <td>40</td>\n",
       "      <td>5</td>\n",
       "      <td>[1, 33]</td>\n",
       "      <td>1819</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.512129</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>269</th>\n",
       "      <td>TS000177773</td>\n",
       "      <td>166</td>\n",
       "      <td>04/11/2018</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1504</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>[164, 210, 313]</td>\n",
       "      <td>1819</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.471071</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1124</th>\n",
       "      <td>TS000112377</td>\n",
       "      <td>171</td>\n",
       "      <td>02/10/2018</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1307</td>\n",
       "      <td>1</td>\n",
       "      <td>40</td>\n",
       "      <td>5</td>\n",
       "      <td>[1, 767, 174, 284, 52, 2, 33]</td>\n",
       "      <td>523</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.516275</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3691</th>\n",
       "      <td>TS000167070</td>\n",
       "      <td>206</td>\n",
       "      <td>04/03/2018</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>[5647]</td>\n",
       "      <td>1819</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.527532</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2003</th>\n",
       "      <td>TS000123900</td>\n",
       "      <td>245</td>\n",
       "      <td>2/25/2018</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>586</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>[1]</td>\n",
       "      <td>1819</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.547123</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5593</th>\n",
       "      <td>TS000798036</td>\n",
       "      <td>175</td>\n",
       "      <td>4/20/2018</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1931</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>[41, 8, 32, 27, 41, 8, 29, 2, 19, 8, 51, 84, 116]</td>\n",
       "      <td>2240</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.549732</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5770</th>\n",
       "      <td>TS000824937</td>\n",
       "      <td>173</td>\n",
       "      <td>4/24/2018</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>729</td>\n",
       "      <td>1</td>\n",
       "      <td>29</td>\n",
       "      <td>2</td>\n",
       "      <td>[820, 51, 8, 452, 108, 110, 2102]</td>\n",
       "      <td>1730</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.511611</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7486</th>\n",
       "      <td>TS000940944</td>\n",
       "      <td>54</td>\n",
       "      <td>5/20/2018</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>178</td>\n",
       "      <td>1</td>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "      <td>[223, 37, 391, 291, 4488, 14, 86, 9, 163, 4489...</td>\n",
       "      <td>1819</td>\n",
       "      <td>1</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.483713</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3324</th>\n",
       "      <td>TS000147099</td>\n",
       "      <td>110</td>\n",
       "      <td>3/16/2018</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1606</td>\n",
       "      <td>1</td>\n",
       "      <td>40</td>\n",
       "      <td>5</td>\n",
       "      <td>[196, 339, 186]</td>\n",
       "      <td>1819</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.503479</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>TS000123376</td>\n",
       "      <td>112</td>\n",
       "      <td>2/23/2018</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1626</td>\n",
       "      <td>1</td>\n",
       "      <td>30</td>\n",
       "      <td>2</td>\n",
       "      <td>[58, 98]</td>\n",
       "      <td>1819</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.555881</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2705</th>\n",
       "      <td>TS000135994</td>\n",
       "      <td>211</td>\n",
       "      <td>03/07/2018</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1885</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>[1077, 1078, 269, 718, 2662, 7, 565, 2, 316, 2...</td>\n",
       "      <td>150</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.516933</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1357</th>\n",
       "      <td>TS000115037</td>\n",
       "      <td>229</td>\n",
       "      <td>2/14/2018</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>10</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1735</td>\n",
       "      <td>1</td>\n",
       "      <td>40</td>\n",
       "      <td>5</td>\n",
       "      <td>[1, 1582, 14, 13, 107, 1153]</td>\n",
       "      <td>1819</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.526485</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3358</th>\n",
       "      <td>TS000148060</td>\n",
       "      <td>230</td>\n",
       "      <td>3/18/2018</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>618</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>[1977, 583, 17, 5824, 2807, 2807, 22, 333, 25,...</td>\n",
       "      <td>336</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.579538</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      CASE_NUMBER  CASE_OWNER_ALIAS OPENED_DATE  SUPPORT_MISSION  \\\n",
       "3460  TS000149858               247   3/19/2018                0   \n",
       "4380  TS000162980               103   3/29/2018                0   \n",
       "110   TS000130911               112  03/05/2018                0   \n",
       "1582  TS000117826               188   2/18/2018                0   \n",
       "6088  TS000841054               212   4/27/2018                0   \n",
       "1823  TS000121491                73   2/22/2018                0   \n",
       "1512  TS000116911                63   2/15/2018                0   \n",
       "1680  TS000119274                 1   2/19/2018                0   \n",
       "269   TS000177773               166  04/11/2018                0   \n",
       "1124  TS000112377               171  02/10/2018                0   \n",
       "3691  TS000167070               206  04/03/2018                0   \n",
       "2003  TS000123900               245   2/25/2018                0   \n",
       "5593  TS000798036               175   4/20/2018                0   \n",
       "5770  TS000824937               173   4/24/2018                0   \n",
       "7486  TS000940944                54   5/20/2018                0   \n",
       "3324  TS000147099               110   3/16/2018                0   \n",
       "75    TS000123376               112   2/23/2018                0   \n",
       "2705  TS000135994               211  03/07/2018                0   \n",
       "1357  TS000115037               229   2/14/2018                0   \n",
       "3358  TS000148060               230   3/18/2018                0   \n",
       "\n",
       "      PRODUCT_NAME  ACCOUNT_PRIORITY  OWNERSHIP_CHANGES  \\\n",
       "3460             6                 2                  1   \n",
       "4380            11                18                  0   \n",
       "110              6                 0                  0   \n",
       "1582             6                10                  1   \n",
       "6088             6                10                  0   \n",
       "1823             6                10                  2   \n",
       "1512             6                18                  2   \n",
       "1680             3                18                  0   \n",
       "269              6                 4                  2   \n",
       "1124             6                 2                  1   \n",
       "3691             6                 2                  1   \n",
       "2003             6                15                  1   \n",
       "5593             6                 7                  0   \n",
       "5770             6                 0                  0   \n",
       "7486             6                 0                  0   \n",
       "3324             6                 7                  0   \n",
       "75               6                 2                  1   \n",
       "2705             6                 8                  0   \n",
       "1357             6                10                  1   \n",
       "3358             6                 7                  3   \n",
       "\n",
       "      SEVERITY_LEVEL_NUMBER_FORMULA  ACCOUNT_NAME  BLUE_DIAMOND_ACCOUNT  \\\n",
       "3460                              1          1268                     1   \n",
       "4380                              0           988                     1   \n",
       "110                               1           216                     1   \n",
       "1582                              1           812                     1   \n",
       "6088                              0           629                     1   \n",
       "1823                              2          1171                     1   \n",
       "1512                              2            92                     1   \n",
       "1680                              1           385                     1   \n",
       "269                               0          1504                     1   \n",
       "1124                              1          1307                     1   \n",
       "3691                              1             2                     1   \n",
       "2003                              1           586                     1   \n",
       "5593                              1          1931                     1   \n",
       "5770                              1           729                     1   \n",
       "7486                              1           178                     1   \n",
       "3324                              1          1606                     1   \n",
       "75                                1          1626                     1   \n",
       "2705                              1          1885                     1   \n",
       "1357                              1          1735                     1   \n",
       "3358                              1           618                     1   \n",
       "\n",
       "      PRODUCT_VERSION  OPERATING_SYSTEM  \\\n",
       "3460               40                 5   \n",
       "4380               40                 5   \n",
       "110                 5                 2   \n",
       "1582                5                 2   \n",
       "6088               40                 5   \n",
       "1823                5                 0   \n",
       "1512               30                 5   \n",
       "1680               40                 5   \n",
       "269                 5                 0   \n",
       "1124               40                 5   \n",
       "3691                5                 2   \n",
       "2003                5                 4   \n",
       "5593                5                 2   \n",
       "5770               29                 2   \n",
       "7486               30                 0   \n",
       "3324               40                 5   \n",
       "75                 30                 2   \n",
       "2705                7                 2   \n",
       "1357               40                 5   \n",
       "3358                2                 2   \n",
       "\n",
       "                                                SUBJECT  CREATED_BY  \\\n",
       "3460  [3, 3386, 1694, 99, 36, 2, 876, 99, 20, 738, 293]        1819   \n",
       "4380                     [22, 499, 5, 642, 159, 7, 349]        1578   \n",
       "110                                  [10, 14, 105, 147]         439   \n",
       "1582  [188, 64, 52, 2, 33, 320, 298, 1363, 4, 1, 60,...        1470   \n",
       "6088  [745, 52, 2, 1774, 2292, 284, 867, 2, 90, 21, ...         412   \n",
       "1823                                          [4660, 1]        1819   \n",
       "1512                                                 []        1819   \n",
       "1680                                            [1, 33]        1819   \n",
       "269                                     [164, 210, 313]        1819   \n",
       "1124                      [1, 767, 174, 284, 52, 2, 33]         523   \n",
       "3691                                             [5647]        1819   \n",
       "2003                                                [1]        1819   \n",
       "5593  [41, 8, 32, 27, 41, 8, 29, 2, 19, 8, 51, 84, 116]        2240   \n",
       "5770                  [820, 51, 8, 452, 108, 110, 2102]        1730   \n",
       "7486  [223, 37, 391, 291, 4488, 14, 86, 9, 163, 4489...        1819   \n",
       "3324                                    [196, 339, 186]        1819   \n",
       "75                                             [58, 98]        1819   \n",
       "2705  [1077, 1078, 269, 718, 2662, 7, 565, 2, 316, 2...         150   \n",
       "1357                       [1, 1582, 14, 13, 107, 1153]        1819   \n",
       "3358  [1977, 583, 17, 5824, 2807, 2807, 22, 333, 25,...         336   \n",
       "\n",
       "      LEGACY_PROBLEM_NUMBER  target  predround   predict  deltaval  \n",
       "3460                      1     0.0          1  0.523025       1.0  \n",
       "4380                      0     0.0          1  0.519243       1.0  \n",
       "110                       0     0.0          1  0.525033       1.0  \n",
       "1582                      0     0.0          1  0.521524       1.0  \n",
       "6088                      0     0.0          1  0.504811       1.0  \n",
       "1823                      1     0.0          1  0.529827       1.0  \n",
       "1512                      1     0.0          1  0.507433       1.0  \n",
       "1680                      1     0.0          1  0.512129       1.0  \n",
       "269                       1     1.0          0  0.471071       1.0  \n",
       "1124                      0     0.0          1  0.516275       1.0  \n",
       "3691                      1     0.0          1  0.527532       1.0  \n",
       "2003                      1     0.0          1  0.547123       1.0  \n",
       "5593                      0     0.0          1  0.549732       1.0  \n",
       "5770                      0     0.0          1  0.511611       1.0  \n",
       "7486                      1     1.0          0  0.483713       1.0  \n",
       "3324                      1     0.0          1  0.503479       1.0  \n",
       "75                        1     0.0          1  0.555881       1.0  \n",
       "2705                      0     0.0          1  0.516933       1.0  \n",
       "1357                      1     0.0          1  0.526485       1.0  \n",
       "3358                      0     0.0          1  0.579538       1.0  "
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# get subset of dataframe with wrong guesses\n",
    "# k1 = df.loc[(df.Product == p_id)\n",
    "dvalidwrong = dvalid.loc[(dvalid.deltaval == 1)]\n",
    "dvalidright = dvalid.loc[(dvalid.deltaval == 0)]\n",
    "dvalidwrong.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CASE_NUMBER</th>\n",
       "      <th>CASE_OWNER_ALIAS</th>\n",
       "      <th>OPENED_DATE</th>\n",
       "      <th>SUPPORT_MISSION</th>\n",
       "      <th>PRODUCT_NAME</th>\n",
       "      <th>ACCOUNT_PRIORITY</th>\n",
       "      <th>OWNERSHIP_CHANGES</th>\n",
       "      <th>SEVERITY_LEVEL_NUMBER_FORMULA</th>\n",
       "      <th>ACCOUNT_NAME</th>\n",
       "      <th>BLUE_DIAMOND_ACCOUNT</th>\n",
       "      <th>PRODUCT_VERSION</th>\n",
       "      <th>OPERATING_SYSTEM</th>\n",
       "      <th>SUBJECT</th>\n",
       "      <th>CREATED_BY</th>\n",
       "      <th>LEGACY_PROBLEM_NUMBER</th>\n",
       "      <th>target</th>\n",
       "      <th>predround</th>\n",
       "      <th>predict</th>\n",
       "      <th>deltaval</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6508</th>\n",
       "      <td>TS000877452</td>\n",
       "      <td>236</td>\n",
       "      <td>05/04/2018</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1001</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>[1, 35, 50]</td>\n",
       "      <td>1912</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.488539</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2945</th>\n",
       "      <td>TS000140258</td>\n",
       "      <td>0</td>\n",
       "      <td>03/12/2018</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1129</td>\n",
       "      <td>1</td>\n",
       "      <td>40</td>\n",
       "      <td>5</td>\n",
       "      <td>[2038, 21, 22, 436]</td>\n",
       "      <td>1007</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.388055</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6663</th>\n",
       "      <td>TS000883281</td>\n",
       "      <td>108</td>\n",
       "      <td>05/08/2018</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1731</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>5</td>\n",
       "      <td>[84, 2922, 845]</td>\n",
       "      <td>50</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.397696</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4871</th>\n",
       "      <td>TS000176645</td>\n",
       "      <td>7</td>\n",
       "      <td>04/11/2018</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>653</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>[1931, 982, 545, 1836, 1358, 8, 7, 304, 274, 2...</td>\n",
       "      <td>1396</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.365404</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4800</th>\n",
       "      <td>TS000175067</td>\n",
       "      <td>95</td>\n",
       "      <td>04/10/2018</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>649</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>[601, 194, 7, 1, 188]</td>\n",
       "      <td>1799</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.409905</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>285</th>\n",
       "      <td>TS000211851</td>\n",
       "      <td>221</td>\n",
       "      <td>4/16/2018</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1306</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>2</td>\n",
       "      <td>[154, 4, 2290]</td>\n",
       "      <td>866</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.411263</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>782</th>\n",
       "      <td>TS000108074</td>\n",
       "      <td>34</td>\n",
       "      <td>02/06/2018</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1708</td>\n",
       "      <td>1</td>\n",
       "      <td>40</td>\n",
       "      <td>5</td>\n",
       "      <td>[2975, 182, 327]</td>\n",
       "      <td>1819</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.423175</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3766</th>\n",
       "      <td>TS000168267</td>\n",
       "      <td>95</td>\n",
       "      <td>04/04/2018</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1917</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>[386, 76, 219, 59, 9, 169, 535, 750, 234, 249,...</td>\n",
       "      <td>2148</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.401332</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4870</th>\n",
       "      <td>TS000176620</td>\n",
       "      <td>82</td>\n",
       "      <td>04/11/2018</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1789</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>[75, 12, 10]</td>\n",
       "      <td>2179</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.490349</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1474</th>\n",
       "      <td>TS000116479</td>\n",
       "      <td>199</td>\n",
       "      <td>2/15/2018</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2032</td>\n",
       "      <td>1</td>\n",
       "      <td>30</td>\n",
       "      <td>0</td>\n",
       "      <td>[184, 82, 137, 1, 118, 320, 551, 144, 16, 158,...</td>\n",
       "      <td>1327</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.450237</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>959</th>\n",
       "      <td>TS000110205</td>\n",
       "      <td>169</td>\n",
       "      <td>02/07/2018</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>746</td>\n",
       "      <td>1</td>\n",
       "      <td>40</td>\n",
       "      <td>5</td>\n",
       "      <td>[3, 39, 1, 126, 526, 154, 7, 46, 511, 2716]</td>\n",
       "      <td>461</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.349724</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3948</th>\n",
       "      <td>TS000153767</td>\n",
       "      <td>66</td>\n",
       "      <td>3/22/2018</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>98</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>[4218, 135, 4219, 1303, 4220, 1738, 79, 37, 27...</td>\n",
       "      <td>1472</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.414864</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>501</th>\n",
       "      <td>TS000105174</td>\n",
       "      <td>212</td>\n",
       "      <td>02/01/2018</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>500</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>[157, 272, 175, 2, 166, 37, 16, 157, 272, 175,...</td>\n",
       "      <td>608</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.447160</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>815</th>\n",
       "      <td>TS000108443</td>\n",
       "      <td>217</td>\n",
       "      <td>02/06/2018</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>18</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>822</td>\n",
       "      <td>1</td>\n",
       "      <td>7</td>\n",
       "      <td>5</td>\n",
       "      <td>[787, 316, 37, 65, 4040, 37, 2415, 202]</td>\n",
       "      <td>1819</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.429659</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3723</th>\n",
       "      <td>TS000167564</td>\n",
       "      <td>56</td>\n",
       "      <td>04/03/2018</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>323</td>\n",
       "      <td>1</td>\n",
       "      <td>30</td>\n",
       "      <td>2</td>\n",
       "      <td>[1383, 168, 22, 68]</td>\n",
       "      <td>1819</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.453325</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1165</th>\n",
       "      <td>TS000112741</td>\n",
       "      <td>86</td>\n",
       "      <td>02/09/2018</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1800</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>[111, 12, 1, 99, 24, 109, 126, 4, 46]</td>\n",
       "      <td>1819</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.426659</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7126</th>\n",
       "      <td>TS000920227</td>\n",
       "      <td>264</td>\n",
       "      <td>5/14/2018</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>661</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>[49, 87, 192, 3, 23, 165, 2512, 18, 143, 4352,...</td>\n",
       "      <td>1819</td>\n",
       "      <td>1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.405113</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3564</th>\n",
       "      <td>TS000164712</td>\n",
       "      <td>60</td>\n",
       "      <td>04/01/2018</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>18</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>20</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>[104, 17, 1037, 640, 123, 1, 19, 6]</td>\n",
       "      <td>679</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.408225</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6431</th>\n",
       "      <td>TS000874282</td>\n",
       "      <td>230</td>\n",
       "      <td>05/03/2018</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>18</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1791</td>\n",
       "      <td>1</td>\n",
       "      <td>40</td>\n",
       "      <td>5</td>\n",
       "      <td>[5911, 171, 5912, 5913, 5914, 5915, 14, 13, 52...</td>\n",
       "      <td>199</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.429674</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3267</th>\n",
       "      <td>TS000146190</td>\n",
       "      <td>88</td>\n",
       "      <td>3/15/2018</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2150</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>[3686, 53, 1, 947, 15]</td>\n",
       "      <td>348</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.488816</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      CASE_NUMBER  CASE_OWNER_ALIAS OPENED_DATE  SUPPORT_MISSION  \\\n",
       "6508  TS000877452               236  05/04/2018                0   \n",
       "2945  TS000140258                 0  03/12/2018                0   \n",
       "6663  TS000883281               108  05/08/2018                0   \n",
       "4871  TS000176645                 7  04/11/2018                0   \n",
       "4800  TS000175067                95  04/10/2018                0   \n",
       "285   TS000211851               221   4/16/2018                0   \n",
       "782   TS000108074                34  02/06/2018                0   \n",
       "3766  TS000168267                95  04/04/2018                0   \n",
       "4870  TS000176620                82  04/11/2018                0   \n",
       "1474  TS000116479               199   2/15/2018                0   \n",
       "959   TS000110205               169  02/07/2018                0   \n",
       "3948  TS000153767                66   3/22/2018                0   \n",
       "501   TS000105174               212  02/01/2018                0   \n",
       "815   TS000108443               217  02/06/2018                0   \n",
       "3723  TS000167564                56  04/03/2018                0   \n",
       "1165  TS000112741                86  02/09/2018                0   \n",
       "7126  TS000920227               264   5/14/2018                0   \n",
       "3564  TS000164712                60  04/01/2018                0   \n",
       "6431  TS000874282               230  05/03/2018                0   \n",
       "3267  TS000146190                88   3/15/2018                0   \n",
       "\n",
       "      PRODUCT_NAME  ACCOUNT_PRIORITY  OWNERSHIP_CHANGES  \\\n",
       "6508             6                18                  0   \n",
       "2945             3                18                  0   \n",
       "6663             6                 4                  2   \n",
       "4871             6                18                  0   \n",
       "4800             6                 4                  0   \n",
       "285              6                 4                  1   \n",
       "782              6                15                  0   \n",
       "3766             6                18                  0   \n",
       "4870             6                18                  0   \n",
       "1474             6                 7                  0   \n",
       "959              6                18                  0   \n",
       "3948             6                18                  0   \n",
       "501              6                18                  0   \n",
       "815              6                18                  2   \n",
       "3723             6                18                  0   \n",
       "1165             6                10                  0   \n",
       "7126             6                 4                  0   \n",
       "3564             6                18                  1   \n",
       "6431             6                18                  1   \n",
       "3267             6                18                  0   \n",
       "\n",
       "      SEVERITY_LEVEL_NUMBER_FORMULA  ACCOUNT_NAME  BLUE_DIAMOND_ACCOUNT  \\\n",
       "6508                              1          1001                     1   \n",
       "2945                              3          1129                     1   \n",
       "6663                              2          1731                     1   \n",
       "4871                              2           653                     1   \n",
       "4800                              2           649                     1   \n",
       "285                               2          1306                     1   \n",
       "782                               2          1708                     1   \n",
       "3766                              2          1917                     1   \n",
       "4870                              1          1789                     1   \n",
       "1474                              1          2032                     1   \n",
       "959                               2           746                     1   \n",
       "3948                              1            98                     1   \n",
       "501                               1           500                     0   \n",
       "815                               2           822                     1   \n",
       "3723                              1           323                     1   \n",
       "1165                              2          1800                     1   \n",
       "7126                              2           661                     1   \n",
       "3564                              3            20                     1   \n",
       "6431                              1          1791                     1   \n",
       "3267                              0          2150                     1   \n",
       "\n",
       "      PRODUCT_VERSION  OPERATING_SYSTEM  \\\n",
       "6508                7                 2   \n",
       "2945               40                 5   \n",
       "6663                7                 5   \n",
       "4871                5                 0   \n",
       "4800                5                 2   \n",
       "285                 7                 2   \n",
       "782                40                 5   \n",
       "3766                2                 0   \n",
       "4870                5                 0   \n",
       "1474               30                 0   \n",
       "959                40                 5   \n",
       "3948                7                 0   \n",
       "501                 5                 5   \n",
       "815                 7                 5   \n",
       "3723               30                 2   \n",
       "1165                5                 0   \n",
       "7126                5                 0   \n",
       "3564                2                 4   \n",
       "6431               40                 5   \n",
       "3267                5                 2   \n",
       "\n",
       "                                                SUBJECT  CREATED_BY  \\\n",
       "6508                                        [1, 35, 50]        1912   \n",
       "2945                                [2038, 21, 22, 436]        1007   \n",
       "6663                                    [84, 2922, 845]          50   \n",
       "4871  [1931, 982, 545, 1836, 1358, 8, 7, 304, 274, 2...        1396   \n",
       "4800                              [601, 194, 7, 1, 188]        1799   \n",
       "285                                      [154, 4, 2290]         866   \n",
       "782                                    [2975, 182, 327]        1819   \n",
       "3766  [386, 76, 219, 59, 9, 169, 535, 750, 234, 249,...        2148   \n",
       "4870                                       [75, 12, 10]        2179   \n",
       "1474  [184, 82, 137, 1, 118, 320, 551, 144, 16, 158,...        1327   \n",
       "959         [3, 39, 1, 126, 526, 154, 7, 46, 511, 2716]         461   \n",
       "3948  [4218, 135, 4219, 1303, 4220, 1738, 79, 37, 27...        1472   \n",
       "501   [157, 272, 175, 2, 166, 37, 16, 157, 272, 175,...         608   \n",
       "815             [787, 316, 37, 65, 4040, 37, 2415, 202]        1819   \n",
       "3723                                [1383, 168, 22, 68]        1819   \n",
       "1165              [111, 12, 1, 99, 24, 109, 126, 4, 46]        1819   \n",
       "7126  [49, 87, 192, 3, 23, 165, 2512, 18, 143, 4352,...        1819   \n",
       "3564                [104, 17, 1037, 640, 123, 1, 19, 6]         679   \n",
       "6431  [5911, 171, 5912, 5913, 5914, 5915, 14, 13, 52...         199   \n",
       "3267                             [3686, 53, 1, 947, 15]         348   \n",
       "\n",
       "      LEGACY_PROBLEM_NUMBER  target  predround   predict  deltaval  \n",
       "6508                      0     0.0          0  0.488539       0.0  \n",
       "2945                      0     0.0          0  0.388055       0.0  \n",
       "6663                      0     0.0          0  0.397696       0.0  \n",
       "4871                      0     0.0          0  0.365404       0.0  \n",
       "4800                      0     0.0          0  0.409905       0.0  \n",
       "285                       0     0.0          0  0.411263       0.0  \n",
       "782                       1     0.0          0  0.423175       0.0  \n",
       "3766                      0     0.0          0  0.401332       0.0  \n",
       "4870                      0     0.0          0  0.490349       0.0  \n",
       "1474                      0     0.0          0  0.450237       0.0  \n",
       "959                       0     0.0          0  0.349724       0.0  \n",
       "3948                      0     0.0          0  0.414864       0.0  \n",
       "501                       0     0.0          0  0.447160       0.0  \n",
       "815                       1     0.0          0  0.429659       0.0  \n",
       "3723                      1     0.0          0  0.453325       0.0  \n",
       "1165                      1     0.0          0  0.426659       0.0  \n",
       "7126                      1     0.0          0  0.405113       0.0  \n",
       "3564                      0     0.0          0  0.408225       0.0  \n",
       "6431                      0     0.0          0  0.429674       0.0  \n",
       "3267                      0     0.0          0  0.488816       0.0  "
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dvalidright.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7f2dcdb10128>"
      ]
     },
     "execution_count": 138,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAADvVJREFUeJzt3X+s3fVdx/Hna2VjjEUHYV6bgpbFaiyr27SyxRlzIxrqcJbFBTtxKxHTaJiZSaMW/9AY04R/MPtHNM1c1kQz0mQ/IGP+IN2Oi2YOYbJBYUgdMCAFsi06Swx62ds/+h3cVco99/Sec9r3fT6Sm/s93/P9ns/7ne+3r/vt93zP96SqkCT19Yp5FyBJmi6DXpKaM+glqTmDXpKaM+glqTmDXpKaM+glqTmDXpKaM+glqblz5l0AwEUXXVSbN2+eyVjPPvss559//kzGOtOs597B/tdz/117v+eee75eVa9fabkzIug3b97M3XffPZOxRqMRi4uLMxnrTLOeewf7X8/9d+09yWPjLOepG0lqzqCXpOYMeklqzqCXpOYMeklqzqCXpOYMeklqzqCXpOYMeklq7oz4ZKzOHpv33TGXcR+96aq5jCt14BG9JDVn0EtScwa9JDVn0EtScwa9JDVn0EtScwa9JDXndfQ6K6zF9ft7ty1x3QSv4zX8Ott5RC9JzRn0ktScQS9JzRn0ktScQS9JzRn0ktScQS9JzRn0ktScQS9JzRn0ktScQS9JzRn0ktScQS9JzRn0ktTc2EGfZEOSf03yqeHxhUnuTPLw8PuCZcvemORokoeSXDmNwiVJ41nNEf0HgAeXPd4HHK6qLcDh4TFJtgK7gMuAHcAtSTasTbmSpNUaK+iTXAxcBXxo2eydwMFh+iBw9bL5t1bVc1X1CHAUuHxtypUkrda4R/QfBH4P+PayeQtVdWyYfgpYGKY3AY8vW+6JYZ4kaQ5W/CrBJL8IPFNV9yRZfKllqqqS1GoGTrIH2AOwsLDAaDRazeoTO378+MzGOtOsRe97ty2tTTFzsHDeZPV32V/c90fzLmNuxvnO2LcDv5TkHcCrge9J8lfA00k2VtWxJBuBZ4blnwQuWbb+xcO871JVB4ADANu3b6/FxcXJu1iF0WjErMY606xF75N85+qZYu+2JW6+b/Vfk/zotYtrX8wcuO8vzruMuVnx1E1V3VhVF1fVZk68yfqZqvo14HZg97DYbuC2Yfp2YFeSc5NcCmwB7lrzyiVJY1n94c2LbgIOJbkeeAy4BqCqjiQ5BDwALAE3VNXzp12pJGkiqwr6qhoBo2H6G8AVp1huP7D/NGuTJK0BPxkrSc0Z9JLUnEEvSc0Z9JLUnEEvSc0Z9JLUnEEvSc0Z9JLUnEEvSc0Z9JLUnEEvSc0Z9JLUnEEvSc0Z9JLUnEEvSc0Z9JLUnEEvSc0Z9JLUnEEvSc0Z9JLUnEEvSc0Z9JLUnEEvSc0Z9JLUnEEvSc2dM+8CtHqb990x0Xp7ty1x3YTrSjp7eUQvSc0Z9JLUnEEvSc0Z9JLUnEEvSc0Z9JLUnEEvSc0Z9JLUnEEvSc0Z9JLU3IpBn+TVSe5K8qUkR5L88TD/wiR3Jnl4+H3BsnVuTHI0yUNJrpxmA5KklzfOEf1zwM9W1ZuANwM7krwN2AccrqotwOHhMUm2AruAy4AdwC1JNkyjeEnSylYM+jrh+PDwlcNPATuBg8P8g8DVw/RO4Naqeq6qHgGOApevadWSpLGNdY4+yYYk9wLPAHdW1ReAhao6NizyFLAwTG8CHl+2+hPDPEnSHIx1m+Kqeh54c5LXAZ9I8saTnq8ktZqBk+wB9gAsLCwwGo1Ws/rEjh8/PrOxpmXvtqWJ1ls4b/J1O5i0/7N9f/mODvv+pNZz77DK+9FX1X8k+Swnzr0/nWRjVR1LspETR/sATwKXLFvt4mHeya91ADgAsH379lpcXJyg/NUbjUbMaqxpmfSe8nu3LXHzfev3Kwgm7f/RaxfXvpg56LDvT2o99w7jXXXz+uFIniTnAT8PfAW4Hdg9LLYbuG2Yvh3YleTcJJcCW4C71rpwSdJ4xjm82QgcHK6ceQVwqKo+leTzwKEk1wOPAdcAVNWRJIeAB4Al4Ibh1I8kaQ5WDPqq+jLwlpeY/w3gilOssx/Yf9rVSZJOm5+MlaTmDHpJas6gl6TmDHpJas6gl6TmDHpJas6gl6TmDHpJas6gl6TmDHpJas6gl6TmDHpJas6gl6TmDHpJas6gl6TmDHpJas6gl6TmDHpJas6gl6TmDHpJas6gl6TmDHpJas6gl6TmDHpJas6gl6TmDHpJas6gl6TmDHpJas6gl6TmDHpJas6gl6TmDHpJas6gl6TmDHpJas6gl6TmDHpJam7FoE9ySZLPJnkgyZEkHxjmX5jkziQPD78vWLbOjUmOJnkoyZXTbECS9PLGOaJfAvZW1VbgbcANSbYC+4DDVbUFODw8ZnhuF3AZsAO4JcmGaRQvSVrZikFfVceq6ovD9H8BDwKbgJ3AwWGxg8DVw/RO4Naqeq6qHgGOApevdeGSpPGs6hx9ks3AW4AvAAtVdWx46ilgYZjeBDy+bLUnhnmSpDk4Z9wFk7wW+BjwO1X1rSQvPFdVlaRWM3CSPcAegIWFBUaj0WpWn9jx48dnNta07N22NNF6C+dNvm4Hk/Z/tu8v39Fh35/Ueu4dxgz6JK/kRMj/dVV9fJj9dJKNVXUsyUbgmWH+k8Aly1a/eJj3XarqAHAAYPv27bW4uDhZB6s0Go2Y1VjTct2+OyZab++2JW6+b+y/7e1M2v+j1y6ufTFz0GHfn9R67h3Gu+omwF8CD1bVny576nZg9zC9G7ht2fxdSc5NcimwBbhr7UqWJK3GOIc3bwfeC9yX5N5h3h8ANwGHklwPPAZcA1BVR5IcAh7gxBU7N1TV82teuSRpLCsGfVX9I5BTPH3FKdbZD+w/jbokSWvET8ZKUnMGvSQ1Z9BLUnMGvSQ1Z9BLUnMGvSQ1Z9BLUnMGvSQ1Z9BLUnMGvSQ1Z9BLUnMGvSQ1Z9BLUnMGvSQ1Z9BLUnMGvSQ1Z9BLUnMGvSQ1Z9BLUnPjfDm4tK5t3nfHXMZ99Kar5jKu+vGIXpKaM+glqTmDXpKaM+glqTmDXpKaM+glqTmDXpKaM+glqTmDXpKaM+glqTmDXpKaM+glqTmDXpKaM+glqTmDXpKaM+glqbkVgz7Jh5M8k+T+ZfMuTHJnkoeH3xcse+7GJEeTPJTkymkVLkkazzhH9B8Bdpw0bx9wuKq2AIeHxyTZCuwCLhvWuSXJhjWrVpK0aisGfVV9DvjmSbN3AgeH6YPA1cvm31pVz1XVI8BR4PI1qlWSNIFJz9EvVNWxYfopYGGY3gQ8vmy5J4Z5kqQ5Oe0vB6+qSlKrXS/JHmAPwMLCAqPR6HRLGcvx48dnNta07N22NNF6C+dNvm4HZ1v/a72fdtj3J7Wee4fJg/7pJBur6liSjcAzw/wngUuWLXfxMO//qaoDwAGA7du31+Li4oSlrM5oNGJWY03LdfvumGi9vduWuPm+0/7bftY62/p/9NrFNX29Dvv+pNZz7zD5qZvbgd3D9G7gtmXzdyU5N8mlwBbgrtMrUZJ0OlY8vEnyUWARuCjJE8AfATcBh5JcDzwGXANQVUeSHAIeAJaAG6rq+SnVLkkaw4pBX1XvOcVTV5xi+f3A/tMpSpK0dvxkrCQ1Z9BLUnMGvSQ1Z9BLUnMGvSQ1d/Z8euQMtHnCDy5J0ix5RC9JzRn0ktScQS9JzRn0ktScQS9JzRn0ktScQS9JzRn0ktScQS9JzRn0ktScQS9JzRn0ktScQS9JzRn0ktScQS9JzRn0ktScQS9JzRn0ktScQS9JzRn0ktScQS9JzRn0ktScQS9JzRn0ktScQS9JzZ0z7wIkvbTN++5Y09fbu22J68Z8zUdvumpNx9Z8eUQvSc0Z9JLUnEEvSc0Z9JLU3NSCPsmOJA8lOZpk37TGkSS9vKkEfZINwJ8BvwBsBd6TZOs0xpIkvbxpXV55OXC0qr4KkORWYCfwwDQGW81laKu5xExar9b60s5xeVnndEwr6DcBjy97/ATw1imNJamJaf2BOZMP8Gbxxy1VtfYvmrwb2FFVvzE8fi/w1qp6/7Jl9gB7hoc/Ajy05oW8tIuAr89orDPNeu4d7H8999+19x+sqtevtNC0juifBC5Z9vjiYd4LquoAcGBK459Skruravusxz0TrOfewf7Xc//ruXeY3lU3/wJsSXJpklcBu4DbpzSWJOllTOWIvqqWkrwf+DtgA/DhqjoyjbEkSS9vajc1q6pPA5+e1uufhpmfLjqDrOfewf7Xc//ruffpvBkrSTpzeAsESWquZdCvdPuFJDuTfDnJvUnuTvLT86hzWsa9/USSn0yyNFwO28YY238xyX8O2//eJH84jzqnYZxtP/R/b5IjSf5h1jVO0xjb/neXbff7kzyf5MJ51DpTVdXqhxNv/v478AbgVcCXgK0nLfNaXjxt9WPAV+Zd9yz7X7bcZzjxPsq75133jLf/IvCpedc6p95fx4lPqP/A8Pj75l33LPs/afl3Ap+Zd92z+Ol4RP/C7Req6n+A79x+4QVVdbyGLQ2cD3R6o2LF/ge/DXwMeGaWxc3AuP13NE7vvwp8vKq+BlBVnbb/arf9e4CPzqSyOesY9C91+4VNJy+U5F1JvgLcAfz6jGqbhRX7T7IJeBfw5zOsa1bG2v7ATw2n7/4myWWzKW3qxun9h4ELkoyS3JPkfTOrbvrG3fYkeQ2wgxMHO+2t2++MrapPAJ9I8jPAnwA/N+eSZumDwO9X1beTzLuWefgiJ05dHE/yDuCTwJY51zQr5wA/AVwBnAd8Psk/V9W/zbesmXsn8E9V9c15FzILHYN+xdsvLFdVn0vyhiQXVVWHe2GM0/924NYh5C8C3pFkqao+OZsSp2qc2298a9n0p5Pc0mT7j7PtnwC+UVXPAs8m+RzwJqBD0K/m3/4u1slpG6Dlm7HnAF8FLuXFN2QuO2mZH+LFN2N/nBM7Q+Zd+6z6P2n5j9Drzdhxtv/3L9v+lwNf67D9x+z9R4HDw7KvAe4H3jjv2mfV/7Dc9wLfBM6fd82z+ml3RF+nuP1Ckt8cnv8L4JeB9yX5X+C/gV+pYQ84243Zf1tj9v9u4LeSLHFi++/qsP3H6b2qHkzyt8CXgW8DH6qq++dX9dpZxb7/LuDv68T/atYFPxkrSc11vOpGkrSMQS9JzRn0ktScQS9JzRn0ktScQS9JzRn0ktScQS9Jzf0fTAT8OUmX02wAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f2dcdb10630>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# dvalid.hist(range = (0,5))\n",
    "# apar_ds.Time_to_relief.hist(range = (0,5))\n",
    "dvalid.predict.hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ,encoding = \"ISO-8859-1\"\n",
    "if dsxmode == False:\n",
    "    dvalidwrong.to_csv('/home/paperspace/data/dmprediction/dvalidwrongmay20.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "810     0.0\n",
       "7110    0.0\n",
       "3967    0.0\n",
       "3800    0.0\n",
       "627     0.0\n",
       "230     0.0\n",
       "3195    0.0\n",
       "361     0.0\n",
       "5882    0.0\n",
       "6011    0.0\n",
       "4215    0.0\n",
       "3192    0.0\n",
       "591     0.0\n",
       "5788    0.0\n",
       "1886    0.0\n",
       "5492    0.0\n",
       "7373    0.0\n",
       "1107    0.0\n",
       "824     0.0\n",
       "2086    0.0\n",
       "6524    0.0\n",
       "2767    0.0\n",
       "5654    0.0\n",
       "648     0.0\n",
       "1400    0.0\n",
       "5226    0.0\n",
       "4942    0.0\n",
       "2969    0.0\n",
       "3590    0.0\n",
       "3861    0.0\n",
       "2428    0.0\n",
       "1347    0.0\n",
       "2021    0.0\n",
       "7254    0.0\n",
       "3997    0.0\n",
       "5377    0.0\n",
       "4383    0.0\n",
       "5244    0.0\n",
       "1659    0.0\n",
       "208     0.0\n",
       "1081    0.0\n",
       "3524    0.0\n",
       "4829    0.0\n",
       "5968    0.0\n",
       "7097    0.0\n",
       "1699    0.0\n",
       "5313    0.0\n",
       "239     0.0\n",
       "680     0.0\n",
       "4965    0.0\n",
       "Name: target, dtype: float64"
      ]
     },
     "execution_count": 140,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dtrain.target[:50]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "337.0\n",
      "percentage correct validate\n",
      "0.773521505376\n"
     ]
    }
   ],
   "source": [
    "# get delta between predictions on training set and actual training target values\n",
    "# hand calculate accuracy on training set as ratio of (total training samples - wrong training predictions)/total training samples\n",
    "\n",
    "deltatr = abs(dvalid.target[:100000] - dvalid.predround[:100000])\n",
    "deltatr[:50]\n",
    "print(deltatr.sum())\n",
    "print(\"percentage correct validate\")\n",
    "print((len(deltatr) - deltatr.sum())/len(deltatr))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_keys(['val_loss', 'val_acc', 'val_weighted_acc', 'loss', 'acc', 'weighted_acc'])\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZIAAAEWCAYAAABMoxE0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XecXFX9+P/Xe2Z7L9n0srvplYSEEAgloYYeESEUBf0AivoRLIDw8yOK+lE/XxUsiCLSlB5KogYwlFCEhCSQhPReNpts73V25v37495NhmXLJLuzs+X9fDzmkZlzy7zvzOa+55xzz7miqhhjjDHHyxPpAIwxxvRulkiMMcZ0iiUSY4wxnWKJxBhjTKdYIjHGGNMplkiMMcZ0iiUSY9ohIo+JyE9DXHeviJwT7piM6WkskRhjjOkUSyTG9AMiEhXpGEzfZYnE9Hpuk9LtIrJBRGpE5K8iMkhEXhGRKhF5XUTSg9a/VEQ2iUi5iKwQkYlBy2aIyEfuds8CcS3e62IRWedu+76ITAsxxotE5GMRqRSRAyLyoxbLT3P3V+4uv8EtjxeRX4vIPhGpEJH33LJ5IpLXyudwjvv8RyKyWET+LiKVwA0iMltEPnDf45CI/EFEYoK2nywiy0WkVEQKRORuERksIrUikhm03kwRKRKR6FCO3fR9lkhMX/F54FxgHHAJ8ApwNzAA5+/8WwAiMg54GrgNyAKWAf8QkRj3pPoy8DcgA3je3S/uticCjwBfBTKBPwNLRSQ2hPhqgC8BacBFwC0istDd70g33t+7MU0H1rnb/QqYCZzqxnQHEAjxM7kMWOy+55OAH/i2+5mcApwNfN2NIRl4HXgVGAqMAd5Q1cPACuDKoP1eBzyjqr4Q4zB9nCUS01f8XlULVPUg8C6wSlU/VtUG4CVghrveVcC/VHW5eyL8FRCPc6KeA0QD96uqT1UXA6uD3uMm4M+qukpV/ar6ONDgbtcuVV2hqp+oakBVN+AkszPdxdcCr6vq0+77lqjqOhHxAF8BblXVg+57vu8eUyg+UNWX3fesU9W1qrpSVZtUdS9OImyO4WLgsKr+WlXrVbVKVVe5yx7HSR6IiBe4GifZGgNYIjF9R0HQ87pWXie5z4cC+5oXqGoAOAAMc5cd1E/PZLov6Pko4Ltu01C5iJQDI9zt2iUiJ4vIW26TUAXwNZyaAe4+drWy2QCcprXWloXiQIsYxonIP0XksNvc9b8hxACwBJgkIrk4tb4KVf3wOGMyfZAlEtPf5OMkBABERHBOogeBQ8Awt6zZyKDnB4CfqWpa0CNBVZ8O4X2fApYCI1Q1FfgT0Pw+B4DRrWxTDNS3sawGSAg6Di9Os1iwllN7PwhsBcaqagpO019HMaCq9cBzODWnL2K1EdOCJRLT3zwHXCQiZ7udxd/FaZ56H/gAaAK+JSJRInI5MDto278AX3NrFyIiiW4nenII75sMlKpqvYjMBq4JWvYkcI6IXOm+b6aITHdrS48AvxGRoSLiFZFT3D6Z7UCc+/7RwA+AjvpqkoFKoFpEJgC3BC37JzBYRG4TkVgRSRaRk4OWPwHcAFwK/D2E4zX9iCUS06+o6jac9v7f4/zivwS4RFUbVbURuBznhFmG05/yYtC2a3D6Sf7gLt/prhuKrwP3ikgV8EOchNa83/3AhThJrRSno/0Ed/H3gE9w+mpKgV8CHlWtcPf5ME5tqgb41FVcrfgeTgKrwkmKzwbFUIXTbHUJcBjYAcwPWv4fnE7+j9z+FWOOELuxlTEmFCLyJvCUqj4c6VhMz2KJxBjTIRE5CViO08dTFel4TM9iTVvGmHaJyOM4Y0xusyRiWmM1EmOMMZ1iNRJjjDGd0i8mchswYIBmZ2dHOgxjjOlV1q5dW6yqLccnfUa/SCTZ2dmsWbMm0mEYY0yvIiL7Ol7LmraMMcZ0kiUSY4wxnWKJxBhjTKf0iz6S1vh8PvLy8qivr490KH1CXFwcw4cPJzra7nVkTH/TbxNJXl4eycnJZGdn8+nJXs2xUlVKSkrIy8sjJycn0uEYY7pZv23aqq+vJzMz05JIFxARMjMzrXZnTD/VbxMJYEmkC9lnaUz/1W+btowxpicqrm5AgMyk1m8vo6ocKK1jX2kNXo8Q5fGgqlQ3NFFV30R1QxP1Pj8NTQHqfX6+MjeH9MSYsMZsiSRCysvLeeqpp/j6179+TNtdeOGFPPXUU6SlpYUpMmN6hoYmP4fK62kKKKqKiDAyI4GYqKMNKYVV9azZW0ZibBS5AxIZmhaP1/PZ2vGB0lpqGpvIzkwkLtp7ZP/7S2opqGygusE5ARdU1rPlUCVbDlUCcOs547hk2pAjNe4mf4CaRj+p8V13UUljU4APdpfwzvYi3ttRzLYCZ17M0VmJnJydxvCMBJoCgs8fYH9pLat2l3K4MrRmZBG4bPpQSyR9VXl5OX/84x8/k0j8fj9er7fN7ZYtWxbu0Ew/V1TVwLoD5ewtrmF/aS0Hymqpqj/6K3dURgKzczKYnZNBvS/AugPlbMgrJzbKwymjMzl19ACivMKHe0pZvbeUwxX1RHs9RHk9pMRFkTMgkdFZSQxLjyc+2ktstIcGX4CNBytYl1fO5vxKdhfVkF9RR8s5ZWO8HsYPTmbMwCQ251ceOekGL585Kp0FUwZzzqRBbD1UyWPv7+XdHcUAeARGZCSgCnlltQRambN2REY8EwankFdWx7ee/phH/7OHq08ayco9Jby5tZCKOh93XzCRG0/P6VST7lvbCnnxo4Os2FoIDZWkRTUwceQQPndeLgOrtpCy81lmfLICVeUF/+ksCcynKjGHU0clcebQOCbElhJTuY/Yyr0AVEz7MnEDRpEUG0VctJe4KCGmeBMyMJQbeHZOv5j9d9asWdpyipQtW7YwceLECEUEixYtYsmSJYwfP57o6GiSkpIYMmQI69atY/PmzSxcuJADBw5QX1/Prbfeys033wwcne6lurqaCy64gNNOO43333+fYcOGsWTJEuLj4yN2TJH+TLuCqvLujmJ2FFYzIj2ekZkJDE2LJzk2qsf2AwUCyvbCKlbuKmFXUQ1NASUQUBr9ASrqfFTU+ahpaCImykNclJeEWC+DU+IYmhZPVnIslXU+SmoaOVRRz/oD5ewvrT2y79T4aIanx5OWEE1clJdor4fthVXsLqr5VAwjMxKobWyiuLrxU+WJMV5GZibiDwTw+ZXSmkYq6nxtHkuut5CbUlYSlTIIb2Yu8YPH4UsZicfjwecPsPVwFZsOVrKjsIqxA5OZO2YAp4zOpMHnZ29JDTsLq1mxrYgdhdVH9jkoJZZrTx7FqMwEdhfVsLPIWTZ6QCK5WUkMSY0jKS6KpNgoMpNiSYp1fl/7A8oLa/P4f//eRlFVA2kJ0Zw1fiCV9U28vqWAq2aN4CcLp3yqhgRQ29jE3z7YxyUnDGVoWtD/x6LtsPll/Ar/3Ovhue1NzIo9yGUJ68mu3YhH/Z/+MKLi0LHnE/D78ex8FQk0QcIAqC3+9HreGI5k3JnXw9QvwI7lsHExlO2Fr6+CgRPa/MzbIyJrVXVWh+tZIoEf/2MTm/Mru/Q9Jw1N4Z5LJre5fO/evVx88cVs3LiRFStWcNFFF7Fx48Yjl8+WlpaSkZFBXV0dJ510Em+//TaZmZmfSiRjxoxhzZo1TJ8+nSuvvJJLL72U6667rkuP41j01ETS3CwS/LqqoYmCCqd5IC0hhrSEaN7ZXsTv3tjB+ryKz+wjxushPTGa9IQYUuOjSYmPJtorHCyv52BZHVX1Pk4Ykcac3EymDkuloLKe3UU17C+tobqhiTpfAFXlG/PHcP7kwccU/57iGv7v1a3sKa5hUEocg1PiiI4SSqobKaluZEdhFWW1zsk5NT6aaK8HrweivR7SEqJJi48hIcZLo99pM69p8HOoop7i6oajxxflISsplinDUpg5Kp2Zo9IZk5VMakLrTTiFVfWs3VtGXLSXE0akkZEYg6qys7CaD3aX0ORXTsrOYOKQZKK8nz7RltY0sqe4mvzyeup8fhp8fjwe4RTfKnLe/S7S0OL/YnQiDJoMQ6ZB7nwYPR9iEtv9zHYWVvPW1kKGpsVz3uRBRHvbuK7IVw9le6CuzHlU5kPRVijaBlWHAAio0uCJJzZ3Lp6cMwiMmsv97xXyuzd3Mjsng99fPYNBKXEA1DX6+cpjq/lgdwkDk2N55PpZTClcCmsfg4NrUQShxTl30BQYtwDSRkBjjfNIGwXjL4C4FGedqgJY/xSU7oaU4ZA6DFJHQEYupAx1Yn3nV/Dx3yDQBOKBnDNh6hUw6TKIPb5aSaiJxJq2eojZs2d/agzG7373O1566SUADhw4wI4dO8jMzPzUNjk5OUyfPh2AmTNnsnfv3m6Lt6domSTqGv1sOVzJpnynnXtzfiXbDlfh8weIj/ESF+2lpqGJ2kZ/q/sbkRHPLy6fytkTB3Gooo59JbUcqqijpKaR0upGymp9VNb7OFBaS6M/wNDUeM6ZOJC4aC8f7S/jD2/uONJcEhftYVRGIqnx0aTGR5NfXsfX/r6WH1w0ia/M7Xj8UlW9jz+8uZNH/rOH2CgvJ+dkUFjVwOZDlfj8AQYkxTIgKYZzJg7i5NxMTs7JYERGQsifXb3PT0lNI6nx0STGeI+pxjUwOY4Lpg5xXgQCcPgTpGQnY6sLGVtbAJ5oKB8P0RMgaRAEfOBvhKrDZBz8iIyDa5lZUwhZE2HwVCjcDB/8AYbOgC88BlFxULoHSnZAwSY4/AmsfwZWPwzeWMieC9EJzknX3whTLoeZXwGPkzDGDExizMCktg/g8Eb46HFY/yw0tPjhEJsKWeOcuMSDB4ivKYaPnoAP/4wnLo3v3PI+owdO584XNnD+/e/w889NZX5uAjc9vZWVe0q4/fzxPLVyH+v+8lWmyKvUpU/gg1G38ptD09hbE8XPz8nkklEByMiBtJEdf+DJg+C0b7e9PHU4XHI/nHYbHPgQcudB0sCO99tFLJFAuzWH7pKYePQX1ooVK3j99df54IMPSEhIYN68ea2O0YiNPXpVh9frpa6urltijZSKOh+b8ivYdLCSzW6S2FVUTZRXSIqNJjbKw6GKuiMn8tT4aCYOSeaqk0aQGOulttFPXaOfxNgoBqXEMiglDo8IZbWNlNX4GJERzyUnDD3y6zUrOZZpw4/hooaGaurWPU/t9reRceeTduLn8UTHQEMVrH6YwMdP8tDIa/nJP2FvcQ3jBifz3o4iPtxTSkZiDDNHpTNjZDpFVQ18sKuEtfvLaGwK8IWZw7l9wXgGJse1/r6BgHui9kF9hdPU4Y2B2hLY9SbsfMM5GSdkOCeX5CEwcg5xo+YyLC2jxYecB3vehYNrnPWGnQiDp0HBRtj2Kux8HTxe55dwejZUHIC97zm/5puJFzQALX95B0saDMmDnV/qTe7f7az/ggU/hyj37zp5MIw65eg2fh/sex+2vwq733b2H5MIjbXwr+86ieaS3zq1l/oKqDgI9eVOsmmocmoehzfC4Q1QstNJSJMudWoDCZkQn+4kveTBTi91S00NsOcdePIKWPckl515B1OGpfLtZ9fxyFNPcV7sT7nOP5NrL/wpF5w2mi9X/pGEj1/lL00X8rND1+It8HBKbiaPnTOWWdkZn91/V0jPdh7dzBJJhCQnJ1NV1fpdSysqKkhPTychIYGtW7eycuXKbo6u51BVnl+Tx4Nv72JP8dF2+cEpcUwamsK8CVmo4jQfNfoZkZHA5KEpTB6awrC0+O7p1zi0AdY8Ap8sJr6xiviYJNj1Erx7D4w9D7b+E+rK8CQM4KsNPyf+hN9yz0pndu5hafGcPXEQZTWN/HtzAc+tyUMEJg1J4UtzRnHZ9GFMHZ569L0Cftj7rnOyP/yJ86jKbz++hAEwbKZzMs1fB5X/cn79I5A13kk6fp+zvDLP2SYmCRqrP72fqDjIOcNZv3QP7F4BiQNg/EWQc7rzCz5psHNC9jc4J+uibVBTDFExTi0lIcOpdaQMPXo8pbvBV+c0XbXHGw25ZzqPYKqw4Vl47W740+lOcmnZPNYsbZQT50k3wbQrnXhCFRULY891PoOP/wanf4/RWUm8cMup7P/DL6gtjeWcmI1ErbgUds0mYe+7+GZ/nca4r/Art+aalhDeq6ciJayJREQWAL8FvMDDqvqLFsvvA+a7LxOAgaqa5i67HviBu+ynqvq4Wz4TeAyIB5YBt2ov7OjJzMxk7ty5TJkyhfj4eAYNGnRk2YIFC/jTn/7EtGnTGD9+PHPmzIlgpMdvb7HT+en1ClEeoabBT355HfnldYwZmMSi2e1X6XcXVXP3S5+wcncpM0amcfv545kyLJUpQ1PavMY+ZA3Vzq/vqkNQUwTVBe7Dfd5U75yQYhKdX9i+WueXrXicX7yDpjjP1z4KB1ZBVDxM/pzT2Tn8JOeX+4d/gXVPwtjz4YzbISMH+et5XL//Lk6+9gXihkxgVGbCkWSnquwrriHDl09KxTY4/B5s90P+IEjMgkPrnRNm5UEnpqzxkH2a8ws0KtY5wYvnaO2k+cQ/eNqRJh8Amhrh4FonIR1c65R5o531h85wthk42WnyyV/n/ILPHOM0l3TQN3GEJ945YQ+e2sF6Xhgw9li/vU8TgRMWOUn7P/c7/R6pbj9CfIaTFGMSIGUYxHfBZfMzvgQv3gh734HceUSX72F02Xvombcjs74My++BT56DU75J9Hk/5Rs99CKNrhS2znYR8QLbgXOBPGA1cLWqbm5j/f8GZqjqV0QkA1gDzMKpH68FZqpqmYh8CNwKrMRJJL9T1Vfai6UnXrXVFzV/ptUNTdy/fDuPvb8bDQTw8+nLmUXAK8Kqu89uMyH8Y30+331+PbFRHu6+cCJXzRqBJ3h8gN8HG190TtR+n3Mi9EY7HY3+JucXsa/O+VXdWAOI2+QTBXXlTpNHS1FxTtNP0iDneXPyCDQ5J9DoRCfBFG452hyTkQsn3QjTr3F+ibcU8Dsny2ale+Cv5zrt+2POcWoUhZvdGOHTzUHifFgacF96YPTZznuNvwCiI3eFXr/mq4Nfj4cx58IVf4VltztNdLdtdPoyAGpLj62200P1hM722cBOVd3tBvQMcBnQaiIBrgbucZ+fDyxX1VJ32+XAAhFZAaSo6gdu+RPAQqDdRGLCT1Vp8gf428p9PPrGes6pXcbahOUkBypoSMmhLnU01WMXkjTjcgoq67ngt+/yzw2HuP7U7M/sa/nmAm57dh0zR6bzh6smMbBuD2xeffQSx8p8+PAhp30+c4zTlt/U4DTNeKKchBGTCIkD3VqF2wHtdzt841LdX6wjnPbwpMGQlAWxKa23jbfU3BxTV+40GXnamWnI02JMUEYOXPMs/P3z8Mnzzi/26ddAXNAv5ZShTi1i4ESnplFb6tSSkgZ2aweqaUN0PEy7CtY+7vww+PhJmHLF0SQCfSKJHItwJpJhwIGg13nAya2tKCKjgBzgzXa2HeY+8lopN2Gmqk5ntXu5ZmOTH2/ARxRNROEDfxN1VaXEL/s+S6PWkhRdAyPOgKEziC/eQXzeajIKV8EpV5CRmMLEISm8+PHBTyeSgJ+N7/+Lta+9xsPJBznDX4D399udGkFLI0+BC3/lNGe0dyIPh842xwybCd/b6ewnlMSVlOU8TM8x44vOj5lnrgFfDcy5JdIRRVQ4E0lr/0PaakdbBCxWPTIip61tQ96niNwM3AwwcmQIl9eZVtU1+imrbaShtpo4rSWeBpKlkRh8n/kyiqljYfI2vNnnwtxbnSt+mm1eCs99Efa/DzlncPmMYfxs2RZ2FVUzOisJAgHKn7iOKXuXMcULgejBeFKnwoQLnP6I5k5hcH6lh3LJZE/mtetcerUh02DICU6/VfbpHV8o0MeF8685DxgR9Ho40NblJYuAb7TYdl6LbVe45cND2aeqPgQ8BE4fSehhG3AGYRWVVxNde5gs6ogWPwioNwaiE5Co+KMdvN5o8EQjFduIumN76zscc7bT77Dln5BzBpdNH8rPX9nCyx8f5Lvnjafx9Z+QtncZD0ct4rKb/oesQcNb348xPcWJ18O/vtPvayMQ3kSyGhgrIjnAQZxkcU3LlURkPJAOfBBU/BrwvyLS3Ht5HnCXqpaKSJWIzAFWAV8Cfh/GY+iXGpv8lBYXMsBfiEfUab+PS4HYZMR7nJPVxSTC6LNg67/ggl8yMCWOuWMG8NLHB/n2wI+Ief83POOfz9Qv/oysQQO69oCMCYcTr4fM0c4I8n4ubI3LqtoEfBMnKWwBnlPVTSJyr4hcGrTq1cAzwZfwup3sP8FJRquBe5s73oFbgIeBncAurKO98wJ+qCmmqfIwlUV51BXsYnDgMETF4smagCcj2+k8PN4k0mzCxc44hfyPAbj8xGEMLF9PYMl/875/EntPvpeTR1sSMb2EN8q5JLofXN7bkbD2UqrqMlUdp6qjVfVnbtkPVXVp0Do/UtXvt7LtI6o6xn08GlS+RlWnuPv8Zm8cQ3I8kpKc6R7y8/O54oorWl1n3rx5tLzMmYYqZ0CY+zHdf//91NYenZTvwgXnU7bjQ6g4QFT1IVJ8RSRLLU2Jg4gaOA6i2xhNfTzGX+CMf9j6TwDOH5vM/TEPctCfzm/SfsBt50d+hgFjzLHr13dI7I2GDh3K4sWLO16xqcG5RLVkp3OZbL0zn9CRRKIBqMznX4/8gsTkZPboEAoSx9E4cCqeIScQlTrUGbfQlRIyYNSpTj8JkPDeLxkpBdwd+Co/vvq0I/eJMMb0LpZIIuTOO+/kj3/845HXP/rRj/jxj3/M2WefzYknnsjUqVNZsmTJZ7bbu3cvU6ZMAaCuro5FixYxbdo0rrrqKmeurUATWnmQr33lOmbNv4RJZy3if379F6g4wO9+ex/5+fnMnz+f+WfMheoCsk++hNUlcWRlZvLkX//MidOnM2XqVO6///4j7zdx4kRuuukmJk+ezHnnnde5Ob0mXgLF22DdU7DyjzRMv4EffONmJg9N7XhbY0yPZNcgArzyfWeEcVcaPBUu+EWbixctWsRtt9125MZWzz33HK+++irf/va3SUlJobi4mDlz5nDppZe2OV/Ugw8+SEJCAhs2bGDDunWcOGsWWroHqmO54447aEjLpd4PX1t0KVds3Mx/f+nz/Oa+3/LW0qcZEOujzDuAJjwMTk1g26YNPProo6xatQpV5eSTT+bMM88kPT2dHTt28PTTT/OXv/yFK6+8khdeeOH4p6ufcBG8cgcs+QakDCN2wU+Y2DxVtjGmV7IaSYTMmDGDwsJC8vPzWb9+Penp6QwZMoS7776badOmcc4553Dw4EEKCgra3Mc777zjnNAba5k2JIZpE8dSTzQ7A8NYvHwliy4+hy9ePI+dO7azcvthpK7EGdxXU0gZKRzwJeP1CKkJMbz33nt87nOfIzExkaSkJC6//HLeffddoIunq08d7sznpAFn2mtLIsb0elYjgXZrDuF0xRVXsHjxYg4fPsyiRYt48sknKSoqYu3atURHR5Odnd3q9PHBpL4Mirejnih8Es1B/wBSKst46IHfsnr1atLT07nhhhto8iTQoFGgAao0HqIHkZMciydowsC2dPl09ef8CAq3OjOpGmN6PauRRNCiRYt45plnWLx4MVdccQUVFRUMHDiQ6Oho3nrrLfbt29f2xgE/Z8ycxJN//zuB2BRe3VrFls2bGZwaC756EhMTSU1NpaCggFdeeYXEuGg0PZvE5BQaYjLJyUoiOe7o5bxnnHEGL7/8MrW1tdTU1PDSSy9x+umnh+fAc+fBnK+FZ9/GmG5nNZIImjx5MlVVVQwbNowhQ4Zw7bXXcskllzBr1iymT5/OhAlt3WdZoWQHt1xzCTfcsZmJp1/K+ElTmDnrJJLiojnhhBOYMWMGkydPJjc3l7lz5wIQl5DM1275Bgsvu5QhQ4bw1ltvHdnjiSeeyA033MDs2bMBuPHGG5kxY0a/vOuiMebY2D3bextfPZTugkATvpRR7Kry0ORXsjMTSYqL7O+CXvuZGmNa1ROmkTddzVfnjAsBGtNy2VWuBFTJzUokIca+SmNMZNjZp7fw1R9JIg2po9lV5kytnjsgifgYG8hnjImcft3Z3mua9ZqOJpH61Fx2lvkQgdysxB6TRHrNZ2mM6XL9NpHExcVRUlLS80+AgQCU7AYN0Jiay66yJrwi5GYl9pgpRVSVkpIS4uK6cF4uY0yv0W+btoYPH05eXh5FRUWRDqV99RVQX0EgcSCF+7ajqmQlx7K7vGf9BoiLi2P4cLuHiDH9Ub9NJNHR0eTk5EQ6jPZVHIQ/nIU/92w+X3oLWw5V8vTNc5g6Mr3jbY0xppv020TSK7x+Dxrwc0/D1azPK+fBa2dyoiURY0wP07PaR8xR+1fCJ8+zJfcG/r4V7lwwgQVTBkc6KmOM+QxLJD1RIACv3EFT0lCu3z6Xk3MyuPn03EhHZYwxrbJE0hNtXAyH1vPn6C9Sq7H86gsn4PHY7TyNMT2TJZKexlcPb/yEkuSJ/OrQVH5w8SRGZCREOipjjGmTJZKeZvXDULGf75V/njPGDWLRSSMiHZExxrTLrtrqSerK0Hf+HxvjZrGqdiqvXz61zbsjGmNMTxHWGomILBCRbSKyU0S+38Y6V4rIZhHZJCJPuWXzRWRd0KNeRBa6yx4TkT1By6aH8xi61Xv3QX0Fd1R8nu+cO46hafGRjsgYYzoUthqJiHiBB4BzgTxgtYgsVdXNQeuMBe4C5qpqmYgMBFDVt4Dp7joZwE7g30G7v11VF4cr9oiozEdX/ZllnjORwVO54dTsSEdkjDEhCWeNZDawU1V3q2oj8AxwWYt1bgIeUNUyAFUtbGU/VwCvqGptGGONvHd+RcDfxM/rPsf/Xj6VKK91Xxljeodwnq2GAQeCXue5ZcHGAeNE5D8islJEFrSyn0XA0y3KfiYiG0TkPhGJbWUbRORmEVkjImt6/Hxa5fvRj57gmaYzmXfyTKaPSIt0RMYYE7JwJpLWeolbTrUbBYwF5gFXAw+LyJGzqIgMAaYCrwVtcxcwATgJyADubO3NVfUhVZ2lqrOysrKO9xi6x9v/R5MKf9bLufXscZGOxhhjjkk4E0keEHzt6nBWlMd6AAAcO0lEQVQgv5V1lqiqT1X3ANtwEkuzK4GXVNXXXKCqh9TRADyK04TWe5XsQtc9xZNNZ3HW7BlkJbdawTLGmB4rnIlkNTBWRHJEJAaniWppi3VeBuYDiMgAnKau3UHLr6ZFs5ZbS0Gc62IXAhvDEn13efv/8BHFQ4GFfPVMmwbFGNP7hO2qLVVtEpFv4jRLeYFHVHWTiNwLrFHVpe6y80RkM+DHuRqrBEBEsnFqNG+32PWTIpKF03S2DvhauI4h7Mr2oZ88xxNNFzJv1hSGpNrlvsaY3iesAxJVdRmwrEXZD4OeK/Ad99Fy2718tnMeVT2rywONlA3PIhrgcf95PHXm6EhHY4wxx8VGtkeKKv6Pn2KNTmL29Ok2n5YxpteywQqRcuBDvOV7eL7pdL50yqhIR2OMMcfNEkmkrH+aBollffKZTBueGulojDHmuFkiiQRfPbrxRV71z2L+tFybmNEY06tZIomE7a8iDRUsbjqdC6cOiXQ0xhjTKZZIImH9M5R5M9mbPIsTrFnLGNPLWSLpbjXF6M7lPN94KgumDbNmLWNMr2eJpLvt+DcSaGJJ0xxr1jLG9AmWSLrb9tco92ZSljzBZvk1xvQJlki6k9+H7nqT5b5pXDBtqDVrGWP6BEsk3enAKqShktebpnPh1MGRjsYYY7qEJZLutOPfNBHF1oSZzBiRHulojDGmS1gi6UaB7a+xWidw2uRsPB5r1jLG9A2WSLpL+X48RVt5vekEzp9szVrGmL7DEkl32fFvAFZHz2JObmaEgzHGmK5j08h3k8D2f3OQQYyeMIOYKMvfxpi+w85o3cFXh+5+mzeaTuD8KdasZYzpWyyRdId9/8Hrr+c/MoMzxmVFOhpjjOlS1rTVDXTXCnxEETv6DBJi7CM3xvQtdlbrBnXb32S9fxzzp2ZHOhRjjOlyYW3aEpEFIrJNRHaKyPfbWOdKEdksIptE5Kmgcr+IrHMfS4PKc0RklYjsEJFnRSQmnMfQaTUlJJRs4gOdzFkTBkY6GmOM6XJhSyQi4gUeAC4AJgFXi8ikFuuMBe4C5qrqZOC2oMV1qjrdfVwaVP5L4D5VHQuUAf8VrmPoEnveBqB08KmkJ/bsnGeMMccjnDWS2cBOVd2tqo3AM8BlLda5CXhAVcsAVLWwvR2KM8vhWcBit+hxYGGXRt3Fqra8QaXGkzvt9EiHYowxYRHORDIMOBD0Os8tCzYOGCci/xGRlSKyIGhZnIisccubk0UmUK6qTe3ss0fR3StYFZjEuVN6dJjGGHPcwtnZ3tpkUtrK+48F5gHDgXdFZIqqlgMjVTVfRHKBN0XkE6AyhH06by5yM3AzwMiRI4/vCDqrbC8pdXnsTLqAczMSIhODMcaEWThrJHnAiKDXw4H8VtZZoqo+Vd0DbMNJLKhqvvvvbmAFMAMoBtJEJKqdfeJu95CqzlLVWVlZkRm7UbP1DQDixp8dkfc3xpjuEM5EshoY615lFQMsApa2WOdlYD6AiAzAaeraLSLpIhIbVD4X2KyqCrwFXOFufz2wJIzH0CklnyynQNOYOXNOpEMxxpiwCVsicfsxvgm8BmwBnlPVTSJyr4g0X4X1GlAiIptxEsTtqloCTATWiMh6t/wXqrrZ3eZO4DsishOnz+Sv4TqGTgkESDv8AR95T2DqcLulrjGm7wrrgERVXQYsa1H2w6DnCnzHfQSv8z4wtY197sa5IqxHa8jfSEqgnNqRp9ktdY0xfZrNtRUmB9b8C4ChJy7oYE1jjOndLJGESdTeFewIDOOESZM6XtkYY3oxSyTh4KtnaMVHbIw70SZpNMb0eZZIwkD3ryRGGykZdGqkQzHGmLALKZGIyAsicpGIWOIJQc2W5fjUS+yYMyIdijHGhF2oieFB4Bpgh4j8QkQmhDGmXi+w6y0+0rFMHDU00qEYY0zYhZRIVPV1Vb0WOBHYCywXkfdF5MsiEh3OAHudmmJSyjbxXmAqk4amRDoaY4wJu5CbqkQkE7gBuBH4GPgtTmJZHpbIeqvdKwDYmzLbOtqNMf1CSGc6EXkRmAD8DbhEVQ+5i54VkTXhCq430l1vUUUiMaNmRjoUY4zpFqH+ZP6Dqr7Z2gJVndWF8fRuqgR2vsl7/slMGZ4R6WiMMaZbhNq0NVFEjkwY5U6q+PUwxdR7Fe/AW53Pe4GpTB2WGulojDGmW4SaSG5y7xECgHtHw5vCE1IvtvcdAN4PTLGOdmNMvxFqIvFI0MyD7v3Y7QbkLR3aQLUnmegBudbRbozpN0I9270GPCcif8K5I+HXgFfDFlUvpYc/YYtmM3WETRtvjOk/Qk0kdwJfBW7BuYXuv4GHwxVUr+RvgoLNrPfNt/4RY0y/ElIiUdUAzuj2B8MbTi9Wugvx17M5MIprLJEYY/qRUMeRjAV+DkwC4prLVTU3THH1Poc/AWCzZjNxiHW0G2P6j1A72x/FqY004dxj/QmcwYmm2eEN+CSaxrTRJMZaR7sxpv8INZHEq+obgKjqPlX9EXBW+MLqhQ5/wj4ZTu5gG4hojOlfQv3pXO9OIb9DRL4JHAQGhi+s3kcPb2SdbyLjBiVFOhRjjOlWodZIbgMSgG8BM4HrgOvDFVSvU1WA1BSyKTCS8YOTIx2NMcZ0qw4TiTv48EpVrVbVPFX9sqp+XlVXhrDtAhHZJiI7ReT7baxzpYhsFpFNIvKUWzZdRD5wyzaIyFVB6z8mIntEZJ37mH4MxxsezR3tgWzGDbJEYozpXzps2lJVv4jMFBFRVQ11x24CegA4F8gDVovIUlXdHLTOWOAuYK6qlolIc3NZLfAlVd0hIkOBtSLyWtA0Lber6uJQYwm7AieRbJeR5GYlRjgYY4zpXqH2kXwMLBGR54Ga5kJVfbGdbWYDO1V1N4CIPANcBmwOWucm4AF37i5UtdD9d3vQe+SLSCGQBZTTEx3+hOKoQWSmDCI2yhvpaIwxpluF2keSAZTgXKl1ifu4uINthgEHgl7nuWXBxgHjROQ/IrJSRBa03ImIzMaZ12tXUPHP3Cav+0QktrU3F5GbRWSNiKwpKirqINROOvwJW3UU461ZyxjTD4U6sv3Lx7FvaaWsZdNYFDAWmAcMB94VkSnNTVgiMgRnvMr17uh6cJrCDuMkl4dwpm+5t5WYH3KXM2vWrJCb5I5ZYy1aspO1voWMtSu2jDH9UKgj2x/ls0kAVf1KO5vlASOCXg8H8ltZZ6Wq+oA9IrINJ7GsFpEU4F/AD4I79oPuztjgxvW9UI4hbAq3IBpgc2AUC61GYozph0Jt2vonzkn9X8AbQApQ3cE2q4GxIpIjIjHAImBpi3Vexhkpj4gMwGnq2u2u/xLwhKo+H7yBW0vBndZ+IbAxxGMIj8MbANikoxhnl/4aY/qhUJu2Xgh+LSJPA693sE2TO3jxNcALPKKqm0TkXmCNqi51l50nIpsBP87VWCUich1wBpApIje4u7xBVdcBT4pIFk7T2TqcKe0jp2QnPoml0DuIURkJEQ3FGGMi4XgnhRoLjOxoJVVdBixrUfbDoOcKfMd9BK/zd+DvbeyzZ03NUr6fIu9AxqQlE+UNtYJnjDF9R6h9JFV8uo/kME4nt6k4wAF/po1oN8b0W6E2bdlZsg2B8gPs8k21K7aMMf1WSG0xIvI5EUkNep0mIgvDF1Yv0ViLp7aYg5plY0iMMf1WqI3696hqRfMLd5zHPeEJqRepyAPgoA6wObaMMf1WqImktfXs7k0V+wEo9g5kWFp8hIMxxpjICDWRrBGR34jIaBHJFZH7gLXhDKxXKHcSiTdjJB5PawP5jTGm7ws1kfw30Ag8CzwH1AHfCFdQvUb5AZrwkjqwwyuhjTGmzwr1qq0aoNX7ifRn/rL9HNIMsgemdryyMcb0UaFetbVcRNKCXqeLyGvhC6t3aCzZx0EdwGi7B4kxph8LtWlrQNBNpXDvH2L3bK84wEEdQM4ASyTGmP4r1EQSEJEjHQEikk0rswH3K34fsXWF5FkiMcb0c6Fewvv/Ae+JyNvu6zOAm8MTUi9ReRAPAapiB5McFx3paIwxJmJC7Wx/VURm4SSPdcASnCu3+i/30l/SRrS/njHG9HGhTtp4I3Arzs2p1gFzgA9wbr3bP5U7dxGOz8qJcCDGGBNZofaR3AqcBOxT1fnADCDMN0Lv2WqL9gKQPiQ3soEYY0yEhdpHUq+q9SKCiMSq6lYRGR/WyHq42sI9VGkaOYMyIh2KMcZEVKiJJM8dR/IysFxEyvjs/df7FX/Zfrv01xhjCL2z/XPu0x+JyFtAKvBq2KLqBWKqD5LPCKal22SNxpj+7Zhn8FXVtzteq48LBEhqKKA6brbdXtcY0+/ZWfB4VBcQjQ9/8vBIR2KMMREX1kQiIgtEZJuI7BSRVid9FJErRWSziGwSkaeCyq8XkR3u4/qg8pki8om7z9+JSLfP395UuheAmMxR3f3WxhjT44QtkYiIF3gAuACYBFwtIpNarDMWuAuYq6qTgdvc8gycOzCeDMwG7hGRdHezB3EGRo51HwvCdQxtKcvfBUDyYLv01xhjwlkjmQ3sVNXdqtoIPANc1mKdm4AH3EkgUdVCt/x8YLmqlrrLlgMLRGQIkKKqH6iqAk8A3X7v+MrDuwEYNHJMd7+1Mcb0OOFMJMOAA0Gv89yyYOOAcSLyHxFZKSILOth2mPu8vX0CICI3i8gaEVlTVNS1YycbSvZTrolkDxncpfs1xpjeKJyJpLW+i5YzBkfhNE/NA64GHnbHq7S1bSj7dApVH1LVWao6KysrK+SgQ+GpyqdABpCeGNOl+zXGmN4onIkkDwie0XA4nx3EmAcsUVWfqu4BtuEklra2zXOft7fPsEuoL6A8qmuTkzHG9FbhTCSrgbEikiMiMcAiYGmLdV4G5gOIyACcpq7dwGvAee6dGNOB84DXVPUQUCUic9yrtb6EMxNxt0r1FVIda/f1MsYYOI4BiaFS1SYR+SZOUvACj6jqJhG5F1ijqks5mjA2A37gdlUtARCRn+AkI4B7VbXUfX4L8BgQD7ziPrqPr45UraQhYUi3vq0xxvRUYUskAKq6DFjWouyHQc8V+I77aLntI8AjrZSvAaZ0ebAh0sp8BAgkD41UCMYY06PYyPZjVFPk3NDKk2aj2o0xBiyRHLOqwr0AxGXYnRGNMQYskRyzhhJnGEty1sgIR2KMMT2DJZJj5K/Io1wTGZBpN7QyxhiwRHLMvFX5HNJMBibHRjoUY4zpESyRHKPY2sMUSiaJsWG94M0YY3oNSyTHKKmxgIpoG4xojDHNLJEcC189yf4KauMGRToSY4zpMSyRHIsqZ1qvpiQb1W6MMc0skRyLioMAaEqrM9cbY0y/ZInkGNSXOrdIiU63Ue3GGNPMEskxqC3aB0DCABvVbowxzSyRHIPGMmcwYma6DUY0xphmlkiOgVbkc0gzbDCiMcYEsURyDKJrmke1x0U6FGOM6TEskRyD+LoCCiWTlHgb1W6MMc0skYTKV09iUxlVMYNw7vJrjDEGLJGEzh2MWB9vo9qNMSaYJZJQVTqJxG6xa4wxn2aJJFTuqHZJtVHtxhgTzBJJiHzlzp0RYzPtzojGGBMsrIlERBaIyDYR2Ski329l+Q0iUiQi69zHjW75/KCydSJSLyIL3WWPicieoGXTw3kMzepLDlCuiaSnpXXH2xljTK8RtutYRcQLPACcC+QBq0VkqapubrHqs6r6zeACVX0LmO7uJwPYCfw7aJXbVXVxuGJvTVN5PoWaboMRjTGmhXDWSGYDO1V1t6o2As8Alx3Hfq4AXlHV2i6N7lhVF1CkqTYY0RhjWghnIhkGHAh6neeWtfR5EdkgIotFpLXZEBcBT7co+5m7zX0i0moVQURuFpE1IrKmqKjouA4gWFRdMcWkMjDFaiTGGBMsnImktVF72uL1P4BsVZ0GvA48/qkdiAwBpgKvBRXfBUwATgIygDtbe3NVfUhVZ6nqrKysrOM7gqM7I7ahmBLSyEiI6dy+jDGmjwlnIskDgmsYw4H84BVUtURVG9yXfwFmttjHlcBLquoL2uaQOhqAR3Ga0MKrsZqYQD21MQPweGxUuzHGBAtnIlkNjBWRHBGJwWmiWhq8glvjaHYpsKXFPq6mRbNW8zbizFOyENjYxXF/VnUhAL74zLC/lTHG9DZhu2pLVZtE5Js4zVJe4BFV3SQi9wJrVHUp8C0RuRRoAkqBG5q3F5FsnBrN2y12/aSIZOE0na0DvhauYzjCTSRN8QPD/lbGGNPbhHUaW1VdBixrUfbDoOd34fR5tLbtXlrpnFfVs7o2yhDUOIlEkiyRGGNMSzayPRRujcSbMjjCgRhjTM9jN9YIQVPFIUSFuNROXv1ljDF9kCWSEDRUFFBLChlJ8ZEOxRhjehxr2gqBv/IwRZpGRqKNITHGmJYskYSippBiTSEzyRKJMca0ZIkkBFG1RRSRRmaiTY9ijDEtWSLpiCqx9cUUaSoZViMxxpjPsETSkfoKvOqjVNJIjrVrE4wxpiVLJB1xx5DUx2TizMpijDEmmCWSjtQ0z7NlY0iMMaY1lkg6Ul0AQCDRpkcxxpjWWCLpiNu05bF5towxplWWSDpSXYhPvcSlDIh0JMYY0yNZIumAv+owJaSQafdqN8aYVlki6YCvssAZQ2KDEY0xplWWSDqgVQU2z5YxxrTDEkkHPDVFFGuqzbNljDFtsETSnkCA6PoSiki1GokxxrTBEkl76srwaJNTI7FEYowxrbJE0h53VHuJpJMSFx3hYIwxpmcKayIRkQUisk1EdorI91tZfoOIFInIOvdxY9Ayf1D50qDyHBFZJSI7RORZEQlfVcEd1d4Qk4nHY/NsGWNMa8KWSETECzwAXABMAq4WkUmtrPqsqk53Hw8HldcFlV8aVP5L4D5VHQuUAf8VrmNoHtXuT7R5towxpi3hrJHMBnaq6m5VbQSeAS7rzA7FmX73LGCxW/Q4sLBTUbbHTSQkDQrbWxhjTG8XzkQyDDgQ9DrPLWvp8yKyQUQWi8iIoPI4EVkjIitFpDlZZALlqtrUwT4RkZvd7dcUFRUd3xFUF9BIFHFJ6ce3vTHG9APhTCStdSpoi9f/ALJVdRrwOk4No9lIVZ0FXAPcLyKjQ9ynU6j6kKrOUtVZWVnH2TRVU0QxaWQm2ah2Y4xpSzgTSR4QXMMYDuQHr6CqJara4L78CzAzaFm+++9uYAUwAygG0kSk+VaFn9lnVwpUFVAYSLHpUYwxph3hTCSrgbHuVVYxwCJgafAKIjIk6OWlwBa3PF1EYt3nA4C5wGZVVeAt4Ap3m+uBJeE6gJJzf8u3fP9t92o3xph2hO0m5KraJCLfBF4DvMAjqrpJRO4F1qjqUuBbInIp0ASUAje4m08E/iwiAZxk9wtV3ewuuxN4RkR+CnwM/DVcx1CkaezXQTYY0Rhj2hG2RAKgqsuAZS3Kfhj0/C7grla2ex+Y2sY+d+NcERZ2pTWNAJZIjDGmHTayvR0lNU73jU3YaIwxbbNE0o7mGol1thtjTNsskbSjtKYRj0BavM2zZYwxbbFE0o6SmkbSE2Jsni1jjGmHJZJ2lFQ32H1IjDGmA2G9aqu3mzY8jZwBSZEOwxhjejRLJO34xvwxkQ7BGGN6PGvaMsYY0ymWSIwxxnSKJRJjjDGdYonEGGNMp1giMcYY0ymWSIwxxnSKJRJjjDGdYonEGGNMp4hz08G+TUSKgH3HufkAnFv89jf98bj74zFD/zxuO+bQjFLVrI5W6heJpDNEZI2qzop0HN2tPx53fzxm6J/HbcfctaxpyxhjTKdYIjHGGNMplkg69lCkA4iQ/njc/fGYoX8etx1zF7I+EmOMMZ1iNRJjjDGdYonEGGNMp1giaYeILBCRbSKyU0S+H+l4wkFERojIWyKyRUQ2icitbnmGiCwXkR3uv+mRjrWriYhXRD4WkX+6r3NEZJV7zM+KSJ+7z7KIpInIYhHZ6n7np/T171pEvu3+bW8UkadFJK4vftci8oiIFIrIxqCyVr9bcfzOPbdtEJETO/PelkjaICJe4AHgAmAScLWITIpsVGHRBHxXVScCc4BvuMf5feANVR0LvOG+7mtuBbYEvf4lcJ97zGXAf0UkqvD6LfCqqk4ATsA5/j77XYvIMOBbwCxVnQJ4gUX0ze/6MWBBi7K2vtsLgLHu42bgwc68sSWSts0GdqrqblVtBJ4BLotwTF1OVQ+p6kfu8yqcE8swnGN93F3tcWBhZCIMDxEZDlwEPOy+FuAsYLG7Sl885hTgDOCvAKraqKrl9PHvGueW4vEiEgUkAIfog9+1qr4DlLYobuu7vQx4Qh0rgTQRGXK8722JpG3DgANBr/Pcsj5LRLKBGcAqYJCqHgIn2QADIxdZWNwP3AEE3NeZQLmqNrmv++L3nQsUAY+6TXoPi0giffi7VtWDwK+A/TgJpAJYS9//rpu19d126fnNEknbpJWyPnuttIgkAS8At6lqZaTjCScRuRgoVNW1wcWtrNrXvu8o4ETgQVWdAdTQh5qxWuP2CVwG5ABDgUScZp2W+tp33ZEu/Xu3RNK2PGBE0OvhQH6EYgkrEYnGSSJPquqLbnFBc1XX/bcwUvGFwVzgUhHZi9NkeRZODSXNbf6Avvl95wF5qrrKfb0YJ7H05e/6HGCPqhapqg94ETiVvv9dN2vru+3S85slkratBsa6V3fE4HTQLY1wTF3O7Rv4K7BFVX8TtGgpcL37/HpgSXfHFi6qepeqDlfVbJzv9U1VvRZ4C7jCXa1PHTOAqh4GDojIeLfobGAzffi7xmnSmiMiCe7fevMx9+nvOkhb3+1S4Evu1VtzgIrmJrDjYSPb2yEiF+L8UvUCj6jqzyIcUpcTkdOAd4FPONpfcDdOP8lzwEic/4xfUNWWHXm9nojMA76nqheLSC5ODSUD+Bi4TlUbIhlfVxOR6TgXGMQAu4Ev4/yg7LPftYj8GLgK5wrFj4EbcfoD+tR3LSJPA/NwposvAO4BXqaV79ZNqn/AucqrFviyqq457ve2RGKMMaYzrGnLGGNMp1giMcYY0ymWSIwxxnSKJRJjjDGdYonEGGNMp1giMaaHE5F5zTMUG9MTWSIxxhjTKZZIjOkiInKdiHwoIutE5M/u/U6qReTXIvKRiLwhIlnuutNFZKV7L4iXgu4TMUZEXheR9e42o93dJwXdR+RJd0CZMT2CJRJjuoCITMQZPT1XVacDfuBanEkCP1LVE4G3cUYbAzwB3Kmq03BmFWgufxJ4QFVPwJkTqnnaihnAbTj3xsnFmS/MmB4hquNVjDEhOBuYCax2KwvxOBPkBYBn3XX+DrwoIqlAmqq+7ZY/DjwvIsnAMFV9CUBV6wHc/X2oqnnu63VANvBe+A/LmI5ZIjGmawjwuKre9alCkf9psV57cxK111wVPA+UH/u/a3oQa9oypmu8AVwhIgPhyL2yR+H8H2ueZfYa4D1VrQDKROR0t/yLwNvufWDyRGShu49YEUno1qMw5jjYrxpjuoCqbhaRHwD/FhEP4AO+gXPzqMkishbn7nxXuZtcD/zJTRTNs/CCk1T+LCL3uvv4QjcehjHHxWb/NSaMRKRaVZMiHYcx4WRNW8YYYzrFaiTGGGM6xWokxhhjOsUSiTHGmE6xRGKMMaZTLJEYY4zpFEskxhhjOuX/B1lK9VXNe44vAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEWCAYAAACXGLsWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8VPXV+PHPmcxk3xcCJMSABGQRWQIiLnUXtYpaF1Bbsa20Wtvap7XVPm21PvV5/LW2tVattda27qUoSuu+r6AsCrLIDhICJITse2bO7497A0MIJEBuJiTn/Xrllcy933vvuRmYk+96RVUxxhhjDsQX6QCMMcb0fJYsjDHGdMiShTHGmA5ZsjDGGNMhSxbGGGM6ZMnCGGNMhyxZGNMFROTvIvKrTpbdJCJnHu55jOlOliyMMcZ0yJKFMcaYDlmyMH2G2/xzs4gsE5FaEfmriGSLyEsiUi0ir4tIWlj5C0VkhYhUiMjbIjIibN84EVniHvdPILbNtb4sIp+6x34oImMOMebrRGSdiOwSkXkiMtDdLiLyexEpEZFK955Gu/vOE5GVbmxbReRHh/QLMyaMJQvT13wFOAsYBlwAvAT8FMjE+f/wPQARGQY8BdwEZAEvAv8WkWgRiQaeAx4D0oF/uefFPXY88AjwLSAD+DMwT0RiDiZQETkd+D/gcmAAsBl42t19NnCKex+pwBVAmbvvr8C3VDUJGA28eTDXNaY9lixMX/NHVd2hqluB94CPVPUTVW0E5gLj3HJXAC+o6muq2gzcDcQBU4DJQAC4R1WbVXUOsDDsGtcBf1bVj1Q1qKr/ABrd4w7GVcAjqrrEje9W4AQRyQeagSTgGEBUdZWqbnOPawZGikiyqpar6pKDvK4x+7BkYfqaHWE/17fzOtH9eSDOX/IAqGoI2ALkuPu26t6rcG4O+/ko4IduE1SFiFQAg9zjDkbbGGpwag85qvomcB9wP7BDRB4SkWS36FeA84DNIvKOiJxwkNc1Zh+WLIxpXzHOhz7g9BHgfOBvBbYBOe62VnlhP28B7lTV1LCveFV96jBjSMBp1toKoKr3quoEYBROc9TN7vaFqjoN6IfTXDb7IK9rzD4sWRjTvtnA+SJyhogEgB/iNCV9CMwHWoDviYhfRC4BJoUd+xfg2yJyvNsRnSAi54tI0kHG8CRwrYiMdfs7/hen2WyTiEx0zx8AaoEGIOj2qVwlIilu81kVEDyM34MxgCULY9qlqquBq4E/AjtxOsMvUNUmVW0CLgFmAuU4/RvPhh27CKff4j53/zq37MHG8Abwc+AZnNrM0cB0d3cyTlIqx2mqKsPpVwH4KrBJRKqAb7v3YcxhEXv4kTHGmI5YzcIYY0yHLFkYY4zpkKfJQkSmishqdwbqLe3s/707y/VTEVnjDjEM35/szkC9z8s4jTHGHJjfqxOLSBTOGPCzgCJgoYjMU9WVrWVU9Qdh5b/LnglRrf4HeMerGI0xxnSOZ8kCZyjhOlXdACAiTwPTgJX7KT8DuK31hYhMALKBl4HCji6WmZmp+fn5hxmyMcb0LYsXL96pqlkdlfMyWeTgTE5qVQQc315BETkKGIy7ho2I+IDf4gwBPGN/FxCRWcAsgLy8PBYtWtQlgRtjTF8hIps7LuVtn4W0s21/43SnA3NUtXXy0A3Ai6q6ZT/lnZOpPqSqhapamJXVYWI0xhhziLysWRThLI/QKhdn+YL2TAe+E/b6BOBkEbkBZ62eaBGpUdV9OsmNMcZ4z8tksRAoEJHBOGvZTAeubFtIRIYDaThLKACgqleF7Z8JFFqiMMaYyPEsWahqi4jcCLwCROEstbxCRO4AFqnqPLfoDOBp9WAqeXNzM0VFRTQ0NHT1qfus2NhYcnNzCQQCkQ7FGNONes1yH4WFhdq2g3vjxo0kJSWRkZHB3guEmkOhqpSVlVFdXc3gwYMjHY4xpguIyGJV7XDEaa+ewd3Q0GCJoguJCBkZGVZTM6YP6tXJArBE0cXs92lM39Trk0VHgqEQO6oaqGtqiXQoxhjTY/X5ZKGKkywavXk+TEVFBQ888MBBH3feeedRUVHRcUFjjOkGfT5ZRPmcZpWgRx39+0sWweCBk9OLL75IamqqJzEZY8zB8nKexRFBRIgSIRjyJlnccsstrF+/nrFjxxIIBEhMTGTAgAF8+umnrFy5kosuuogtW7bQ0NDA97//fWbNmgVAfn4+ixYtoqamhnPPPZeTTjqJDz/8kJycHJ5//nni4uI8idcYY9rTZ5LFL/+9gpXFVe3uq2sKEuUTYvwHV9EaOTCZ2y4YdcAyd911F8uXL+fTTz/l7bff5vzzz2f58uW7h54+8sgjpKenU19fz8SJE/nKV75CRkbGXudYu3YtTz31FH/5y1+4/PLLeeaZZ7j6antSpjGm+/SZZHEgIs4cgu4wadKkveYo3HvvvcydOxeALVu2sHbt2n2SxeDBgxk7diwAEyZMYNOmTd0SqzHGtOozyeJANYD1pTUAHJ2V6HkcCQkJu39+++23ef3115k/fz7x8fGceuqp7c5hiImJ2f1zVFQU9fX1nsdpjDHh+nwHN+Bpn0VSUhLV1dXt7qusrCQtLY34+Hg+//xzFixY4EkMxhhzuPpMzeJAonxCsNmbZJGRkcGJJ57I6NGjiYuLIzs7e/e+qVOn8uCDDzJmzBiGDx/O5MmTPYnBGGMOV69eG2rVqlWMGDGiw2OLK+opr21iVE6KV+H1Kp39vRpjej5bG+ogRPmEoGq3dXIbY8yRxpIFYRPzPOq3MMaYI50lC5wObvBuFrcxxhzpLFlgNQtjjOmIJQssWRhjTEcsWWDJwhhjOmLJAvD1oD6LxERnFnlxcTGXXnppu2VOPfVU2g4Tbuuee+6hrq5u92tb8twYczg8TRYiMlVEVovIOhG5pZ39vxeRT92vNSJS4W4fKyLzRWSFiCwTkSu8jLO1ZhHqQTWLgQMHMmfOnEM+vm2ysCXPjTGHw7NkISJRwP3AucBIYIaIjAwvo6o/UNWxqjoW+CPwrLurDviaqo4CpgL3iIhnn3Q+AcGbJT9+8pOf7PU8i9tvv51f/vKXnHHGGYwfP55jjz2W559/fp/jNm3axOjRowGor69n+vTpjBkzhiuuuGKvtaGuv/56CgsLGTVqFLfddhvgLE5YXFzMaaedxmmnnQY4S57v3LkTgN/97neMHj2a0aNHc8899+y+3ogRI7juuusYNWoUZ599tq1BZYzZzcvlPiYB61R1A4CIPA1MA1bup/wM4DYAVV3TulFVi0WkBMgCDr0d5aVbYPtn7e4SYEhTC36fgD+q8+fsfyyce9cBi0yfPp2bbrqJG264AYDZs2fz8ssv84Mf/IDk5GR27tzJ5MmTufDCC/f7fOs//elPxMfHs2zZMpYtW8b48eN377vzzjtJT08nGAxyxhlnsGzZMr73ve/xu9/9jrfeeovMzMy9zrV48WL+9re/8dFHH6GqHH/88XzpS18iLS3NlkI3xuyXl81QOcCWsNdF7rZ9iMhRwGDgzXb2TQKigfXt7JslIotEZFFpaelhBSuAF41Q48aNo6SkhOLiYpYuXUpaWhoDBgzgpz/9KWPGjOHMM89k69at7NixY7/nePfdd3d/aI8ZM4YxY8bs3jd79mzGjx/PuHHjWLFiBStX7i8XO95//30uvvhiEhISSExM5JJLLuG9994DbCl0Y8z+eVmzaO/P5P19Hk8H5qjqXs8aFZEBwGPANaoa2udkqg8BD4GzNtQBo+mgBlBcUk2Uz8fgzIQDljsUl156KXPmzGH79u1Mnz6dJ554gtLSUhYvXkwgECA/P7/dpcnDtVfr2LhxI3fffTcLFy4kLS2NmTNndnieAy1pYkuhG2P2x8uaRREwKOx1LlC8n7LTgafCN4hIMvAC8DNV9Xztbp+Hy5RPnz6dp59+mjlz5nDppZdSWVlJv379CAQCvPXWW2zevPmAx59yyik88cQTACxfvpxly5YBUFVVRUJCAikpKezYsYOXXnpp9zH7Wxr9lFNO4bnnnqOuro7a2lrmzp3LySef3IV3a4zpjbysWSwECkRkMLAVJyFc2baQiAwH0oD5YduigbnAo6r6Lw9jhGAz7FxLsqRRFvLm4UejRo2iurqanJwcBgwYwFVXXcUFF1xAYWEhY8eO5Zhjjjng8ddffz3XXnstY8aMYezYsUyaNAmA4447jnHjxjFq1CiGDBnCiSeeuPuYWbNmce655zJgwADeeuut3dvHjx/PzJkzd5/jm9/8JuPGjbMmJ2PMAXm6RLmInAfcA0QBj6jqnSJyB7BIVee5ZW4HYlX1lrDjrgb+BqwIO91MVf10f9c65CXKQyHYvpRKfyZbW1IYOTD5YG6xT7Ilyo3pPTq7RLmnDz9S1ReBF9ts+0Wb17e3c9zjwONexrabzwcShZ/g7mXK9zcqyRhj+iqbwQ0Q5SeKFlSVHjCJ2xhjepxenyw61czmCxDlDsTqCUt+9GT2gChj+qZenSxiY2MpKyvr+APOF8CnLYAtJnggqkpZWRmxsbGRDsUY08087bOItNzcXIqKiuhwwl59OdpUy45QI1oeQ7S/V+fQwxIbG0tubm6kwzDGdLNenSwCgQCDBw/uuOD798Drt3FpwyPcN/MkTj8m2/vgjDHmCGJ/QgMk9Qegn5RTVd8S4WCMMabnsWQBkNgPgCwqqWpojnAwxhjT81iyAEh0mp2ypILKOksWxhjTliUL2J0sBvqrrGZhjDHtsGQBEJcOPj+5/ioq6y1ZGGNMW5YswFnyI6Ef/aMqrYPbGGPaYcmiVWI/+ol1cBtjTHssWbRK6k8m5dYMZYwx7bBk0SqxH6nBcqtZGGNMOyxZtErMJjFYQXVdY6QjMcaYHseSRavEbHyECDTuImSLCRpjzF4sWbRqnZhHBTVNNiLKGGPCWbJotXsWd6XN4jbGmDYsWbRy14fqJ9bJbYwxbXmaLERkqoisFpF1InJLO/t/LyKful9rRKQibN81IrLW/brGyziBsGaoShs+a4wxbXj2PAsRiQLuB84CioCFIjJPVVe2llHVH4SV/y4wzv05HbgNKAQUWOweW+5VvETHEwwkkdVSYbO4jTGmDS9rFpOAdaq6QVWbgKeBaQcoPwN4yv35HOA1Vd3lJojXgKkexgpAKKEfWVJhzVDGGNOGl8kiB9gS9rrI3bYPETkKGAy8eTDHisgsEVkkIos6fHRqJ0hSNllSSZU1QxljzF68TBbSzrb9TWCYDsxR1eDBHKuqD6lqoaoWZmVlHWKYe0Ql96efVFBW23TY5zLGmN7Ey2RRBAwKe50LFO+n7HT2NEEd7LFdRhKz6SeV7Khs8PpSxhhzRPEyWSwECkRksIhE4ySEeW0LichwIA2YH7b5FeBsEUkTkTTgbHebtxL7kUA9u8orOi5rjDF9iGejoVS1RURuxPmQjwIeUdUVInIHsEhVWxPHDOBpVdWwY3eJyP/gJByAO1R1l1ex7pbUH4CmSs8rMcYYc0TxLFkAqOqLwItttv2izevb93PsI8AjngXXHndinlaXoKqItNd1YowxfY/N4A7nTsxLDe2iwpb8MMaY3SxZhEt0mqGypIJt1sltjDG7WbIIF59ByBdggOxiW2V9pKMxxpgew5JFOJ+PUPIg8mSH1SyMMSaMJYs2ojKGcJSUst2ShTHG7GbJog1Jz+co3w62VVgzlDHGtLJk0VbaYJKoo6aiJNKRGGNMj2HJoq30wQD4KjZHOBBjjOk5LFm0lZYPQHztF4RNKjfGmD7NkkVbbrLoH9xuD0EyxhiXJYu2ohNoiMkiT0rYVmWd3MYYA5Ys2tWScpQ7IsqGzxpjDFiyaJcvI98m5hljTBhLFu2IyRpKf8optedaGGMMYMmiXVEZQ/CJ0rhzU6RDMcaYHsGSRXt2z7XYFNk4jDGmh7Bk0R53+GxstU3MM8YYsGTRvoQsmnxxJDdstYl5xhiDx8lCRKaKyGoRWScit+ynzOUislJEVojIk2Hbf+1uWyUi90p3PuNUhOr4XAaGtlPdaBPzjDHGs2dwi0gUcD9wFlAELBSReaq6MqxMAXArcKKqlotIP3f7FOBEYIxb9H3gS8DbXsXbVlNSHkdVfc72ygaSYwPddVljjOmRvKxZTALWqeoGVW0CngamtSlzHXC/qpYDqGrrUq8KxALRQAwQAHZ4GOs+JH0wg6SE4vLa7rysMcb0SF4mixxgS9jrIndbuGHAMBH5QEQWiMhUAFWdD7wFbHO/XlHVVW0vICKzRGSRiCwqLS3t0uBj+w0lVpqpLNnScWFjjOnlvEwW7fUxtO0t9gMFwKnADOBhEUkVkaHACCAXJ8GcLiKn7HMy1YdUtVBVC7Oysro0+MQBBQA0lqzv0vMaY8yRyMtkUQQMCnudCxS3U+Z5VW1W1Y3AapzkcTGwQFVrVLUGeAmY7GGs+/BnOHMtWnZu6M7LGmNMj+RlslgIFIjIYBGJBqYD89qUeQ44DUBEMnGapTYAXwBfEhG/iARwOrf3aYbyVGoeQXz4Kjd162WNMaYn8ixZqGoLcCPwCs4H/WxVXSEid4jIhW6xV4AyEVmJ00dxs6qWAXOA9cBnwFJgqar+26tY2xUVoCpmAEm1WwiFbK6FMaZv82zoLICqvgi82GbbL8J+VuC/3K/wMkHgW17G1hkNyfnk1W+lqLyevIz4SIdjjDERYzO4DyAqcyj5soM126siHYoxxkSUJYsDSM4ZTpLUU7T1i0iHYowxEWXJ4gBis4cDUF38eYQjMcaYyLJkcSAZQwAIla6LcCDGGBNZliwOJCWPIFHEVW+yEVHGmD7NksWBRPmpTRhErm5ja0V9pKMxxpiIsWTRgVD6EAbLNtaWVEc6FGOMiRhLFh2Iyx5GvuxgrQ2fNcb0YZYsOhCTXUCcNLFj66ZIh2KMMRFjyaIj6UcD0LRjTYQDMcaYyLFk0ZEMJ1n4KzbYiChjTJ9lyaIjybkEfdEMDG2juNJGRBlj+iZLFh3x+WhMzmewbGdtSU2kozHGmIjoVLIQke+LSLI4/ioiS0TkbK+D6yn8mUPJl+2s3WHDZ40xfVNnaxZfV9Uq4GwgC7gWuMuzqHqY6H5DOcq3gzXbKiMdijHGRERnk0Xr87TPA/6mqktp/xnbvVPGUKJpobTI1ogyxvRNnU0Wi0XkVZxk8YqIJAEh78LqYdwRUezaQENzMLKxGGNMBHQ2WXwDuAWYqKp1QACnKapvcOda5LGNVdtsJrcxpu/pbLI4AVitqhUicjXwM6DvNOAn9ScUiGeIbGP51r5z28YY06qzyeJPQJ2IHAf8GNgMPNrRQSIyVURWi8g6EbllP2UuF5GVIrJCRJ4M254nIq+KyCp3f34nY+16Iki/kYz1b+YzSxbGmD6os8miRVUVmAb8QVX/ACQd6AARiQLuB84FRgIzRGRkmzIFwK3Aiao6CrgpbPejwG9UdQQwCSjpZKyekLzJjGY9q7aURjIMY4yJiM4mi2oRuRX4KvCCmwgCHRwzCVinqhtUtQl4GifZhLsOuF9VywFUtQTATSp+VX3N3V7j9pVETt4JBGgmbucy6+Q2xvQ5nU0WVwCNOPMttgM5wG86OCYH2BL2usjdFm4YMExEPhCRBSIyNWx7hYg8KyKfiMhv3AS1FxGZJSKLRGRRaanHf/HnnQBAIZ9bJ7cxps/pVLJwE8QTQIqIfBloUNWO+izam4fRdiU+P1AAnArMAB4WkVR3+8nAj4CJwBBgZjtxPaSqhapamJWV1ZlbOXQJGTSnFzDR97l1chtj+pzOLvdxOfAxcBlwOfCRiFzawWFFwKCw17lAcTtlnlfVZlXdCKzGSR5FwCduE1YL8BwwvjOxesmffyKFUWtZXrQr0qEYY0y36mwz1H/jzLG4RlW/htMf8fMOjlkIFIjIYBGJBqYD89qUeQ44DUBEMnGanza4x6aJSGt14XRgZSdj9YwcdQJJ1FH9xWeRDsUYY7pVZ5OFr7Xz2VXW0bFujeBG4BVgFTBbVVeIyB0icqFb7BWgTERWAm8BN6tqmaoGcZqg3hCRz3CatP7S6bvyittvkV2+xDq5jTF9ir+T5V4WkVeAp9zXVwAvdnSQqr7Ytpyq/iLsZwX+y/1qe+xrwJhOxtc9UvOoj+vP+JrP+Xx7NWMHpUY6ImOM6Rad7eC+GXgI58P7OOAhVf2Jl4H1SCLooBOY6FvNZ0UVkY7GGGO6TWdrFqjqM8AzHsZyRIgbeiLxa+aydeMqOCE/0uEYY0y3OGCyEJFq9h3uCk4fgqpqsidR9WBy1BQAdPN8nMnpxhjT+x0wWajqAZf06JOyRtDoTya/dhlbdtUxKD0+0hEZY4zn7BncB8vno3nQFE6O+ox310R0uSpjjOk2liwOQcLo88iVnaxfvjDSoRhjTLewZHEIpOBsAJKL3qQl2HceGGiM6bssWRyK5AFUpo5kSmgxS4tsnShjTO9nyeIQxYw4lwmyho9XrI10KMYY4zlLFocodtT5RInS8PmrkQ7FGGM8Z8niUA0cR20gnaPLP6CyvjnS0RhjjKcsWRwqn4/6vNM4xbeUBWu3RzoaY4zxlCWLw5A69gJSpZbNS9+JdCjGGOMpSxaHwV9wOkGiiN30OqFQe6uiGGNM72DJ4nDEplCWOZFTmj9kWVF5pKMxxhjPWLI4TImTryXft4PVHzwX6VCMMcYzliwOU/zYS6jwpTFo3RM4z3Iyxpjex5LF4fJHs2Xw5UxuWcz6NSsiHY0xxnjCkkUXGHjGDYQQKt59MNKhGGOMJzxNFiIyVURWi8g6EbllP2UuF5GVIrJCRJ5ssy9ZRLaKyH1exnm4MgbmszBuCsOL50JTXaTDMcaYLudZshCRKOB+nMfJjQRmiMjINmUKgFuBE1V1FHBTm9P8D3BETGIoH3kNSVpD6YInOy5sjDFHGC9rFpOAdaq6QVWbgKeBaW3KXAfcr6rlAKq6+2lCIjIByAaOiMWXxp78ZVaHcvF99CcItkQ6HGOM6VJeJoscYEvY6yJ3W7hhwDAR+UBEFojIVAAR8QG/BW4+0AVEZJaILBKRRaWlpV0Y+sEbmBbPcylfJaN2HXxkfRfGmN7Fy2Qh7WxrO7bUDxQApwIzgIdFJBW4AXhRVbdwAKr6kKoWqmphVlZWF4R8eDImXsbrwXGE3vwVlG+OdDjGGNNlvEwWRcCgsNe5QHE7ZZ5X1WZV3QisxkkeJwA3isgm4G7gayJyl4exdolLJgzil6Fv0BwSeOG/wOZdGGN6CS+TxUKgQEQGi0g0MB2Y16bMc8BpACKSidMstUFVr1LVPFXNB34EPKqq7Y6m6knSE6IZM3IUf9ArYN3rsPyZSIdkjDFdwrNkoaotwI3AK8AqYLaqrhCRO0TkQrfYK0CZiKwE3gJuVtUyr2LqDpdPHMSD9WdQnjYGXvlv6+w2xvQK0luWqCgsLNRFixZFOgyCIeXk//cm0xOX8L2yX8HMFyD/pEiHZYwx7RKRxapa2FE5m8HdxaJ8wqWFg/hz8RA0KhZW/TvSIRljzGGzZOGByybkUkcsG1KOd5JFL6m9GWP6LksWHhiUHs9JQzN5omoMVG2F4iWRDskYYw6LJQuPzJySzzM1xxISvzVFGWOOeJYsPHL6Mf0YOGAgS3yj0ZXzrCnKGHNEs2ThERHhu6cPZW7DeGTXeij9PNIhGWPMIbNk4aGpo/qzNu0UQgihlW3nIxpjzJHDkoWHfD5hxhkTWRwqoPaTZyAUjHRIxhhzSCxZeOyCMQN5M/YskipXo49dDDUlHR9kjDE9jCULj/mjfAw+63pubp5FaPMCePBk2PRBpMMyxpiDYsmiG1wyPocl6efz7dj/h0YnwKMXwtrXIh2WMcZ0miWLbuCP8nHzOcfw2q5+zC18DPqNhH9+FbZ8HOnQjDGmUyxZdJNzRmUzLi+VX7+9nYYrZkPyAHjiMihZFenQjDGmQ5YsuomI8JOpx7C9qoG/L6uDr84Ffww8dok9Vc8Y0+NZsuhGk4dkcNrwLB54ax27ogfC1c9Cc63Th1HV9iGCxhjTc1iy6Ga3njeC2qYgv375c+g/2kkYtTvh0WlQUxrp8Iwxpl2WLLrZsOwkvnHSYJ5euIUlX5RDbiFcORsqtoDNwzDG9FCWLCLg+2cU0D85lp/NXU5LMAT5J8L0J6BsLTx0KhR/EukQu0Z9hT1W1phewtNkISJTRWS1iKwTkVv2U+ZyEVkpIitE5El321gRme9uWyYiV3gZZ3dLiPHz8y+PZOW2Kh5f4HZuDz0Dvv4KIPDIVFj6dERjPGzBZrh3HCx4INKRGGO6gGfJQkSigPuBc4GRwAwRGdmmTAFwK3Ciqo4CbnJ31QFfc7dNBe4RkVSvYo2E847tz8kFmfz21TWs2lblbBw4Fma9DTmFMPdb8OF9kQzx8FR8AfW7YOO7kY7EGNMFvKxZTALWqeoGVW0CngamtSlzHXC/qpYDqGqJ+32Nqq51fy4GSoAsD2PtdiLCnRcdS0KMn8senM97a93O7cQs+NpzMHIavPrf8MnjkQ30UO3a4Hzfutie5WFML+BlssgBtoS9LnK3hRsGDBORD0RkgYhMbXsSEZkERAPr29k3S0QWicii0tIjbyRRXkY8c78zhdy0OK7920L+tcj9dUUF4JK/wJDTYN53YdV/IhvooWhNFvW7oHxjZGMxxhw2L5OFtLOt7Z+YfqAAOBWYATwc3twkIgOAx4BrVTW0z8lUH1LVQlUtzMo6MiseA1LimP3tE5g8JIOb5yzjmcVFzg5/DFzxOORMgDnXwucvRDbQg9WaLAC22jPIjTnSeZksioBBYa9zgbYzz4qA51W1WVU3Aqtxkgcikgy8APxMVRd4GGfEJccGeGTmRKYcncGtz37G4s27nB0xic6w2uzR8PSV8O5vjpwmnV0bnDWw/HFOU5Qx5ojmZbJYCBSIyGARiQamA20fF/cccBqAiGTiNEttcMvPBR5V1X95GGOPEe338cBV4xmYGsu3HltMUXmdsyM+Ha59EY69HN78lVPL2Lmu5z9IqWw9ZBY4nfaWLIw54nmWLFS1BbgReAVYBcxW1RUicoeIXOgWewUoE5GVwFvAzapaBlwOnALMFJFP3a+xXsXaU6TGR/PwNRP0CUn1AAAcG0lEQVRpbAnxzX8soqbRnaMQiINLHoIzfwkrnoP7JsD/5jhzMuY/AC1NEY17H8EWqNgM6UOcZrRtS52htMaYI5bokdKs0YHCwkJdtGhRpMPoEu+uKeXavy/k+MHpPDJzIrGBqD07S9dA0cewY6XzvWghpA2Gs34JIy4Eaa+rqJvt2gj3joUL74PoeJjzdZj1jlPLMMb0KCKyWFULOypnM7h7oFOGZXH3ZWP4cH0ZNz65hOZgWN9+1jAYdzVM/V/45utw1TPgj4XZX4PnbugZfRq73IFrrTULsKYoY45wlix6qIvH5fI/00bx+qoSfvSvpYRC+0kCBWfCt9+Hk/4Llj4J7/9uzz5VWPVvWPIoNNV1T+Dg1CwAMo6G1KMgPtNGRBlzhPNHOgCzf189IZ/qxhZ+/fJqmlpC/O7yscRFR+1bMMoPZ/zCmTX9xh2QOQwGnwL/+S9YPscp8/rtMPE6OP5bTqe5l3ZtgEA8JGY7zWI5E2Br72giNKavsppFD3fDqUP52fkjeHnFdq54aD4lVQ3tFxSBafc5H8zPzoIHT4IVc+H0n8HMFyF3ErxzFzx8JrQ0ehv0rg1OE1Rr/0nOBChdDQ1V3l7XGOMZSxZHgG+ePIS/fLWQdSU1TLv/A5ZvrWy/YCAOpj8JcenO9MdrX4JTbnZWtb3yaZjxT6c/4aMHvQ24bL2TLFrlTAAUtn3q7XWNMZ6xZHGEOHNkNnO+PQUBvvKnD5nTOtO7raT+8J0FcONCyDt+733Dp8KwqfDOb7x7bkYoCOWb2iSL8c73ooXeXNMY4zlLFkeQkQOTmffdkxifl8aP/rWUnz+3nKaWfVZBgZgkCMS2f5KzfwUt9c4EPy9UFkGoee9kEZ9OadIx1C1+umeM1jLGHDRLFkeYzMQYHvvGJL51yhAeW7CZ8+59b8+KtUAopCzfWsn2yv30bWQWwKRZzgipbcu6PsDWYbMZR+/eVNfUwl27TiW+Yg2sf6Prr2mM8ZwliyOQP8rHreeN4JGZhTQHQ3z1rx8z69FF/HTuZ0z+vzf48h/f59S73+Khd9c7T+Jr60s/hrg0eP47XT//oXUBwbCaxaptVcwLTmGHptL83h+79nrGmG5hyeIIdvox2bz6g1P48dThvL9uJ89/spXC/DR+c+kYTi7I4n9f/JyLHviAFcVtOsTj0uCCPzhDbf9yOjx6EWx6v/0mopoSKFoMK5+Hz+Z0vLTIro3O4oGJ/Xdv+qyokmb8/KPlHAKb34YdKw7/5o0x3cqW++gl6puCiLB7aRBV5eXl2/nFvBVU1Tfzm8uO48LjBu59UGM1LPwrzL8Pakth0GQ4+Ycw+GRn2O3Cv+47P+Lo0+HyR51+kfY8NQPKN8MNH+7e9MPZS3lnTSkZvhrmNX+LmOO+AhfZ41aN6Qk6u9yHJYterqymkesfX8LHm3bx3dOH8oMzh+HztVk/qrkeljwGH/wBqoogKhqCTc7kvrFXQdZwSM5xmqxe+CFkj4Kr5kBSNoRCUFcG8Rng88F9k5x+kelP7D79Ob9/l4GpsRyVkUDBotu50v8W8oPlzsgtY0xEdTZZ2AzuXi4jMYbHv3k8P39uOX98cx1Liyr58TnDGZ2TsqdQIA6OnwUTZsJns52kMOpiyD9574UJB4yBlFxnHao/nwJxqU6zU7ARohOh/7FOn8XwPQ88rG8KsrakmnNGZTN5SAa3zp/Klb7XnGeMT/4ODDkV/NHd9eswxhwiSxZ9QLTfx11fOZZROcnc/cpqvvzH9zl7ZDY3nj6UMbmpewr6o51FCsddvf+TFZwFM1+A12+DmGQoOBuSBzpJYttS54FNR520u/jKbVWEFEblpDBxcDoVsYN4MeubnF88G568DGJTneud/EPvlyExxhwySxZ9hIjwtRPymTY2h79/sImH39/Aqyt3MGpgMldMHMS043JIiQ907mQ54+Gaf3eqaOts82NzUghE+ThjRD9+uupszr71VwQ2vu3UZBY8AJ88Bqf8GCZd5zxS1hjTo9hoqD4mJS7A988s4P2fnM4vLxyFKvzi+RVMuesN7n1jLXVNLV16vc+2VpKREM2AFGeS4Nkj+1NZ38zCL2qc5qpLH4FvfwC5E+HV/4bfHuOsbfXZHKgv79JYjDGHzmoWfVRKXIBrpuRzzZR8lm+t5L431/G719bw+ILNfP/MAr4yPnfvhy4douVbKxmdk4K4fR9fGpZFbMDHnMVFTBma6RTKHglXPwPr34KlT8PaV2HZP0GiILcQhp4JQ05z+kyOpFpHsMUZMJCWH+lIjDlsNhrK7LZ48y7ufGEVS76oIDU+wOWFg7hsQi79kmNJiI7CH3VwFdGG5iCjbnuF6790ND86Z/ju7f/30ir+/M4GfnvZcXxlQu6+B4aCULQI1r0G616H4k+c7VHR0H+Mk1yiop1kEh0PWSOg/2hn9FZUJ5vSvNZcD/+82on/mC/D6T+HfsdEOipj9mFDZ80hUVXmbyjj8QWbeWXFDoJhD11KjvVzxcRBXHfyEPol72ftqTCffFHOxQ98yINXT2Dq6D3DZFvcWedLvijnmeun7D0yqz21O2HzB04C2boYdq6FUAto0HmoU8h9vrcv4CwzklngJI7MYc7PGQUQm7z/89eUOvNKGiqdkWGBOOd5HIFY53t8JqQd5QwP7sxjaxurnfkmm96H42bA5/+BphoYMx0mfxsGHNfxOYzpJj0iWYjIVOAPQBTwsKre1U6Zy4HbcRbVXqqqV7rbrwF+5hb7lar+40DXsmTR9bZXNvDumlKqG1uobWxhzY5qXvxsG/4oH5cX5nLxuFzGDUrdd96G67H5m/j58yv44JbTyUmN22tfWU0jF/zxfUSEf3/3JNITDnH4bLAZytbB9uVQssJJJDvXOKOzQmH9LzEpkDzAGbmVMghS8yCxn9PktfqlvcvuTyAekgZAQqaTQOJSnW3R8c4kxbg0Z3n4BX9yktrFD8KYy6G2zHmC4cK/Oos4DhznjADLKXSSWXTCod27MV0g4slCRKKANcBZQBGwEJihqivDyhQAs4HTVbVcRPqpaomIpAOLgEKcJLIYmKCq++3xtGTRPTaX1fLgO+uZs7iI5qCSmRjNGcdkc8aIfpxUkEl89J5usB/PWcprK3ew5Odn7e6zCLesqIJLH5xP/+RYvn5iPpcWDiIxpou60YLNzhyQnWugbC1UFe/5qtzizFgHp7Zw3AwY91WnVtJc5zQh7f6qc5Y8qfgCKjZD9Xao2+kkgIZKaK6FplpnEmMrXwAu+xuMuGDvmOrLYdlsWPx3KFm5Z3tKnlsbKoCMoc5clqT+bmLq50x2NMYjPSFZnADcrqrnuK9vBVDV/wsr82tgjao+3ObYGcCpqvot9/WfgbdV9an9Xc+SRfeqrGvm7TUlvL6qhLc/L6G6sYVov4/JQzIYOSCZozLieejdDeSmxfHYN47f73neX7uTu19dzadbKkiK8XPl8XnMOmUIGYked2Q31ULVNqeG0RWTApsboKHCSQixKU4NZn9UnRpQ6SooXQM7VztJbec6J/mE8wWcc6Xm7akRpR3l1GACcU6tJDrRaWaLSXJ+7kxTmTGunjCDOwfYEva6CGj7qTEMQEQ+wGmqul1VX97PsTltLyAis4BZAHl5eV0WuOlYSnyAaWNzmDY2h+ZgiIWbdvHmqhLeWVPK/PU7aQ46f4Tssx5VGycVZHJSQSaffFHOIx9s4i/vbeDxBZv5+kmD+cZJg0mN92h2d3QCZA7tuvMFYiHQv3NLmIhA1jDnK5wqVG9zkli1+1W1FSq2OLWhDW872zjAH3g+PyRkOU1l0Ul7EocvylngMRDrfPfHOF+BOCe5xaY63/0xEBXjlItJciZeRidAfYVTo6ovd17HpTvNbr4o0JATe3z63muGqTrlm+v2LFIZk+hcyxLaEcfLZNHev4a2/8r9QAFwKpALvCciozt5LKr6EPAQODWLwwnWHLpAlI8pR2cy5ehMfgYEQ8r2qga2VdQzamAHndeucXlp/DEvje+fMZTfv76WP765jvvfWsew7CTGH5XG2EGpjMlNYWhW4kGPyjpiiDi1iAPVSloanQdMNVQ4nfvNdU7neWO184zz+nKoLXEGBTSF1VJamtxmswan36SlCVoanOODHawkfDBiU5x1xJrrncTW0s5zVQLxzj1GJzp9RcEmZ2RbTJJTQ8ocBife5Kw9Fi7YAlFtPrLKN8GaV51RcLEpzvGB+D2DFOLSnK+eMkruCOZlsigCBoW9zgWK2ymzQFWbgY0ishoneRThJJDwY9/2LFLTpaJ8Qk5q3D6d2p0xtF8S9185nhtPq+Ll5dtZ8kU5/15azJMffQFAbMDH4MxEWvNFYoyfyyYM4oLjBhLt76VJJJw/Zq8HS3WJ5nonkTRUOR/uwSYnibQmoOY6pzM/PsP54G2qhbpd7qRJBXF/77WlULnV6RcKxDp9Lq1JobUm0VDl9h1tdc4bFe3UhjTo7Ksrg48fch7OddJNMOoS+PwFZ95NySrIm+wsOZM2GD590hmgcKCaVquY5LCEIeCPdZvx4p19sSnOl/ic+29pdJJW4dedgRHG0z4LP04H9xnAVpwO7itVdUVYmak4nd7XiEgm8Akwlj2d2u7Dm1mC08G9a3/Xsz6L3isUUjbsrGX51kqWFVWyuWzPX8wby2rZUFpLVlIMMyblMT4vlYLsJAamxLbbqd6dVJX65uBenf6mE3auc9Ye+/w/e7blFDqz/De/D9s/c7Yl9IMJ18DYK52ms8YqJ8E11e4ZnFBf7ia2Xe5wawXUSQZNtWFJsdL50pCTwKICULbeSWSjL3FqOtkjI/Lr8FrEO7jdIM4D7sHpj3hEVe8UkTuARao6T5z/zb8FpgJB4E5Vfdo99uvAT91T3amqfzvQtSxZ9E2qyrtrd/LI+xt5Z82ex8smREcxNDuJYf0SKchOJDMxhpS4AKnx0YwYkOT5B3hTS4ifPLOMeUuLOWtENldPPoopR2fsd5ixacemD6B4CQw/b+/aVFWxM0Ag7wRvVyzetRE++rOzblmwCc79tbMyc+sfITvXOkO0h5x6ZK0s0EaPSBbdyZKFKa9tYm1JDetKalizo5q1JdWs3l7DzprGvcr5fcJxg1I5YUgGeRnxpMdHk5bgJJK0+GiSY/2H1S9SWd/Mtx9bzPwNZZx3bH/mry+jvK6Z/Ix4vjxmIOce25+RA5IjXvMxnVRbBs9eB+vfYMfRl/J+yoVcVP8sUaueB9SZczP+a87cmfQhne+8r90Jm96Dje/Clo+djv+Mo53h0wmZYaPbfO4ggqDbD5PuDCaIS9+3D+cQWLIwxlVZ30x5bRMV9c3srG5k8RflzF9fxmdbK/eaoR4uIyGa7ORY+qfEkhjjxyfOyr25aXGcOSKbY3NSEIHPt1fz0vLtbNpZS3pCNFlJMTz/6VY27qzl15eO4eJxuTQ0B3lp+TZmLyzio41lhBRy0+IYOyiVEQOSGTUwmSlHZ/aNPpcjVSjIjn/fTvYn9wJQSxwVo2eSc+yX0CWPwpqXEQ05/R79RsLA8c4zYsLXBasphc/+BUULnRpT+SZne3QSDJroNIuVrXP6bTpFnKSSmO2soXbBHw7p1ixZGNOB+qYgO2saKa9roqy2icq6Zsrrmiiva6a0upEdVQ1sr2ygrqkFxRnlVVxRT0ghOzmG2EAUm8vq8AnkpMVRUdtMdWMLybF+Hrx6wp6FEsOU1TTy2sodvLW6hBXFVRSV1wOQmRjN9Il5XDFxEBV1zXy6pZwVxVXOiNSYKBJj/ByVkcCxOSkcnZXQe0eE9VDbKuuZdt8HnMhSZg2v54frj2NluY9xeamsL6khsWE7p0d9wrlZZRyfsB3/9qVOTWDc1c4yL8v+CUufcgYQpAxyZvHnTID8k2DA2L1rCPUVTh9LY40z0k1Dzmgx8TnzcOp2uV87oWaHM2k0OQfOv/uQ7s2ShTEeKK9tciYjriyhrqmFs0b25+xR2WS6kwgbmp1nocf4O7dib1VDM4s27eKJBV/w5uoSwv87psUHiPb7qGsMUtvUQmslKDbgIzEmQHMwRHMwRGKMn4Hu6LMBKbFkJ8eSnRJLZkI0ibF+EmP8xASiCAaVoCoxfh/9k2Ot/6STahtbuOzB+Xyxq45nrp/C8P5J1DcFueeNNSxYX8bIgcmMHZTK1vJ6Hnh7PWkJ0fzmnCy+tOMxZMk/nP6OqBgYO8N5OmTb+TURZsnCmCPMll11vLx8O/1TYhk7KJXctLjd/RrBkLKhtIbPtlayoriKuqYgMX4ffp9Q1dBMcUUDWyvq2VZZT0NzqMNrRft95KXHMygtjrQEp68mxu9jW2UDW8vrKattpF9SLLlpcWQnx1LT2EJFXRNVDS2kxgXISo4hKzEGnwgtoRDBEAxIiaUgO5GjsxJ3L2/f+vlypPbPNLYE+dZji3l3TSmPzJzIqcP7HbD88q2V/OhfS/l8ezX9k2O5YrhwYdIa+hdeSEL6gSeoRoolC2P6IFWlqqGFHVUN7KptoqahhZrGFhpbgkT5fET5oK4pyBdldWwqq6WovJ4Kt/mtoTnIgJQ4ctLiyEiIZkeVk4BKqhtJjPaTlhBNUqyfCreZrinYflISgYDPR0soREghECUkxQZIjPGTHOcnOTZASlyAhBg/MX4fMf4oEmKiSI2PJjUuQHpC9O6vxBg/9c1B6pqClFY3srSogk+/qGBTWS25aXEMyUpkcGYCA1PjGJgSy4DUuENeX0xV90pqzcEQNzyxhNdW7uD/LjmWGZM6t0pEU0uIl5Zv44Vl23h7TSlNLc7vaVB6HEOzEkmJCxAX7ScxJoq89HiOzkokPzOBlqBSXuf0rSVER9EvKZZ+bnOnlyxZGGMOSiik7TZNtf0Qbd1WVd+CovijfPgEtpbXs2aHMxqtoSWI3yf4RGgKhqhpaKG6oZmqhhYq65uprG+mrrGFxpYQjS0hapta6OxHUV56PEOyEiiuqGdTWd3uD+NWmYkxDMlMIC8jnkCUj5ZgiKAqKXEB+iXFkpkYTXldExt31rJxZy0l1Y2U1zZRWd9MfkYClxUO4qJxA/mf/6zkxc+2c8e0UXzthPxD+p1WNzQzf30Zq7dXs3pHNRtKa6lpbKGuKUh1QzONLR3XAmMDPpJjAyTF+slMjGFASiz9U+JIiw8QHx1FXLSf7OQYTi7IOqQYLVkYY44YoZBS1dBMRV0zu+qa2FXTxK7aJmqbWogLRBEX7dQ8Rg9M3muRyWBI2VZZz7bKBoor6imuaGDTzlo27Kxhc1nd7pqNT4SKuiZqm4K7j01PiCY/I54BKXGkJTi1nYUby/l40565vz87fwTfPHmIJ/esquyoamR9aQ2bymqJjvKRnhBNSlyAmsYWSqobKa1upLK+mSo3we6saWS7O/Cidf01gHF5qcy94cRDiqMnLCRojDGd4vOJ0wwVH00+nX++R5RPyE2LJzctvlPlaxtbKK1uJDU+sN9FKjeU1vDMkiJyUuO58njvFigVEfqnOMOzT2xn5NyBhEK6u3muvinYLesyWrIwxvQZCTF+Ejro0xiSlcjN5/TsR+D6fNKpe+nSa3bblYwxxhyxLFkYY4zpkCULY4wxHbJkYYwxpkOWLIwxxnTIkoUxxpgOWbIwxhjTIUsWxhhjOtRrlvsQkVJg82GcIhPY2UXhHCn64j1D37zvvnjP0Dfv+2Dv+ShV7XBhqV6TLA6XiCzqzPoovUlfvGfom/fdF+8Z+uZ9e3XP1gxljDGmQ5YsjDHGdMiSxR4PRTqACOiL9wx987774j1D37xvT+7Z+iyMMcZ0yGoWxhhjOmTJwhhjTIf6fLIQkakislpE1onILZGOxysiMkhE3hKRVSKyQkS+725PF5HXRGSt+z0t0rF2NRGJEpFPROQ/7uvBIvKRe8//FJH2H5l2BBORVBGZIyKfu+/5Cb39vRaRH7j/tpeLyFMiEtsb32sReURESkRkedi2dt9bcdzrfr4tE5Hxh3rdPp0sRCQKuB84FxgJzBCRkZGNyjMtwA9VdQQwGfiOe6+3AG+oagHwhvu6t/k+sCrs9f8Dfu/ecznwjYhE5a0/AC+r6jHAcTj332vfaxHJAb4HFKrqaCAKmE7vfK//Dkxts21/7+25QIH7NQv406FetE8nC2ASsE5VN6hqE/A0MC3CMXlCVbep6hL352qcD48cnPv9h1vsH8BFkYnQGyKSC5wPPOy+FuB0YI5bpDfeczJwCvBXAFVtUtUKevl7jfOY6DgR8QPxwDZ64Xutqu8Cu9ps3t97Ow14VB0LgFQRGXAo1+3rySIH2BL2usjd1quJSD4wDvgIyFbVbeAkFKBf5CLzxD3Aj4GQ+zoDqFDVFvd1b3zPhwClwN/c5reHRSSBXvxeq+pW4G7gC5wkUQkspve/163299522WdcX08W0s62Xj2WWEQSgWeAm1S1KtLxeElEvgyUqOri8M3tFO1t77kfGA/8SVXHAbX0oian9rht9NOAwcBAIAGnCaat3vZed6TL/r339WRRBAwKe50LFEcoFs+JSAAnUTyhqs+6m3e0Vkvd7yWRis8DJwIXisgmnCbG03FqGqluUwX0zve8CChS1Y/c13Nwkkdvfq/PBDaqaqmqNgPPAlPo/e91q/29t132GdfXk8VCoMAdMRGN0yE2L8IxecJtq/8rsEpVfxe2ax5wjfvzNcDz3R2bV1T1VlXNVdV8nPf2TVW9CngLuNQt1qvuGUBVtwNbRGS4u+kMYCW9+L3GaX6aLCLx7r/11nvu1e91mP29t/OAr7mjoiYDla3NVQerz8/gFpHzcP7ajAIeUdU7IxySJ0TkJOA94DP2tN//FKffYjaQh/Mf7jJVbdt5dsQTkVOBH6nql0VkCE5NIx34BLhaVRsjGV9XE5GxOJ360cAG4FqcPw577XstIr8ErsAZ+fcJ8E2c9vle9V6LyFPAqThLke8AbgOeo5331k2c9+GMnqoDrlXVRYd03b6eLIwxxnSsrzdDGWOM6QRLFsYYYzpkycIYY0yHLFkYY4zpkCULY4wxHbJkYUwPICKntq6Ka0xPZMnCGGNMhyxZGHMQRORqEflYRD4VkT+7z8qoEZHfisgSEXlDRLLcsmNFZIH7HIG5Yc8YGCoir4vIUveYo93TJ4Y9g+IJd0KVMT2CJQtjOklERuDMED5RVccCQeAqnEXrlqjqeOAdnBm1AI8CP1HVMTgz51u3PwHcr6rH4axf1Lr8wjjgJpxnqwzBWdvKmB7B33ERY4zrDGACsND9oz8OZ8G2EPBPt8zjwLMikgKkquo77vZ/AP8SkSQgR1XnAqhqA4B7vo9Vtch9/SmQD7zv/W0Z0zFLFsZ0ngD/UNVb99oo8vM25Q60hs6BmpbC1ywKYv8/TQ9izVDGdN4bwKUi0g92P/f4KJz/R60rm14JvK+qlUC5iJzsbv8q8I77DJEiEbnIPUeMiMR3610YcwjsLxdjOklVV4rIz4BXRcQHNAPfwXm40CgRWYzzhLYr3EOuAR50k0Hryq/gJI4/i8gd7jku68bbMOaQ2KqzxhwmEalR1cRIx2GMl6wZyhhjTIesZmGMMaZDVrMwxhjTIUsWxhhjOmTJwhhjTIcsWRhjjOmQJQtjjDEd+v8u8XXS9gQQsAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# chart accuracy and loss for train and validation sets\n",
    "\n",
    "print(modelfit.history.keys())\n",
    "#  acc\n",
    "plt.plot(modelfit.history['acc'])\n",
    "plt.plot(modelfit.history['val_acc'])\n",
    "plt.title('model accuracy')\n",
    "plt.ylabel('accuracy')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'validation'], loc='upper left')\n",
    "plt.show()\n",
    "# Loss\n",
    "plt.plot(modelfit.history['loss'])\n",
    "plt.plot(modelfit.history['val_loss'])\n",
    "plt.title('model loss')\n",
    "plt.ylabel('loss')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'validation'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Confusion matrix <a name='confusionmatrix' />\n",
    "<a href=#linkanchor>Back to link list</a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWcAAAEWCAYAAAC6xlbpAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xm8XeO9x/HPN+dEDBEZEJEYUjU0uLjUVIpqEdoKVWKo0GhuiaGlJbSXlurVIm3dotJriKHSFC3X7KZF1RBqTkwxROZEEARJzjm/+8d6ju6cnGGfc/bJWWf5vvNar+z9rGev9ay11/6d337Ws9ZWRGBmZvnSrbMbYGZmK3JwNjPLIQdnM7MccnA2M8shB2czsxxycDYzy6EuFZwlrSbpfyUtkvSndiznSEn3VrJtnUHSXZJGtPG1P5P0lqS5lW5XVyPpGEkPlTz/QNJn2rCcDjuuJPWQNFXSgAous+z2NtxHlVa6/LStL0pap6PW1xV0SHCWdISkJ9JBPicFkd0qsOhDgP5Av4j4ZlsXEhE3RMQ+FWjPciTtKSkk/blB+Tap/P4yl/MTSde3VC8ihkbE+Da0c0PgNGBIRKzX2tc3scyQtDi95wslTZJ0WIM696d62zQo/3Mq37MSbWmviOgZEa81V0fSxqnN1SWv65DjKhkFPBgRcyq1wEq2N723x1ViWRGxBLgKGFOJ5XVVFQ/Okk4Ffg38nCyQbghcCny9AovfCHg5ImoqsKyOsgDYRVK/krIRwMuVWoEy7XnvNgQWRsT8Nqy7upnZ20RET2Bz4Brgt5LOaVDnZeDokuX1A3Yh22/t1kL7urLvAtd1diNWoj8AIyT16OyGdJqIqNgErAV8AHyzmTo9yIL37DT9GuiR5u0JzCTL6uYDc4Bj07yfAkuBZWkdI4GfANeXLHtjIIDq9PwY4DXgfeB14MiS8odKXrcr8DiwKP2/a8m8+4HzgH+k5dwLrN3EttW3/3fA6FRWBcwCzgbuL6n7G2AG8B7wT2D3VL5fg+18pqQd56d2fAR8NpUdl+ZfDtxcsvxfAJMANWjjl9Pr69Lyr0nlXwemAO+m5X6u5DVvAGcAzwJL6vdvg+UG8NkGZYcAH5N906nfhrPTPqpKZSemts8E9mxiv16T9ul96T14ANiowbpHA68Ar6eyLVL9t4GXgENL6vcDbkv7fnJ6fx9qbFuA1YCLgenp+Hgolb2Z6n2Qpl3ouONqw/Se1R/Xg9P71C09/z0wv6T+dcD3Sj6TV5J9lmYBPyvZ9w3bu0/aV4uAy9J+Pq60LnAR8A7Z52lomnc+UJve6w+A37b3PUh1XgH2qGSM6kpTpYPzfkANjXx4S+qcCzwKrAusAzwMnJfm7Zlefy7QHdgf+BDok+b/hOWDccPnG6cPTDWwRnrjN0/zBgBbNjwogb7pYPtWet3h6XlpQHkV2IzsQ3k/cEET27YnWZDZFXgsle0P3AMcx/LB+ah0gFaT/TGaC6za2HaVtONNYMv0mu4sH5xXJ8tKjwF2B94CBjXXzpLnmwGLga+k5Z4OTANWSfPfAJ4GNgBWa2KZjQXn7un9HFqyDceRBaL6sslkga2l4Pw+8EWyP+6/YcVgel96L1dL7/0M4Ni0r7ZL+2NIqj8BmJjqbUUWtJoKzpemdg8k+0O7a2rDxpQkAh18XB0ATGlQ9iawfXr8ElkS8rmSedulx38Grkjbum7a3//RSHvXJvu8HJzaewpZglAanJcB30n74Xiy5Eql721J+9r1HqQ6twEnr6xgmLep0t0a/YC3ovluhyOBcyNifkQsIMuIv1Uyf1mavywi7iT7S7x5G9tTB2wlabWImBMRUxqpcwDwSkRcFxE1EXEj8CLwtZI6V0fEyxHxEdkBtW1zK42Ih4G+kjYn+wp/bSN1ro+IhWmdF5N94FvazmsiYkp6zbIGy/uQbD+OBa4HToqImS0sr95hwB0RcV9a7kVkAWPXkjqXRMSMtA/Kkpb1FlmgKnUtcLSkLYDeEfFIGYu7IyIejKw/8kdkXUcblMz/r4h4O7Xvq8AbEXF12ldPATcD35RUBXwDODsiFkfE80Cj/fap6+jbwCkRMSsiaiPi4dSGllTyuOpN9sep1APAHpLqzxnclJ4PBnoBz0jqT5YcfC9t63zgV8DwRtaxP9kfgFvS5/cSsoSh1PSI+H1E1JLtswFkXZeNqcR78H7a9k+lSgfnhcDaLfT7rU/2FbHe9FT2yTIaBPcPgZ6tbUhELCYLOt8F5ki6IwWDltpT36aBJc9LD9Jy23Md2Vf2vciyl+VI+oGkF9LIk3fJvn6u3cIyZzQ3MyIeI8ugRPZhL9dy+yAi6tK6SvdBs+tujKTuZN+O3m4w6xbgS2T7p9x+1E/WHxEfpGWu39h8snMTO0l6t34iSwrWS+2pblC/4ftfb21gVbIMt7UqeVy9A6zZoOwBsm9AXwQeJMtc90jT39N7uBHZt5c5JfvhCrIMurH2lu7jIPs2U2puyfwP08Om2lyJ92BNsu6bT6VKB+dHyPokhzVTZzbZG1dvw1TWFovJvs7XW27kQUTcExFfIfsL/yJZ31xL7alv06w2tqnedcAJwJ0lBzIAknYn6zo4lKzLpjdZP5/qm97EMpu9haCk0WQZ+Oy0/HIttw8kiawLo3QftOX2hQeSdWtMLi1M++Musq/G5QbnT7JkST3JsvHS46a0fTOAByKid8nUMyKOJzvxWFO6PLL3uzFvkfWjbtLIvJb2RyWPq2eBwQ2SngfIuq/2TI8fAr5AFpwfSHVmkH0e1y7ZD70iYstG1jEHGFT/JB0Dgxqp15SG+6MS78HngGda0YZCqWhwjohFZCd8LpU0TNLqkrpLGirpl6najcCPJa0jae1Uv8VhY014GviipA0lrQWcWT9DUn9JB0pag+wA/YCsm6OhO4HN0vC/6jT8awhwexvbBEBEvE72QflRI7PXJDs4FwDVks4m+ypabx6wcWtGZEjajOxkz1Fk3RunS2q2+6XEROAASXunbPc0sn32cLnrb9CWvpKOJOuv/UVELGyk2llkJ3veKHOx+0vaTdIqZCePHo2IprL528ne02+l46+7pM9L+lz6Sn4L8JN0fA4hG02zgpR9XgWMlbS+pCpJu6QRBAvIjqemxkNX7LhK3VPTgB1Lyl4hO0l4FFkQfI/suPkGKThHNuzuXuBiSb0kdZO0iaQ9GlnNHcDW6XNbTXaCtTXDLOex/L5o13sgaSDZH+BHW9GGQqn4ULrUf3oq8GOyA3gG2dfXv6QqPwOeIMsGngOeTGVtWdd9wB/Tsv7J8gd+t9SO2WRfgfcgy9QaLmMhWf/YaWTdMqcDX42It9rSpgbLfigiGvtWcA9wN9kJvOlk2VlpoKm/wGahpCdbWk/6MF1PFgifSR/cs4DryhmKFBEvkX3I/5ssW/wa8LWIWNrSaxt4RtIHZIHkOOD7EXF2E+ucHRGtuajhD8A5ZO/l9qm9jYqI98lGHgwne//nko1eqd8XJ5J9HZ9LdrLx6mbW+wOy4/TxtO5fkI2S+JA0eiZ9bd+5QRsqfVxdwfLnZiALwgtL/kg9QPbtq/SYORpYBZhK1j1yE9k3yeWkdn0T+GVq7xCyz2k5/euQnaQ9RNI7ki6pwHtwBDC+zP79Qqo/02qWW5KuIRtd8uPObktnSX9knwL2jgpeiNLM+rqR9TkfGRF/6+j1NVh3D7LujC9GG8biF0VRB+ybFUrKIId05Dok7Qs8RtZd8kOyLHyldyukbW3s5P2nSpe6t4aZdahdyEam1HdtDWvN0EmrLHdrmJnlkDNnM7Mc6vA+52VvvebU3FZw+g5ndXYTLId+9cYEtVyrea2JOd3X/ky719dRnDmbmeWQR2uYWbHU1XZ2CyrCwdnMiqU2z7d7L5+Ds5kVSnbVfdfn4GxmxVLn4Gxmlj/OnM3McsgnBM3McsiZs5lZ/oRHa5iZ5ZBPCJqZ5ZC7NczMcsgnBM3McsiZs5lZDvmEoJlZDvmEoJlZ/kS4z9nMLH/c52xmlkPu1jAzyyFnzmZmOVS7rLNbUBEOzmZWLO7WMDPLIXdrmJnlUEEy526d3QAzs4qqqyt/aoGkqyTNl/R8SVlfSfdJeiX936dk3pmSpkl6SdK+JeXbS3ouzbtEklpat4OzmRVK1C4reyrDNcB+DcrGAJMiYlNgUnqOpCHAcGDL9JrLJFWl11wOfAfYNE0Nl7kCB2czK5aoK39qaVERDwJvNyg+EBifHo8HhpWUT4iIJRHxOjAN2FHSAKBXRDwaEQFcW/KaJrnP2cyKpeP7nPtHxJz0eC7QPz0eCDxaUm9mKluWHjcsb5YzZzMrllZkzpJGSXqiZBrVqlVlmXB0xGY4czazYmlF5hwR44BxrVzDPEkDImJO6rKYn8pnARuU1BuUymalxw3Lm+XM2cyKpYJ9zk24DRiRHo8Abi0pHy6ph6TBZCf+JqcukPck7ZxGaRxd8pomOXM2s2KpqdzN9iXdCOwJrC1pJnAOcAEwUdJIYDpwKEBETJE0EZgK1ACj41/3Lz2BbOTHasBdaWqWg7OZFUsFrxCMiMObmLV3E/XPB85vpPwJYKvWrNvB2cyKpSBXCDo4m1mx+N4aZmY55MzZzCyHnDmbmeVQBUdrdCYHZzMrluiQC/ZWOgdnMysW9zmbmeWQg7OZWQ75hKCZWQ7V1rZcpwtwcDazYnG3hplZDjk4m5nlkPuczczyJ+o8ztnMLH/crWFmlkMerWFmlkPOnM3McsjBufh+/POxPPiPyfTt05u/XP+7Fea/Nn0G/3n+WKa+PI2TR43g2CMOafc6ly5dypnnXczUl16h91q9uOjcMxk4oD+z587jlDPPo64uqKmp4YhDvs5hBx3Q7vVZ61T36M6JfzyH6h7dqarqxjN3Pcbdv7ppuTpbfWV7hp56KBFBXU0tfz73Wl5/4qV2rbdqlWqOHDuaQVsN5sN3P2D8ib/hnZkLWH/IRnzzZyNZtedq1NXWcd+lf+Hp2x9p17q6vILc+Mi/vt2MYft/hd+N/VmT89fqtSZjvv9djjn8G61e9qw58zjmxNNXKL/l9nvptWZP7pp4Fd86bBhjL7sKgHX69eWGK8Zy8/hLufH3v+bK6ycyf8HCVq/X2qdmyTIuO+I8Lhp6BhfuP4Yt9tiWjbb77HJ1Xv7H81w49Awu2n8MN55+BYf9YlTZy+8zaB1GTzh7hfKdD92LjxZ9wM/3/B4PXHkHXxtzBADLPlrKDadexi/2+SFXjLiAg84+mlV7rd6+jezq6urKn3KsxcxZ0hbAgcDAVDQLuC0iXujIhuXBDttuzaw585qc369Pb/r16c2DDz++wrz/veev3PCnW1m2rIZ/23JzfnzaaKqqqlpc51///ggnjDwKgH323J2fj72ciKB79+6f1Fm6bBl1BckOuqKlHy4BoKq6iqrqqhUStfr5AKus3gNK5m8/bDd2P2Y/qlepZvrT07jpx1eWNfRrq3124O5fZxn6M3c+xsE/PRaABa/P+aTOe/Pf4f2F79Gzby8+fu/Dtm5e11eQoXTNZs6SzgAmAAImp0nAjZLGdHzzuqZX33iTuyc9wHW/u5ibx19Kt27duP3ev5X12vkLFrLeumsDUF1dRc81VufdRe8BMGfeAg46+ni+fNDRjDzym6y7Tr8O2wZrmrqJH9x5Aef9cxwvPfQcbz49bYU6W+/7ecZMupjvXHUGN56edYmtu8n6bPfVXbjkkHO4aP8x1NXWsf2w3cpa51r9+/Lu7OybUl1tHR+//xFr9FlzuTobbrMJ1d2rWTi96YTiU6G2tvwpx1rKnEcCW0bEstJCSWOBKcAFjb1I0ihgFMBlF/+M445u6tfFi+mxJ55m6ovTGD7yFACWLFlC3z69ATj5zHOZNXsey2qWMWfeAr4xYjQARx16IAcdsE+zyx3Qfx3+fO3lzF+wkJPPPJev7LUba/ft07EbYyuIuuCi/cewaq/V+fYVp7HeZoOY+/LM5eo8d8/jPHfP43xmxy3Y/9RDufyo89nsC1szaOvBnHrb+QB077EKHyzM/vAee8Wp9NtgXaq6V9Nn/bX5wZ3ZR+vBq+9i8p8eaLFNvdbpzZFjR/OHH1xGfMq/VUXOuyvK1VJwrgPWB6Y3KB+Q5jUqIsYB4wCWvfXap+5IiQi+PvTLfP/4Y1eYd8l/Zf2Js+bM40fnX8w1v/3lcvPXXacfc+e/xXrrrkNNTS0fLP6Q3mv1WqHOZz+zEU8+8zz77LV7x22INevj9z5k2iNT2GKPbVcIzvVem/wi/TZclzX6rIkEj9/8IHf8csIK9a7+j7FA1ud8xEXHc+nwc5ebv2je2/Revx+L5r5Nt6purLrmaix+530AevRcje9cfQZ3XvRHpj+1Yhb/qfNp6NYAvgdMknSXpHFpuhuYBJzS8c3rmnbeYVvuu/8hFr7zLgCL3nuf2XPL+6q51247c+ud/wfAvff/nZ223wZJzJ2/gI+XLPlkeU89O5WNNxzUMRtgTVqj75qfnHDr3qM7m+/2b8x/dfZyddbeqP8njwdtuTFVq3Rn8Tvv8/I/nmeboTvRs1/2x3b1tdagz8C1y1rv8/f9kx2/8UUAttl/J6Y9PAWAqu5VfPuK03j8lgd55q7H2r19hRB15U851mzmHBF3S9oM2JHlTwg+HhH57rCpgB+ecwGPP/Us7777HnsPO4oTRn6LmvTjkYcddABvLXybw0aezAeLP6Rbt25cP/Ev3HrDFWwyeCNO+s7RjPrej6iLOrpXV/OjU09g/fX6t7BGOPir+3LmeRcy9NBvs1avNbnwp1nX/mtvzODC3/4eSUQExxx+MJttMrhDt99W1GvdPhxx8fF069YNdevG03c8wtS/PsmuR34ZgIdv+D/+behOfP7g3amtqWXZx0u59sTfADBv2izuvHgi373uLCRRW1PLzWdfxTuz3mpxvY9N/BtHjh3NWff/mg/f/YDrTroEgG0P2IVNdtyCNfr0ZMdD9gDgDz+4nNlTG37Z/RQpSOasju6f+jR2a1jLTt/hrM5uguXQr96YoPYuY/HZw8uOOWuc2/71dRRfhGJmxZLz7opyOTibWbEUpFvDwdnMCqUoQ+l8+baZFUtdlD+1QNL3JU2R9LykGyWtKqmvpPskvZL+71NS/0xJ0yS9JGnf9myGg7OZFUuFgrOkgcDJwA4RsRVQBQwHxgCTImJTsmHFY1L9IWn+lsB+wGWSWr5nQxMcnM2sWCp7+XY1sJqkamB1YDbZvYbGp/njgWHp8YHAhIhYEhGvA9PIhiG3iYOzmRVK1EXZk6RRkp4omT65hWBEzAIuAt4E5gCLIuJeoH9E1N9xai5QfwHDQGBGSVNm8q/rQ1rNJwTNrFhaMVqj9FYTDaW+5AOBwcC7wJ8kHdXg9SGpQ4aHODibWbFUbrTGl4HXI2IBgKRbgF2BeZIGRMQcSQOA+an+LGCDktcPSmVt4m4NMyuWyo3WeBPYWdLqkgTsDbwA3AaMSHVGALemx7cBwyX1kDQY2JTsNstt4szZzIqlQhehRMRjkm4CngRqgKfIukB6AhMljSS7Y+ehqf4USROBqan+6Pbcg8jB2cwKJWordxFKRJwDnNOgeAlZFt1Y/fOB8yuxbgdnMysWX75tZpY/5fwmY1fg4GxmxeLgbGaWQ8W475GDs5kVS9QUIzo7OJtZsRQjNjs4m1mx+ISgmVkeOXM2M8sfZ85mZnnkzNnMLH+iprNbUBkOzmZWKOHM2cwshxyczczyx5mzmVkOOTibmeVQ1Kqzm1ARDs5mVijOnM3McijqnDmbmeWOM2czsxyKcOZsZpY7zpzNzHKozqM1zMzyxycEzcxyyMHZzCyHohi3c3ZwNrNiceZsZpZDHkpnZpZDtR6tYWaWP0XJnLt1dgPMzCop6lT21BJJvSXdJOlFSS9I2kVSX0n3SXol/d+npP6ZkqZJeknSvu3ZDgdnMyuUiPKnMvwGuDsitgC2AV4AxgCTImJTYFJ6jqQhwHBgS2A/4DJJVW3dDgdnMyuUSmXOktYCvghcCRARSyPiXeBAYHyqNh4Ylh4fCEyIiCUR8TowDdixrdvh4GxmhVJb163sSdIoSU+UTKNKFjUYWABcLekpSf8jaQ2gf0TMSXXmAv3T44HAjJLXz0xlbeITgmZWKK25CCUixgHjmphdDfw7cFJEPCbpN6QujJLXh6QOuezFmbOZFUpdqOypBTOBmRHxWHp+E1mwnidpAED6f36aPwvYoOT1g1JZmzg4m1mhRKjsqfnlxFxghqTNU9HewFTgNmBEKhsB3Joe3wYMl9RD0mBgU2ByW7fD3RpmVigVvrfGScANklYBXgOOJUtqJ0oaCUwHDs3WG1MkTSQL4DXA6IiobeuKFR18l5DqVQYW5DYkZtbRapbOavcVJE8MGlZ2zNlh5l9ye8WKM2czK5TaumL01jo4m1mhFOWruoOzmRVKGaMwugQHZzMrlKLc+MjB2cwKpSA/vu3gbGbFEjhzNjPLnRp3a5iZ5Y8zZzOzHHKfs5lZDjlzNjPLIWfOZmY5VOvM2cwsf8r43dYuwcHZzAqlzpmzmVn++MZHZmY55BOCZmY5VCd3a5iZ5U6bfxcqZxyczaxQPFrDzCyHPFrDzCyHPFrDzCyH3K1hZpZDHkpnZpZDtc6czczyx5mzmVkOOTibmeVQQX5C0MHZzIrFmbOZWQ4V5fLtbp3dADOzSqpT+VM5JFVJekrS7el5X0n3SXol/d+npO6ZkqZJeknSvu3ZDgdnMyuUulZMZToFeKHk+RhgUkRsCkxKz5E0BBgObAnsB1wmqaqt2+HgbGaFUsngLGkQcADwPyXFBwLj0+PxwLCS8gkRsSQiXgemATu2dTscnM2sUKIVk6RRkp4omUY1WNyvgdNZPpb3j4g56fFcoH96PBCYUVJvZiprE58QNLNCac29NSJiHDCusXmSvgrMj4h/StqzideHpA6515KDs5kVSgVHa3wB+Lqk/YFVgV6SrgfmSRoQEXMkDQDmp/qzgA1KXj8olbWJuzXMrFDqiLKn5kTEmRExKCI2JjvR99eIOAq4DRiRqo0Abk2PbwOGS+ohaTCwKTC5rdvhzNnMCmUlXIRyATBR0khgOnAoQERMkTQRmArUAKMjos2JvIOzmRVKR3QAR8T9wP3p8UJg7ybqnQ+cX4l1OjibWaH48m0zsxyq6ZjBEyudg7OZFUoxQrODs5kVjLs1zMxyqKUhcl2Fg7OZFUoxQrODs5kVjLs1zMxyqLYgubODs5kVijNnM7McCmfOZmb5U5TM2XelW0nWWqsXf5wwjuefe4Dnnr2fnXfavrObZDlw0okjefqpSTzz9F85+aTjOrs5hVCpu9J1NmfOK8mvxp7LPff8jcOGj6J79+6svvpqnd0k62Rbbrk5I0cewS67HsDSpcu48/YbuOPO/+PVV9/o7KZ1afkOueVz5rwS9Oq1JrvvthNXXX0jAMuWLWPRovc6uVXW2bbYYlMmT36Kjz76mNraWh78+6McNGxoZzery6shyp7yzMF5JRg8eEPeemshV/7Pr3h88j1c8bsLnTkbU6a8yG677UTfvn1YbbVVGbrflxg0aP3OblaXF634l2dtDs6Sjm1m3ic/mlhXt7itqyiM6qoqtttua6644lo+v+O+LF78IWecfmJnN8s62YsvTuPCCy/lrjv/wJ2338DTz0yhtrYop7M6TyV/fbsztSdz/mlTMyJiXETsEBE7dOu2RjtWUQwzZ81h5sw5TH78KQBuueUOttt2605uleXB1ddMYKedh7LX3t/g3XcX8corr3V2k7q8omTOzZ4QlPRsU7P418+BWwvmzVvAzJmz2WyzTXj55Vf50pd244UXXu7sZlkOrLNOPxYsWMgGG6zPsGFD+cJuX+vsJnV5ec+Iy9XSaI3+wL7AOw3KBTzcIS0qqFO+/59cO/6/WWWV7rz++puMPO7Uzm6S5cCf/vh7+vbrw7JlNZx88o98orgCaiPfGXG5WgrOtwM9I+LphjMk3d8hLSqoZ56Zws677N/ZzbCc2fNLB3d2Ewon7+OXy9VscI6Ikc3MO6LyzTEza5+89yWXyxehmFmhfFr6nM3MupRPRbeGmVlX424NM7Mc+rSM1jAz61LcrWFmlkM+IWhmlkPuczYzyyF3a5iZ5VAU5ISg7+dsZoVSS5Q9NUfSBpL+JmmqpCmSTknlfSXdJ+mV9H+fktecKWmapJck7due7XBwNrNCqeBvCNYAp0XEEGBnYLSkIcAYYFJEbApMSs9J84YDWwL7AZdJqmrrdjg4m1mhRETZUwvLmRMRT6bH7wMvAAOBA4Hxqdp4YFh6fCAwISKWRMTrwDRgx7Zuh4OzmRVKazLn0l9tStOoxpYpaWNgO+AxoH9EzEmz5vKve9sPBGaUvGxmKmsTnxA0s0JpzVC6iBgHjGuujqSewM3A9yLiPUmlrw9JHXIG0sHZzAqlkpdvS+pOFphviIhbUvE8SQMiYo6kAcD8VD4L2KDk5YNSWZu4W8PMCqVSJwSVpchXAi9ExNiSWbcBI9LjEcCtJeXDJfWQNBjYFJjc1u1w5mxmhVLBi1C+AHwLeE5S/a9BnQVcAEyUNBKYDhwKEBFTJE0EppKN9BgdEbVtXbk6esB29SoDizEi3Mw6XM3SWWq5VvN2Xn/PsmPOo7Pvb/f6OoozZzMrFF++bWaWQ77xkZlZDtVGMW4a6uBsZoVSlBsfOTibWaG4z9nMLIfc52xmlkN17tYwM8sfZ85mZjnk0RpmZjnkbg0zsxxyt4aZWQ45czYzyyFnzmZmOVTb9rt05oqDs5kVii/fNjPLIV++bWaWQ86czcxyyKM1zMxyyKM1zMxyyJdvm5nlkPuczcxyyH3OZmY55MzZzCyHPM7ZzCyHnDmbmeWQR2uYmeWQTwiameWQuzXMzHLIVwiameWQM2czsxwqSp+zivJXpiuQNCoixnV2OyxffFxYY7p1dgM+ZUZ1dgMsl3xc2AocnM3McsjB2cwshxycVy73K1pjfFzYCnxC0Mwsh5w5m5nlkIOzmVkOOTivJJL2k/SSpGmSxnR2e6zzSbpK0nxJz3d2Wyx/HJxXAklVwKXAUGAIcLikIZ3bKsuBa4D9OrsRlk8OzivHjsAUTNPjAAAC0klEQVS0iHgtIpYCE4ADO7lN1ski4kHg7c5uh+WTg/PKMRCYUfJ8ZiozM2uUg7OZWQ45OK8cs4ANSp4PSmVmZo1ycF45Hgc2lTRY0irAcOC2Tm6TmeWYg/NKEBE1wInAPcALwMSImNK5rbLOJulG4BFgc0kzJY3s7DZZfvjybTOzHHLmbGaWQw7OZmY55OBsZpZDDs5mZjnk4GxmlkMOztYoSbWSnpb0vKQ/SVq9HcvaU9Lt6fHXm7srn6Tekk4oeb6+pJvaum6zrsrB2ZryUURsGxFbAUuB75bOVKbVx09E3BYRFzRTpTdwQkn92RFxSGvXY9bVOThbOf4OfFbSxume1NcCzwMbSNpH0iOSnkwZdk/45P7VL0p6Eji4fkGSjpH02/S4v6Q/S3omTbsCFwCbpKz9wrTO51P9VSVdLek5SU9J2qtkmbdIulvSK5J+uXJ3j1nlOThbsyRVk92H+rlUtClwWURsCSwGfgx8OSL+HXgCOFXSqsDvga8B2wPrNbH4S4AHImIb4N+BKcAY4NWUtf+wQf3RQETE1sDhwPi0LoBtgcOArYHDJG2AWRfm4GxNWU3S02QB903gylQ+PSIeTY93JvvxgH+kuiOAjYAtgNcj4pXILkG9vol1fAm4HCAiaiNiUQtt2q1+WRHxIjAd2CzNmxQRiyLiY2BqaodZl1Xd2Q2w3PooIrYtLZAEWbb8SRFwX0Qc3qDecq9bSZaUPK7Fx7Z1cc6crT0eBb4g6bMAktaQtBnwIrCxpE1SvcObeP0k4Pj02ipJawHvA2s2Uf/vwJGp/mbAhsBLldgQs7xxcLY2i4gFwDHAjZKeJbvD2hapa2EUcEc6ITi/iUWcAuwl6Tngn8CQiFhI1k3yvKQLG9S/DOiW6v8ROCYilmBWQL4rnZlZDjlzNjPLIQdnM7MccnA2M8shB2czsxxycDYzyyEHZzOzHHJwNjPLof8HjaxmMaI50WUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f2dcdc1f7f0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn import metrics\n",
    "\n",
    "cfmap=metrics.confusion_matrix(y_true=dvalid['target'],  # True labels\n",
    "                         y_pred=dvalid[\"predround\"])\n",
    "\n",
    "label = [\"0\", \"1\"]\n",
    "sns.heatmap(cfmap, annot = True, xticklabels = label, yticklabels = label)\n",
    "plt.xlabel(\"Prediction\")\n",
    "plt.title(\"Confusion Matrix for DM prediction (weighted)\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['loss', 'acc', 'weighted_acc']"
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.metrics_names"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Kaggle submission that was used as input for this notebook\n",
    "https://www.kaggle.com/knowledgegrappler/a-simple-nn-solution-with-keras-0-48611-pl"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Summary\n",
    "This notebook shows methods for dealing with structured data in the context of a neural network.\n",
    "\n",
    "# Author\n",
    "\n",
    "Mark Ryan is a manager at IBM Canada.\n",
    "\n",
    "Copyright © IBM Corp. 2018. This notebook and its source code are released under the terms of the MIT License."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}