Chatbot Word Embedding.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     /Users/ankitkothari/nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     /Users/ankitkothari/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
      "[nltk_data]     /Users/ankitkothari/nltk_data...\n",
      "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
      "[nltk_data]       date!\n",
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     /Users/ankitkothari/nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import time\n",
    "import re\n",
    "import slackclient\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "#NLTK\n",
    "import nltk\n",
    "from nltk.tokenize import TreebankWordTokenizer\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import wordnet\n",
    "nltk.download('stopwords')\n",
    "from nltk import pos_tag\n",
    "nltk.download('punkt')\n",
    "nltk.download('averaged_perceptron_tagger')\n",
    "nltk.download('wordnet')\n",
    "stopwords = stopwords.words('english')\n",
    "from nltk import word_tokenize, sent_tokenize\n",
    "\n",
    "#SKLEARN\n",
    "from sklearn.metrics import pairwise_distances # to perfrom cosine similarity\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer # to perform tfidf\n",
    "\n",
    "#TENSORFLOW\n",
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### CONNECT THE SLACKBOT BY THIS SECRET TOKEN "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [],
   "source": [
    "slack_clients = slackclient.SlackClient('xoxb-599099752130-1205175943715-yE5CFoCgrpstZcl0Pyh1oIm5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### DATA: QUESTIONS AND RESPONSES"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Context</th>\n",
       "      <th>Text Response</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>What is the best temperature to brew coffee?</td>\n",
       "      <td>According to chemical studies, the optimal water temperature for drip coffee is 95-98C. According to my notes, colder water doesn't extract enough caffeine/essential oils from the beans, and above such temperature the acidity increases wildly.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Quality of coffee</td>\n",
       "      <td>The quality of a brew depends on the following factors (in no particular order):\\nTime since grinding the beans.\\nTime since roasting.\\nCleanliness with brewing equipment.\\nBean quality (what crop, etc.).\\nWater quality.\\n</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>What is the difference between arabica and robusta?</td>\n",
       "      <td>Arabica beans and robusta beans are two different species of coffee. They are the primary species of coffee that find their way into the American cup. The general differences are those of taste, and the conditions under which the two species differ in production.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Just how much ground coffee do I need for x amount of coffee?</td>\n",
       "      <td>a. Whatever seems right to you. b. It may change slightly from coffee to coffee and according to freshness.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>What are the different between Preparation Methods\\n</td>\n",
       "      <td>Drip\\nFrench Press\\nEspresso\\nPercolator</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                         Context  \\\n",
       "0  What is the best temperature to brew coffee?                    \n",
       "1  Quality of coffee                                               \n",
       "2  What is the difference between arabica and robusta?             \n",
       "3  Just how much ground coffee do I need for x amount of coffee?   \n",
       "4  What are the different between Preparation Methods\\n            \n",
       "\n",
       "                                                                                                                                                                                                                                                             Text Response  \n",
       "0  According to chemical studies, the optimal water temperature for drip coffee is 95-98C. According to my notes, colder water doesn't extract enough caffeine/essential oils from the beans, and above such temperature the acidity increases wildly.                      \n",
       "1  The quality of a brew depends on the following factors (in no particular order):\\nTime since grinding the beans.\\nTime since roasting.\\nCleanliness with brewing equipment.\\nBean quality (what crop, etc.).\\nWater quality.\\n                                           \n",
       "2  Arabica beans and robusta beans are two different species of coffee. They are the primary species of coffee that find their way into the American cup. The general differences are those of taste, and the conditions under which the two species differ in production.  \n",
       "3  a. Whatever seems right to you. b. It may change slightly from coffee to coffee and according to freshness.                                                                                                                                                              \n",
       "4  Drip\\nFrench Press\\nEspresso\\nPercolator                                                                                                                                                                                                                                 "
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.set_option('display.max_colwidth', -1)\n",
    "data = pd.read_csv('./coffee.csv')\n",
    "data.ffill(axis=0, inplace= True)\n",
    "context = data['Context'].values\n",
    "context_string = data['Context'].str.cat(sep='\\n')\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Text Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    unk what best temperature brew coffee             \n",
       "1    unk quality coffee                                \n",
       "2    unk what difference arabica robusta               \n",
       "3    unk just much ground coffee i need x amount coffee\n",
       "4    unk what different preparation methods            \n",
       "Name: nltk_cleaning, dtype: object"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def nltk_cleaning(text):\n",
    "  token_text = word_tokenize(text)\n",
    "  clean_text = [\"unk\"]\n",
    "  lemma = wordnet.WordNetLemmatizer()\n",
    "  tag_list = pos_tag(token_text, tagset=None)\n",
    "  for token, pos_token in tag_list:\n",
    "   if token not in '\\n\\n \\n\\n\\n!\"-#$%&()--.*''+,-/:;``<=>[``?@[\\\\]^_`''{|}~\\t\\n`\\'\\'' and (token not in stopwords):\n",
    "     if pos_token.startswith('V'):  # Verb\n",
    "         pos_val='v'\n",
    "     elif pos_token.startswith('J'): # Adjective\n",
    "         pos_val='a'\n",
    "     elif pos_token.startswith('R'): # Adverb\n",
    "         pos_val='r'\n",
    "     else:\n",
    "         pos_val='n' # Noun\n",
    "     lemma_token= lemma.lemmatize(token,pos_val)\n",
    "     clean_text.append(lemma_token.lower())\n",
    "   else:\n",
    "      continue \n",
    "  return \" \".join(clean_text)\n",
    "data['nltk_cleaning']= data['Context'].apply(nltk_cleaning)\n",
    "data['nltk_cleaning'].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### EMBEDDING DIMENSIONS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading word vectors...\n",
      "Found 400000 word vectors.\n"
     ]
    }
   ],
   "source": [
    "####word embeddings \n",
    "EMBEDDING_DIM = 100\n",
    "\n",
    "# load in pre-trained word vectors\n",
    "print('Loading word vectors...')\n",
    "word2vec = {}\n",
    "with open(os.path.join('./glove.6B.%sd.txt' % EMBEDDING_DIM)) as f:\n",
    "  # is just a space-separated text file in the format:\n",
    "  # word vec[0] vec[1] vec[2] ...\n",
    "  for line in f:\n",
    "    values = line.split()\n",
    "    word = values[0]\n",
    "    vec = np.asarray(values[1:], dtype='float32')\n",
    "    word2vec[word] = vec\n",
    "print('Found %s word vectors.' % len(word2vec))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### TOKENIZATION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "length of sequences 27\n",
      "[[1, 3, 17, 18, 9, 2], [1, 19, 2], [1, 3, 10, 20, 21], [1, 22, 5, 23, 2, 24, 25, 26, 27, 2], [1, 3, 6, 28, 29], [1, 30, 5, 2], [1, 31, 2], [1, 3, 32, 2, 11], [1, 3, 6, 11, 33], [1, 4, 5, 7, 8, 2], [1, 4, 12, 34, 8, 2], [1, 3, 35, 36, 2], [1, 37, 2, 13], [1, 38, 14, 2, 39], [1, 4, 40, 7, 41, 2], [1, 42, 2, 43], [1, 4, 2, 44], [1, 3, 10, 45, 46, 2], [1, 47, 2, 48, 49], [1, 3, 2, 7], [1, 4, 50, 2, 51], [1, 4, 12, 8, 2, 52, 14, 53], [1, 3, 54], [1, 3, 55, 56, 9, 6, 57, 2], [1, 15, 2, 58, 16, 13], [1, 15, 2, 59, 16, 60], [1, 61, 62, 2, 63]]\n",
      "63\n",
      "[['unk', 'what', 'best', 'temperature', 'brew', 'coffee'], ['unk', 'quality', 'coffee'], ['unk', 'what', 'difference', 'arabica', 'robusta'], ['unk', 'just', 'much', 'ground', 'coffee', 'i', 'need', 'x', 'amount', 'coffee'], ['unk', 'what', 'different', 'preparation', 'methods'], ['unk', 'effects', 'much', 'coffee'], ['unk', 'varieties', 'coffee'], ['unk', 'what', 'mean', 'coffee', 'roast'], ['unk', 'what', 'different', 'roast', 'degree'], ['unk', 'how', 'much', 'caffeine', 'cup', 'coffee'], ['unk', 'how', 'many', 'calorie', 'cup', 'coffee'], ['unk', 'what', 'fair', 'trade', 'coffee'], ['unk', 'is', 'coffee', 'bad'], ['unk', 'can', 'drink', 'coffee', 'pregnant'], ['unk', 'how', 'long', 'caffeine', 'last', 'coffee'], ['unk', 'where', 'coffee', 'come'], ['unk', 'how', 'coffee', 'decaffeinate'], ['unk', 'what', 'difference', 'filter', 'instant', 'coffee'], ['unk', 'why', 'coffee', 'become', 'popular'], ['unk', 'what', 'coffee', 'caffeine'], ['unk', 'how', 'remove', 'coffee', 'stain'], ['unk', 'how', 'many', 'cup', 'coffee', 'safe', 'drink', 'day'], ['unk', 'what', 'espresso'], ['unk', 'what', '’', 'cold', 'brew', 'different', 'iced', 'coffee'], ['unk', 'does', 'coffee', 'ever', 'go', 'bad'], ['unk', 'does', 'coffee', 'make', 'go', 'poop'], ['unk', 'who', 'produce', 'coffee', 'world']]\n"
     ]
    }
   ],
   "source": [
    "sentences = data['nltk_cleaning'].fillna(\"DUMMY_VALUE\").values\n",
    "tokenizer= tf.keras.preprocessing.text.Tokenizer(filters='')\n",
    "tokenizer.fit_on_texts(sentences)\n",
    "sequences = tokenizer.texts_to_sequences(sentences)\n",
    "print(\"length of sequences\",len(sequences))\n",
    "max_len = [len(s) for s in sentences]\n",
    "#max_index = np.argmax(max_len)\n",
    "#print(max_len[max_index])\n",
    "print(sequences)\n",
    "print(len(tokenizer.index_word))\n",
    "token_words = [[tokenizer.index_word[i] for i in j] for j in sequences]\n",
    "print(token_words)\n",
    "data['question_tokens']=token_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     [unk, what, best, temperature, brew, coffee]                 \n",
       "1     [unk, quality, coffee]                                       \n",
       "2     [unk, what, difference, arabica, robusta]                    \n",
       "3     [unk, just, much, ground, coffee, i, need, x, amount, coffee]\n",
       "4     [unk, what, different, preparation, methods]                 \n",
       "5     [unk, effects, much, coffee]                                 \n",
       "6     [unk, varieties, coffee]                                     \n",
       "7     [unk, what, mean, coffee, roast]                             \n",
       "8     [unk, what, different, roast, degree]                        \n",
       "9     [unk, how, much, caffeine, cup, coffee]                      \n",
       "10    [unk, how, many, calorie, cup, coffee]                       \n",
       "11    [unk, what, fair, trade, coffee]                             \n",
       "12    [unk, is, coffee, bad]                                       \n",
       "13    [unk, can, drink, coffee, pregnant]                          \n",
       "14    [unk, how, long, caffeine, last, coffee]                     \n",
       "15    [unk, where, coffee, come]                                   \n",
       "16    [unk, how, coffee, decaffeinate]                             \n",
       "17    [unk, what, difference, filter, instant, coffee]             \n",
       "18    [unk, why, coffee, become, popular]                          \n",
       "19    [unk, what, coffee, caffeine]                                \n",
       "20    [unk, how, remove, coffee, stain]                            \n",
       "21    [unk, how, many, cup, coffee, safe, drink, day]              \n",
       "22    [unk, what, espresso]                                        \n",
       "23    [unk, what, ’, cold, brew, different, iced, coffee]          \n",
       "24    [unk, does, coffee, ever, go, bad]                           \n",
       "25    [unk, does, coffee, make, go, poop]                          \n",
       "26    [unk, who, produce, coffee, world]                           \n",
       "Name: question_tokens, dtype: object"
      ]
     },
     "execution_count": 150,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data['question_tokens']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### DEFINING SOME SENTENCE EMBEDDING FUNCTIONS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "#COMPUTING VECTORS FOR THE QUESTIONS\n",
    "def question_embedding(ss1):\n",
    "  v1=np.mean([word2vec[word] if word in word2vec else word2vec['unk'] for word in ss1], axis=0)\n",
    "  #print(v1)\n",
    "  return v1\n",
    "\n",
    "def cosine_similarity_sentence(ss1,ss2):\n",
    "  v1=np.mean([word2vec[word] for word in ss1], axis=0)\n",
    "  v2=np.mean([word2vec[word] for word in ss2], axis=0)\n",
    "  cosine_sent = 1- pairwise_distances([v1],[v2], metric = 'cosine' )\n",
    "  return cosine_sent[0]*100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(27, 100)"
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "que_list = [question_embedding(x) for x in data['question_tokens']]\n",
    "que_array = np.array(que_list)\n",
    "que_array.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "#data['question_embedding']= data['question_tokens'].apply(lambda x: question_embedding(x))\n",
    "#features = data['question_embedding'].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [],
   "source": [
    "#features.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.027166 , -0.1762   , -0.19623  ,  0.33527  ,  0.062392 ,\n",
       "       -0.29748  ,  0.46416  , -0.13271  ,  0.38312  , -0.37872  ,\n",
       "        0.53709  , -0.071442 , -0.15837  ,  0.049163 , -0.2225   ,\n",
       "        0.33305  ,  0.11272  , -0.10672  ,  0.48584  , -0.19743  ,\n",
       "       -0.8179   , -0.43915  , -0.68904  , -0.23316  ,  0.082357 ,\n",
       "        0.49117  , -0.08154  ,  0.10273  ,  0.39784  ,  0.68379  ,\n",
       "       -0.072532 , -0.26836  ,  0.52545  , -0.14456  , -0.3563   ,\n",
       "       -0.19699  ,  0.28968  , -0.71389  ,  0.10056  ,  0.41272  ,\n",
       "        0.52239  , -0.26027  ,  0.016722 ,  0.54869  ,  0.076753 ,\n",
       "        0.010193 , -0.11065  ,  0.47543  , -0.20591  ,  0.11673  ,\n",
       "       -0.096665 ,  0.13824  , -0.16024  , -0.43239  ,  0.23957  ,\n",
       "        0.21804  ,  0.48378  ,  0.27944  , -0.74636  , -0.60598  ,\n",
       "       -0.37709  ,  0.28384  , -0.1613   , -0.0017167,  0.35972  ,\n",
       "        0.091564 , -0.070139 , -0.9614   , -0.25841  ,  0.33928  ,\n",
       "        0.44946  , -0.1805   ,  0.051543 ,  0.26107  ,  0.16685  ,\n",
       "        0.72182  , -0.25338  , -0.685    ,  0.31886  , -0.28397  ,\n",
       "        0.1987   , -0.55063  ,  0.10243  , -0.81482  ,  0.48636  ,\n",
       "        0.19379  , -0.12213  , -0.09981  ,  0.56565  ,  0.35586  ,\n",
       "       -0.035299 ,  0.14357  ,  0.25993  , -0.33945  ,  0.47758  ,\n",
       "        0.010572 , -0.15973  , -0.37226  , -0.28782  , -0.015834 ],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#assert('UNK' in word2idx)\n",
    "word2vec['unk']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "27\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(27, 100)"
      ]
     },
     "execution_count": 156,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X= np.array([str(i)+\"_vector\" for i in range(100)])\n",
    "X = X.reshape(1,100)\n",
    "print(len(que_array))\n",
    "match_frame = pd.DataFrame(que_array)\n",
    "match_frame.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>90</th>\n",
       "      <th>91</th>\n",
       "      <th>92</th>\n",
       "      <th>93</th>\n",
       "      <th>94</th>\n",
       "      <th>95</th>\n",
       "      <th>96</th>\n",
       "      <th>97</th>\n",
       "      <th>98</th>\n",
       "      <th>99</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-0.127751</td>\n",
       "      <td>0.244368</td>\n",
       "      <td>0.254793</td>\n",
       "      <td>-0.031823</td>\n",
       "      <td>-0.279911</td>\n",
       "      <td>0.025219</td>\n",
       "      <td>0.405200</td>\n",
       "      <td>-0.042898</td>\n",
       "      <td>-0.279454</td>\n",
       "      <td>-0.173665</td>\n",
       "      <td>...</td>\n",
       "      <td>0.177067</td>\n",
       "      <td>0.281130</td>\n",
       "      <td>-0.069608</td>\n",
       "      <td>-0.125257</td>\n",
       "      <td>-0.388032</td>\n",
       "      <td>-0.016751</td>\n",
       "      <td>-0.185640</td>\n",
       "      <td>-0.259940</td>\n",
       "      <td>0.371643</td>\n",
       "      <td>0.480574</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.087919</td>\n",
       "      <td>0.271073</td>\n",
       "      <td>-0.045070</td>\n",
       "      <td>0.285327</td>\n",
       "      <td>-0.050055</td>\n",
       "      <td>-0.312195</td>\n",
       "      <td>0.354410</td>\n",
       "      <td>0.028920</td>\n",
       "      <td>-0.088700</td>\n",
       "      <td>-0.094040</td>\n",
       "      <td>...</td>\n",
       "      <td>0.181078</td>\n",
       "      <td>0.041523</td>\n",
       "      <td>-0.134333</td>\n",
       "      <td>-0.295643</td>\n",
       "      <td>-0.374707</td>\n",
       "      <td>0.016127</td>\n",
       "      <td>-0.242853</td>\n",
       "      <td>-0.329727</td>\n",
       "      <td>0.429050</td>\n",
       "      <td>0.444885</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.416187</td>\n",
       "      <td>0.395458</td>\n",
       "      <td>0.178634</td>\n",
       "      <td>0.279764</td>\n",
       "      <td>-0.326506</td>\n",
       "      <td>-0.505535</td>\n",
       "      <td>0.306110</td>\n",
       "      <td>-0.179845</td>\n",
       "      <td>-0.107523</td>\n",
       "      <td>-0.379778</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.218687</td>\n",
       "      <td>0.163467</td>\n",
       "      <td>-0.146594</td>\n",
       "      <td>-0.079616</td>\n",
       "      <td>-0.274517</td>\n",
       "      <td>0.308600</td>\n",
       "      <td>-0.159803</td>\n",
       "      <td>-0.391722</td>\n",
       "      <td>0.004196</td>\n",
       "      <td>0.505209</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.181798</td>\n",
       "      <td>0.641966</td>\n",
       "      <td>0.159485</td>\n",
       "      <td>0.123393</td>\n",
       "      <td>-0.360254</td>\n",
       "      <td>0.062509</td>\n",
       "      <td>0.124223</td>\n",
       "      <td>0.196119</td>\n",
       "      <td>0.004125</td>\n",
       "      <td>-0.136333</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.028014</td>\n",
       "      <td>0.080020</td>\n",
       "      <td>-0.056650</td>\n",
       "      <td>-0.181108</td>\n",
       "      <td>-0.520920</td>\n",
       "      <td>-0.176884</td>\n",
       "      <td>-0.040635</td>\n",
       "      <td>-0.186591</td>\n",
       "      <td>0.451035</td>\n",
       "      <td>0.383742</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.208121</td>\n",
       "      <td>0.256008</td>\n",
       "      <td>-0.085700</td>\n",
       "      <td>0.166956</td>\n",
       "      <td>-0.303599</td>\n",
       "      <td>-0.124764</td>\n",
       "      <td>0.072421</td>\n",
       "      <td>0.093126</td>\n",
       "      <td>-0.163662</td>\n",
       "      <td>0.233790</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.155851</td>\n",
       "      <td>0.018432</td>\n",
       "      <td>-0.017286</td>\n",
       "      <td>0.119013</td>\n",
       "      <td>-0.014388</td>\n",
       "      <td>-0.178452</td>\n",
       "      <td>-0.570388</td>\n",
       "      <td>-0.401391</td>\n",
       "      <td>0.330326</td>\n",
       "      <td>0.503553</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 100 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          0         1         2         3         4         5         6  \\\n",
       "0 -0.127751  0.244368  0.254793 -0.031823 -0.279911  0.025219  0.405200   \n",
       "1  0.087919  0.271073 -0.045070  0.285327 -0.050055 -0.312195  0.354410   \n",
       "2  0.416187  0.395458  0.178634  0.279764 -0.326506 -0.505535  0.306110   \n",
       "3 -0.181798  0.641966  0.159485  0.123393 -0.360254  0.062509  0.124223   \n",
       "4 -0.208121  0.256008 -0.085700  0.166956 -0.303599 -0.124764  0.072421   \n",
       "\n",
       "          7         8         9  ...        90        91        92        93  \\\n",
       "0 -0.042898 -0.279454 -0.173665  ...  0.177067  0.281130 -0.069608 -0.125257   \n",
       "1  0.028920 -0.088700 -0.094040  ...  0.181078  0.041523 -0.134333 -0.295643   \n",
       "2 -0.179845 -0.107523 -0.379778  ... -0.218687  0.163467 -0.146594 -0.079616   \n",
       "3  0.196119  0.004125 -0.136333  ... -0.028014  0.080020 -0.056650 -0.181108   \n",
       "4  0.093126 -0.163662  0.233790  ... -0.155851  0.018432 -0.017286  0.119013   \n",
       "\n",
       "         94        95        96        97        98        99  \n",
       "0 -0.388032 -0.016751 -0.185640 -0.259940  0.371643  0.480574  \n",
       "1 -0.374707  0.016127 -0.242853 -0.329727  0.429050  0.444885  \n",
       "2 -0.274517  0.308600 -0.159803 -0.391722  0.004196  0.505209  \n",
       "3 -0.520920 -0.176884 -0.040635 -0.186591  0.451035  0.383742  \n",
       "4 -0.014388 -0.178452 -0.570388 -0.401391  0.330326  0.503553  \n",
       "\n",
       "[5 rows x 100 columns]"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "match_frame.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Text Vectorization "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "pairwise distance provide distance between two array.so more pairwise distance means less similarity.while cosine similarity is 1-pairwise_distance so more cosine similarity means more similarity between two arrays."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [],
   "source": [
    "sent = 'how many cups can i have in a day'\n",
    "Question = nltk_cleaning(sent).split(' ')\n",
    "q_array = np.array(Question)\n",
    "Question_tf = question_embedding(q_array)\n",
    "Question_tf=Question_tf.reshape(-1,100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'On average, the regular person is safe drinking 3-5 cups of coffee a day. Ultimately, its up to you to understand how your body responds to coffee, and to judge when you shouldn’t have any more. If you respond well to large amounts of caffeine, and don’t become jittery or get stomach pain, then you can drink what you want, but if you know that drinking lots of coffee keeps you up at night and leads to headaches in the morning, cut down to 3 cups a day. Drinking more than 5 cups of coffee a day has been linked to a raise in cholesterol, so if you’re a heavy coffee drinker sticking to 5 is the recommended amount.'"
      ]
     },
     "execution_count": 160,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cosine_value_tf = 1- pairwise_distances(match_frame, Question_tf, metric = 'cosine' )\n",
    "index_value = cosine_value_tf.argmax() \n",
    "answer = data['Text Response'].iloc[index_value]\n",
    "answer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Parsing the Incoming Commands from Slack"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "def parse_bot_commands(slack_events):\n",
    "    for event in slack_events:\n",
    "        if event[\"type\"] == \"message\" and not \"subtype\" in event:\n",
    "            message = event[\"text\"]\n",
    "            return message, event[\"channel\"], event['user']\n",
    "    return None, None, None\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### How to interpret the incoming message from Slack and send appropriate responses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done\n"
     ]
    }
   ],
   "source": [
    "def handle_command(command, channel,user):\n",
    "    # Default response is help text for the user\n",
    "    accepted_users = ['UHYFFGK7S']\n",
    "    ending_text = ['bye','done', 'thanks', 'exit', 'ok', 'x']\n",
    "    if user not in accepted_users:\n",
    "          response = \"Not an authorised user\"\n",
    "    elif command in ending_text:\n",
    "          response = \"Bye! Thanks for chatting\"\n",
    "    else:\n",
    "      try:\n",
    "        Question = nltk_cleaning(command).split(' ')\n",
    "        q_array = np.array(Question)\n",
    "        print(q_array)\n",
    "        print(Question)\n",
    "        print(q_array.shape)\n",
    "        Question_tf = question_embedding(q_array)\n",
    "        Question_tf=Question_tf.reshape(-1,100)\n",
    "        print(Question_tf.shape)\n",
    "        print(Question_tf)\n",
    "        cosine_value_tf = 1- pairwise_distances(match_frame, Question_tf, metric = 'cosine' )\n",
    "        index_value = cosine_value_tf.argmax() \n",
    "        response = data['Text Response'].iloc[index_value]\n",
    "      except:\n",
    "          response='Sorry, Not sure what you mean'\n",
    "        \n",
    "    \n",
    "\n",
    "    # Sends the response back to the channel\n",
    "    slack_clients.api_call(\n",
    "        \"chat.postMessage\",\n",
    "        type=\"divider\",\n",
    "        channel=channel,\n",
    "        #text=':coffee:'+':coffee:'+\"*\"+response+\"*\" + ':coffee:'+':coffee:',\n",
    "        attachments=[{\n",
    "        \"blocks\":[\n",
    "\t\t{\n",
    "\t\t\t\"type\": \"section\",\n",
    "\t\t\t\"block_id\": \"section567\",\n",
    "\t\t\t\"text\": {\n",
    "\t\t\t\t\"type\": \"mrkdwn\",\n",
    "\t\t\t\t\"text\": ':coffee:'+':coffee:'+\"*\"+response+\"*\" + ':coffee:'+':coffee:'\n",
    "\t\t\t},\n",
    "\t\t\t\"accessory\": {\n",
    "\t\t\t\t\"type\": \"image\",\n",
    "\t\t\t\t\"image_url\": \"https://png.pngtree.com/png-clipart/20190903/original/pngtree-yellow-woven-bag-of-coffee-beans-png-image_4418658.jpg\",\n",
    "\t\t\t\t\"alt_text\": \"Haunted hotel image\"\n",
    "\t\t\t}}\n",
    "\t\t]}]\n",
    "    )\n",
    "\n",
    "##Running the application\n",
    "print(\"Done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starter Bot connected and running!\n"
     ]
    }
   ],
   "source": [
    "# constants\n",
    "RTM_READ_DELAY = 1 # 1 second delay between reading from RTM\n",
    "if __name__ == \"__main__\":\n",
    "    if slack_clients.rtm_connect(with_team_state=False):\n",
    "        print(\"Starter Bot connected and running!\")\n",
    "        # Read bot's user ID by calling Web API method `auth.test`\n",
    "        starterbot_id = slack_clients.api_call(\"auth.test\")[\"user_id\"]\n",
    "        while True:\n",
    "            command, channel, user = parse_bot_commands(slack_clients.rtm_read())\n",
    "            if command == 'shutdownthebot':\n",
    "                break\n",
    "            else:\n",
    "               if command is None:\n",
    "                  continue\n",
    "               else:\n",
    "                  handle_command(command, channel, user)\n",
    "            time.sleep(RTM_READ_DELAY)\n",
    "    else:\n",
    "        print(\"Connection failed. Exception traceback printed above.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7.3 64-bit",
   "language": "python",
   "name": "python37364bit2b8427184f3a4d4aa3afd1d3859411ec"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}