neural_network.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.1.0\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dropout, Dense, Conv2D, MaxPool2D, Flatten\n",
    "from tensorflow.keras.callbacks import TensorBoard\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import random\n",
    "import time\n",
    "import os\n",
    "from transformers import BertTokenizer, TFBertModel\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GPUs Available:  0\n"
     ]
    }
   ],
   "source": [
    "gpus = tf.config.experimental.list_physical_devices('GPU')\n",
    "print(\"GPUs Available: \", len(gpus))\n",
    "if len(gpus)>0:\n",
    "    tf.config.experimental.set_memory_growth(gpus[0], True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## insert data\n",
    "### inspect the data\n",
    "\n",
    "- multiclass claasification, implicite feedback\n",
    "- negative sampling\n",
    "- watched history, the user's gepgraphic region, normalzied feature: [0-1] user's gender, logged-in tstae, age; example age(days since upload); serving:0\n",
    "- generate a fixed number of training exampls per user (prevent a small cohort of highly actice users from dominating the loss)\n",
    "- discarding sequence onformation aand represeting search queries with an unordered bag of tokens\n",
    "- predicting future watch, not predicting held-out watch (what most collaborative filtering systems do)\n",
    "\n",
    "- a vocabulary of 1M videos and 1M search tokens were embeded with 256 floats each in a maximum bag size of 50 recent watches and 50 recent searches\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# basePath = os.path.dirname(os.path.abspath(__file__))\n",
    "## embeded video watched, embeded search tokens, geographic embeddings, example page, gender\n",
    "articles = pd.read_csv('data/articles.csv')\n",
    "users = pd.read_csv('data/user.csv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "columns for article: Index(['aid', 'title', 'subTitle', 'totalClapCount', 'tags', 'date'], dtype='object')\n",
      "columns for user: Index(['name', 'bio', 'searchHist', 'watchHistTime', 'watchHist', 'interests'], dtype='object')\n"
     ]
    }
   ],
   "source": [
    "articles = articles[[ 'aid', 'title','subTitle','totalClapCount', 'tags','date']]\n",
    "users = users[['name','bio','searchHist','watchHistTime','watchHist','interests']]\n",
    "print(\"columns for article:\",articles.columns)\n",
    "print(\"columns for user:\",users.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get example age and  normalize totalClapCount into 0-1 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "articles['example_age'] = (pd.to_datetime(\"now\") - pd.to_datetime(articles['date']))\\\n",
    "            /np.timedelta64(1,'D')\n",
    "\n",
    "def normalize_col(df,col_name):\n",
    "    df[col_name] = (df[col_name] - df[col_name].min()) / (df[col_name].max() - df[col_name].min())\n",
    "    return df\n",
    "\n",
    "articles = normalize_col(articles,'totalClapCount')\n",
    "articles = normalize_col(articles,'example_age')\n",
    "articles.drop(columns={'date'}, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>aid</th>\n",
       "      <th>title</th>\n",
       "      <th>subTitle</th>\n",
       "      <th>totalClapCount</th>\n",
       "      <th>tags</th>\n",
       "      <th>example_age</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>Ascent of data Science, SAS and Big data Analy...</td>\n",
       "      <td>Various associations in the present days are o...</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>['Data Science', 'Sas Training In Pune', 'Hado...</td>\n",
       "      <td>0.310136</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>EPQ draft 1 (4844 words)</td>\n",
       "      <td>Introduction</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>['Technology', 'Robotics', 'Automation', 'Epq']</td>\n",
       "      <td>0.400908</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>5</td>\n",
       "      <td>What Is The Importance Of Machine Learning?</td>\n",
       "      <td>Machine learning is the reason for every techn...</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>['Machine Learning', 'Machine Learning Trainin...</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>AI VS Dota2 Champions</td>\n",
       "      <td>What do we know about artificial intelligence?...</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>['Artificial Intelligence', 'Dota 2', 'Cryptoc...</td>\n",
       "      <td>0.127080</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>Private Business, Government and Blockchain</td>\n",
       "      <td>A major private IT company implements blockcha...</td>\n",
       "      <td>0.00753</td>\n",
       "      <td>['Blockchain', 'Samsung', 'It']</td>\n",
       "      <td>0.016641</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   aid                                              title  \\\n",
       "0    4  Ascent of data Science, SAS and Big data Analy...   \n",
       "1    3                           EPQ draft 1 (4844 words)   \n",
       "2    5        What Is The Importance Of Machine Learning?   \n",
       "3    2                              AI VS Dota2 Champions   \n",
       "4    1        Private Business, Government and Blockchain   \n",
       "\n",
       "                                            subTitle  totalClapCount  \\\n",
       "0  Various associations in the present days are o...         0.00000   \n",
       "1                                       Introduction         0.00000   \n",
       "2  Machine learning is the reason for every techn...         0.00000   \n",
       "3  What do we know about artificial intelligence?...         0.00000   \n",
       "4  A major private IT company implements blockcha...         0.00753   \n",
       "\n",
       "                                                tags  example_age  \n",
       "0  ['Data Science', 'Sas Training In Pune', 'Hado...     0.310136  \n",
       "1    ['Technology', 'Robotics', 'Automation', 'Epq']     0.400908  \n",
       "2  ['Machine Learning', 'Machine Learning Trainin...     0.000000  \n",
       "3  ['Artificial Intelligence', 'Dota 2', 'Cryptoc...     0.127080  \n",
       "4                    ['Blockchain', 'Samsung', 'It']     0.016641  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "articles.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>bio</th>\n",
       "      <th>searchHist</th>\n",
       "      <th>watchHistTime</th>\n",
       "      <th>watchHist</th>\n",
       "      <th>interests</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>melodyucros</td>\n",
       "      <td>Entrepreneurial Techie who loves helping start...</td>\n",
       "      <td>['']</td>\n",
       "      <td>[None, 1809, 8460]</td>\n",
       "      <td>[None, 210, 210]</td>\n",
       "      <td>['Career Change']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>techdatasolutions18</td>\n",
       "      <td>Tech data Providing Courses Like SaS Training ...</td>\n",
       "      <td>['']</td>\n",
       "      <td>[None, 4633, 4532]</td>\n",
       "      <td>[None, 386, 386]</td>\n",
       "      <td>['Revnet']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>babaevanar</td>\n",
       "      <td>NaN</td>\n",
       "      <td>['']</td>\n",
       "      <td>[None]</td>\n",
       "      <td>[None]</td>\n",
       "      <td>['Vectorization', 'Farmers', 'Additive M', 'De...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>tasty231</td>\n",
       "      <td>NaN</td>\n",
       "      <td>['', 'OBIZCOIN', 'Classification', 'Evolution'...</td>\n",
       "      <td>[None, 1375, 996, 829, 3966, 4020, 4454, 2168,...</td>\n",
       "      <td>[None, 78, 226, 168, 465, 478, 466, 381, 415, ...</td>\n",
       "      <td>['Obizcoin', 'Io18', 'Classification', 'Evolut...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>stewartalsop</td>\n",
       "      <td>NaN</td>\n",
       "      <td>['', 'Transparency', 'Nips2016', 'Nips']</td>\n",
       "      <td>[None, 2378, 3778, 8930, 6172, 5499]</td>\n",
       "      <td>[None, 126, 128, 390, 52, 52]</td>\n",
       "      <td>['Transparency', 'Nips2016', 'Cloud Tpu']</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  name                                                bio  \\\n",
       "0          melodyucros  Entrepreneurial Techie who loves helping start...   \n",
       "1  techdatasolutions18  Tech data Providing Courses Like SaS Training ...   \n",
       "2           babaevanar                                                NaN   \n",
       "3             tasty231                                                NaN   \n",
       "4         stewartalsop                                                NaN   \n",
       "\n",
       "                                          searchHist  \\\n",
       "0                                               ['']   \n",
       "1                                               ['']   \n",
       "2                                               ['']   \n",
       "3  ['', 'OBIZCOIN', 'Classification', 'Evolution'...   \n",
       "4           ['', 'Transparency', 'Nips2016', 'Nips']   \n",
       "\n",
       "                                       watchHistTime  \\\n",
       "0                                 [None, 1809, 8460]   \n",
       "1                                 [None, 4633, 4532]   \n",
       "2                                             [None]   \n",
       "3  [None, 1375, 996, 829, 3966, 4020, 4454, 2168,...   \n",
       "4               [None, 2378, 3778, 8930, 6172, 5499]   \n",
       "\n",
       "                                           watchHist  \\\n",
       "0                                   [None, 210, 210]   \n",
       "1                                   [None, 386, 386]   \n",
       "2                                             [None]   \n",
       "3  [None, 78, 226, 168, 465, 478, 466, 381, 415, ...   \n",
       "4                      [None, 126, 128, 390, 52, 52]   \n",
       "\n",
       "                                           interests  \n",
       "0                                  ['Career Change']  \n",
       "1                                         ['Revnet']  \n",
       "2  ['Vectorization', 'Farmers', 'Additive M', 'De...  \n",
       "3  ['Obizcoin', 'Io18', 'Classification', 'Evolut...  \n",
       "4          ['Transparency', 'Nips2016', 'Cloud Tpu']  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "users.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "name             object\n",
       "bio              object\n",
       "searchHist       object\n",
       "watchHistTime    object\n",
       "watchHist        object\n",
       "interests        object\n",
       "dtype: object"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "users.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import ast\n",
    "def convert_to_list(x):\n",
    "    # convert string list into list, remove None in list, remove duplicates\n",
    "    return list(set(list(filter(None.__ne__, ast.literal_eval(x))))) \n",
    "\n",
    "def format_list_user(x):\n",
    "    x['searchHist'] = list(filter(None, ast.literal_eval(x['searchHist'])))\n",
    "    x['watchHistTime'] = list(filter(None, ast.literal_eval(x['watchHistTime']))) \n",
    "    x['watchHist'] = list(filter(None, ast.literal_eval(x['watchHist']))) \n",
    "    x['interests'] = list(filter(None, ast.literal_eval(x['interests']))) \n",
    "    return x\n",
    "\n",
    "\n",
    "users = users.apply(format_list_user, axis=1)\n",
    "articles['tags'] = articles['tags'].apply(convert_to_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>bio</th>\n",
       "      <th>searchHist</th>\n",
       "      <th>watchHistTime</th>\n",
       "      <th>watchHist</th>\n",
       "      <th>interests</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>melodyucros</td>\n",
       "      <td>Entrepreneurial Techie who loves helping start...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[1809, 8460]</td>\n",
       "      <td>[210, 210]</td>\n",
       "      <td>[Career Change]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>techdatasolutions18</td>\n",
       "      <td>Tech data Providing Courses Like SaS Training ...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[4633, 4532]</td>\n",
       "      <td>[386, 386]</td>\n",
       "      <td>[Revnet]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>babaevanar</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Vectorization, Farmers, Additive M, Decision ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>tasty231</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[OBIZCOIN, Classification, Evolution, Mechanic...</td>\n",
       "      <td>[1375, 996, 829, 3966, 4020, 4454, 2168, 3969,...</td>\n",
       "      <td>[78, 226, 168, 465, 478, 466, 381, 415, 421]</td>\n",
       "      <td>[Obizcoin, Io18, Classification, Evolution, Me...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>stewartalsop</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[Transparency, Nips2016, Nips]</td>\n",
       "      <td>[2378, 3778, 8930, 6172, 5499]</td>\n",
       "      <td>[126, 128, 390, 52, 52]</td>\n",
       "      <td>[Transparency, Nips2016, Cloud Tpu]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                  name                                                bio  \\\n",
       "0          melodyucros  Entrepreneurial Techie who loves helping start...   \n",
       "1  techdatasolutions18  Tech data Providing Courses Like SaS Training ...   \n",
       "2           babaevanar                                                NaN   \n",
       "3             tasty231                                                NaN   \n",
       "4         stewartalsop                                                NaN   \n",
       "\n",
       "                                          searchHist  \\\n",
       "0                                                 []   \n",
       "1                                                 []   \n",
       "2                                                 []   \n",
       "3  [OBIZCOIN, Classification, Evolution, Mechanic...   \n",
       "4                     [Transparency, Nips2016, Nips]   \n",
       "\n",
       "                                       watchHistTime  \\\n",
       "0                                       [1809, 8460]   \n",
       "1                                       [4633, 4532]   \n",
       "2                                                 []   \n",
       "3  [1375, 996, 829, 3966, 4020, 4454, 2168, 3969,...   \n",
       "4                     [2378, 3778, 8930, 6172, 5499]   \n",
       "\n",
       "                                      watchHist  \\\n",
       "0                                    [210, 210]   \n",
       "1                                    [386, 386]   \n",
       "2                                            []   \n",
       "3  [78, 226, 168, 465, 478, 466, 381, 415, 421]   \n",
       "4                       [126, 128, 390, 52, 52]   \n",
       "\n",
       "                                           interests  \n",
       "0                                    [Career Change]  \n",
       "1                                           [Revnet]  \n",
       "2  [Vectorization, Farmers, Additive M, Decision ...  \n",
       "3  [Obizcoin, Io18, Classification, Evolution, Me...  \n",
       "4                [Transparency, Nips2016, Cloud Tpu]  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "users.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "assert (users['watchHistTime'].apply(len) == users['watchHist'].apply(len)).all()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# active users"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "act_users = users.copy()\n",
    "act_users['watchHist'] = act_users['watchHist'].apply(lambda x: list(set(x))) # remove duplicated watched item ids\n",
    "act_users = act_users.explode('watchHist')\n",
    "act_users = act_users.drop(columns = {'watchHistTime'})\n",
    "act_users['bio'] = act_users['bio'].fillna(' ')\n",
    "\n",
    "act_users = act_users.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(287, 5)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "act_users.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1a49ee8588>"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD8CAYAAABn919SAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi40LCBodHRwOi8vbWF0cGxvdGxpYi5vcmcv7US4rQAADT1JREFUeJzt3WGI3Hedx/HPx0QxyWpaiQ69TXEVSj3pcvQySDUgs41y1RTrA+Va2tKKsk/uak5yHPHg6CO5PLiIpcjBUquFhg4aCymtqKU6JwdnuN20sG1XqdTYJsZsJdfVLYEa7nsPdg7CknUz//9/59f9zvsFZXcm/9nf99edvDP7350dR4QAAJvf20oPAABoBkEHgCQIOgAkQdABIAmCDgBJEHQASIKgA0ASBB0AkiDoAJDE1mEutmvXrpiYmBjmkrW98cYb2rFjR+kxhoo9jwb2vHnMzc39PiLeu95xQw36xMSEZmdnh7lkbb1eT51Op/QYQ8WeRwN73jxs/+ZKjuOUCwAkQdABIAmCDgBJEHQASIKgA0ASBB0AkiDoAJAEQQeAJAg6ACQx1GeKAm9V82eWdO+hp4qsferw/iLrIh8eoQNAEgQdAJIg6ACQBEEHgCQIOgAkQdABIAmCDgBJEHQASIKgA0ASBB0AkiDoAJAEQQeAJAg6ACRB0AEgiXWDbvth24u2n7/kuvfYftr2S/23V2/smACA9VzJI/TvSLpl1XWHJD0TEddJeqZ/GQBQ0LpBj4ifSTq/6urbJD3Sf/8RSZ9teC4AwICqnkNvRcRZSeq/fV9zIwEAqnBErH+QPSHpyYi4oX/59Yi46pI//5+IuOx5dNvTkqYlqdVq7el2uw2MPTzLy8saGxsrPcZQLZ5f0rkLZdaeHN9ZZN1R3PMo3rc3656npqbmIqK93nFVX1P0nO1rIuKs7WskLa51YETMSJqRpHa7HZ1Op+KSZfR6PW22met68OhxHZkv83Kzp+7sFFl3FPc8ivft7HuuesrlCUn39N+/R9LxZsYBAFR1JT+2+Jik/5J0ve3Ttr8o6bCkT9p+SdIn+5cBAAWt+zVmRNyxxh/ta3gWAEANPFMUAJIg6ACQBEEHgCQIOgAkQdABIAmCDgBJEHQASIKgA0ASBB0AkiDoAJAEQQeAJAg6ACRB0AEgiTK/0R9Yw8Shp4qse3CyyLJAo3iEDgBJEHQASIKgA0ASBB0AkiDoAJAEQQeAJAg6ACRB0AEgCYIOAEkQdABIgqADQBIEHQCSIOgAkARBB4AkCDoAJFEr6La/YvsF28/bfsz2O5saDAAwmMpBtz0u6cuS2hFxg6Qtkm5vajAAwGDqnnLZKmmb7a2Stkv6bf2RAABVOCKq39g+IOlrki5I+nFE3HmZY6YlTUtSq9Xa0+12K69XwvLyssbGxkqPMVSL55d07kLpKYartU3F9jw5vrPIuqN4396se56ampqLiPZ6x1UOuu2rJX1f0t9Kel3S9yQdi4hH17pNu92O2dnZSuuV0uv11Ol0So8xVA8ePa4j86P1crMHJy8W2/Opw/uLrDuK9+3NumfbVxT0OqdcPiHp1xHxWkT8SdLjkj5W4+MBAGqoE/RXJN1ke7ttS9onaaGZsQAAg6oc9Ig4IemYpJOS5vsfa6ahuQAAA6p10jAi7pd0f0OzAABq4JmiAJAEQQeAJAg6ACRB0AEgCYIOAEkQdABIgqADQBIEHQCSIOgAkARBB4AkCDoAJEHQASAJgg4ASYzWy9JUMH9mSfceeqrI2qVeyQbA5sQjdABIgqADQBIEHQCSIOgAkARBB4AkCDoAJEHQASAJgg4ASRB0AEiCoANAEgQdAJIg6ACQBEEHgCQIOgAkUSvotq+yfcz2L2wv2P5oU4MBAAZT9/ehPyDphxHxOdvvkLS9gZkAABVUDrrtd0v6uKR7JSki3pT0ZjNjAQAGVeeUywclvSbp27aftf2Q7R0NzQUAGJAjotoN7bakn0vaGxEnbD8g6Q8R8S+rjpuWNC1JrVZrT7fbrTnycC2eX9K5C2XWnhzfWWTdknsupbVNI/d5Xl5e1tjYWJG1S9mse56ampqLiPZ6x9U5h35a0umIONG/fEzSodUHRcSMpBlJarfb0el0aiw5fA8ePa4j82VeevXUnZ0i65bccykHJy+O3Oe51+tps/19rCv7niufcomI30l61fb1/av2SXqxkakAAAOr+5DkPklH+z/h8rKkL9QfCQBQRa2gR8RzktY9rwMA2Hg8UxQAkiDoAJAEQQeAJAg6ACRB0AEgCYIOAEkQdABIgqADQBIEHQCSIOgAkARBB4AkCDoAJEHQASAJgg4ASRB0AEiCoANAEgQdAJIg6ACQBEEHgCQIOgAkQdABIAmCDgBJEHQASIKgA0ASBB0AkiDoAJAEQQeAJAg6ACRB0AEgidpBt73F9rO2n2xiIABANU08Qj8gaaGBjwMAqKFW0G3vlrRf0kPNjAMAqMoRUf3G9jFJ/yrpXZL+MSJuvcwx05KmJanVau3pdruV1po/s1R5zjpa26RzF4osrcnxnUXWXTy/VGzPpZT8PJcyivft5eVljY2NFVm7jqmpqbmIaK933NaqC9i+VdJiRMzZ7qx1XETMSJqRpHa7HZ3Omof+WfceeqrS7eo6OHlRR+Yr/2+q5dSdnSLrPnj0eLE9l1Ly81zKKN63e72eqjZoM6hzymWvpM/YPiWpK+lm2482MhUAYGCVgx4RX42I3RExIel2ST+JiLsamwwAMBB+Dh0AkmjkBFpE9CT1mvhYAIBqeIQOAEkQdABIgqADQBIEHQCSIOgAkARBB4AkCDoAJEHQASAJgg4ASRB0AEiCoANAEgQdAJIg6ACQxGi9RMsmM1HsVZqKLAtsuPkzS0Ve/ezU4f1DWYdH6ACQBEEHgCQIOgAkQdABIAmCDgBJEHQASIKgA0ASBB0AkiDoAJAEQQeAJAg6ACRB0AEgCYIOAEkQdABIonLQbV9r+6e2F2y/YPtAk4MBAAZT5/ehX5R0MCJO2n6XpDnbT0fEiw3NBgAYQOVH6BFxNiJO9t//o6QFSeNNDQYAGEwj59BtT0i6UdKJJj4eAGBwjoh6H8Aek/Qfkr4WEY9f5s+nJU1LUqvV2tPtdiutM39mqc6YlbW2SecuFFm6GPY8Gtjz8EyO76x1+6mpqbmIaK93XK2g2367pCcl/Sgivr7e8e12O2ZnZyutVe71NS/qyPxovfQqex4N7Hl46r6mqO0rCnqdn3KxpG9JWriSmAMANladc+h7Jd0t6Wbbz/X/+3RDcwEABlT5a4+I+E9JbnAWAEANPFMUAJIg6ACQBEEHgCQIOgAkQdABIAmCDgBJEHQASIKgA0ASBB0AkiDoAJAEQQeAJAg6ACRB0AEgCYIOAEkQdABIgqADQBIEHQCSIOgAkARBB4AkCDoAJEHQASAJgg4ASRB0AEiCoANAEgQdAJIg6ACQBEEHgCQIOgAkQdABIIlaQbd9i+1f2v6V7UNNDQUAGFzloNveIumbkj4l6cOS7rD94aYGAwAMps4j9I9I+lVEvBwRb0rqSrqtmbEAAIOqE/RxSa9ecvl0/zoAQAGOiGo3tD8v6W8i4kv9y3dL+khE3LfquGlJ0/2L10v6ZfVxi9gl6felhxgy9jwa2PPm8f6IeO96B22tscBpSddecnm3pN+uPigiZiTN1FinKNuzEdEuPccwsefRwJ7zqXPK5b8lXWf7A7bfIel2SU80MxYAYFCVH6FHxEXbfy/pR5K2SHo4Il5obDIAwEDqnHJRRPxA0g8amuWtatOeLqqBPY8G9pxM5W+KAgDeWnjqPwAkQdDXYPta2z+1vWD7BdsHSs80LLa32H7W9pOlZxkG21fZPmb7F/3P90dLz7TRbH+lf79+3vZjtt9Zeqam2X7Y9qLt5y+57j22n7b9Uv/t1SVnbBpBX9tFSQcj4i8l3STp70boVxsckLRQeoghekDSDyPiQ5L+Ssn3bntc0pcltSPiBq38UMPtZafaEN+RdMuq6w5JeiYirpP0TP9yGgR9DRFxNiJO9t//o1b+kqd/Jqzt3ZL2S3qo9CzDYPvdkj4u6VuSFBFvRsTrZacaiq2SttneKmm7LvMcks0uIn4m6fyqq2+T9Ej//UckfXaoQ20wgn4FbE9IulHSibKTDMU3JP2TpP8tPciQfFDSa5K+3T/N9JDtHaWH2kgRcUbSv0l6RdJZSUsR8eOyUw1NKyLOSisP2iS9r/A8jSLo67A9Jun7kv4hIv5Qep6NZPtWSYsRMVd6liHaKumvJf17RNwo6Q0l+zJ8tf5549skfUDSX0jaYfuuslOhCQT9z7D9dq3E/GhEPF56niHYK+kztk9p5bdn3mz70bIjbbjTkk5HxP9/9XVMK4HP7BOSfh0Rr0XEnyQ9LuljhWcalnO2r5Gk/tvFwvM0iqCvwba1cl51ISK+XnqeYYiIr0bE7oiY0Mo3yX4SEakfuUXE7yS9avv6/lX7JL1YcKRheEXSTba39+/n+5T8G8GXeELSPf3375F0vOAsjav1TNHk9kq6W9K87ef61/1z/9mxyOU+SUf7v5PoZUlfKDzPhoqIE7aPSTqplZ/melYJn0Fp+zFJHUm7bJ+WdL+kw5K+a/uLWvmH7fPlJmwezxQFgCQ45QIASRB0AEiCoANAEgQdAJIg6ACQBEEHgCQIOgAkQdABIIn/A61LTB09Mz4rAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "act_users.groupby('name')['watchHist'].count().hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    57.000000\n",
       "mean      5.035088\n",
       "std       2.427263\n",
       "min       1.000000\n",
       "25%       4.000000\n",
       "50%       5.000000\n",
       "75%       6.000000\n",
       "max      11.000000\n",
       "Name: watchHist, dtype: float64"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "act_users.groupby('name')['watchHist'].count().describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# act_users.groupby('name')['watchHist'].count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# meas search amount is 5, generate 10 for each user"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['aid', 'title', 'subTitle', 'totalClapCount', 'tags', 'example_age'], dtype='object')"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "articles.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_pos = pd.merge(act_users,articles, left_on=['watchHist'],right_on =['aid'],how='inner')\\\n",
    "        .drop(columns = ['watchHist'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(287, 10)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_pos.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_need_sample= act_users.groupby('name')['watchHist'].count().reset_index(name = 'count')\n",
    "\n",
    "negative_list = []\n",
    "for index, row in user_need_sample.iterrows():\n",
    "    temp = articles.sample(n= 12 - row['count'])\n",
    "    temp['name'] = row['name']\n",
    "    negative_list.append(temp)\n",
    "\n",
    "negative_df = pd.concat(negative_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['aid', 'title', 'subTitle', 'totalClapCount', 'tags', 'example_age',\n",
       "       'name'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "negative_df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "57"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(negative_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['name', 'bio', 'searchHist', 'interests', 'aid', 'title', 'subTitle',\n",
       "       'totalClapCount', 'tags', 'example_age'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_pos.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['name', 'bio', 'searchHist', 'watchHist', 'interests'], dtype='object')"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "act_users.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(397, 7)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "negative_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "100"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(users['name'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "trian_neg = pd.merge(negative_df,users,on=['name'],how='inner').sample(n=train_pos.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['aid', 'title', 'subTitle', 'totalClapCount', 'tags', 'example_age',\n",
       "       'name', 'bio', 'searchHist', 'watchHistTime', 'watchHist', 'interests'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trian_neg.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(287, 12)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trian_neg.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(287, 5)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "act_users.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove the negative articles that appeared in watched list\n",
    "trian_neg['deleted'] = trian_neg.apply(lambda x: x['aid'] in x['watchHist'], axis=1)\n",
    "trian_neg = trian_neg[trian_neg['deleted']==False].drop(columns={'watchHist','deleted','watchHistTime'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "trian_neg['label'] = 0\n",
    "train_pos['label'] = train_pos['aid']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/rachelchen/.local/lib/python3.6/site-packages/ipykernel_launcher.py:1: FutureWarning: Sorting because non-concatenation axis is not aligned. A future version\n",
      "of pandas will change to not sort by default.\n",
      "\n",
      "To accept the future behavior, pass 'sort=False'.\n",
      "\n",
      "To retain the current behavior and silence the warning, pass 'sort=True'.\n",
      "\n",
      "  \"\"\"Entry point for launching an IPython kernel.\n"
     ]
    }
   ],
   "source": [
    "train_data = pd.concat([train_pos,trian_neg])\n",
    "train_data = train_data.drop(columns={'aid','name'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "bio                True\n",
       "example_age       False\n",
       "interests         False\n",
       "label             False\n",
       "searchHist        False\n",
       "subTitle           True\n",
       "tags              False\n",
       "title             False\n",
       "totalClapCount    False\n",
       "dtype: bool"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.isnull().any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data['subTitle'] = train_data['subTitle'].fillna('')\n",
    "train_data['bio'] = train_data['bio'].fillna('')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(568, 9)"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['bio', 'example_age', 'interests', 'label', 'searchHist', 'subTitle',\n",
       "       'tags', 'title', 'totalClapCount'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "## use the embedding "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['bio', 'example_age', 'interests', 'label', 'searchHist', 'subTitle',\n",
       "       'tags', 'title', 'totalClapCount'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "## preprare for bert embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data['interests'] = train_data['interests'].apply(lambda x: ' '.join(x))\n",
    "train_data['tags'] = train_data['tags'].apply(lambda x: ' '.join(x))\n",
    "train_data['searchHist'] = train_data['searchHist'].apply(lambda x: ' '.join(x))\n",
    "\n",
    "train_data['article_emb'] = train_data['title']+ train_data['subTitle']+train_data['tags'] \n",
    "train_data['user_beh_emb'] = train_data['searchHist'] \n",
    "train_data['user_emb'] = train_data['bio'] + train_data['interests']\n",
    "\n",
    "train_data = train_data[['article_emb','user_beh_emb','user_emb','totalClapCount','label','example_age']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_bert_emb(df,col_name,max_length=10):\n",
    "    tokenizer = BertTokenizer.from_pretrained('bert-base-cased')\n",
    "    model = TFBertModel.from_pretrained('bert-base-cased')\n",
    "    batch_encoding = tokenizer.batch_encode_plus(df[col_name].tolist(), max_length=max_length, pad_to_max_length=True)\n",
    "\n",
    "    outputs = model(tf.convert_to_tensor(batch_encoding['input_ids'])) # shape: (batch,sequence length, hidden state)\n",
    "    embeddings_video = tf.reduce_mean(outputs[0],1)\n",
    "    return embeddings_video.numpy().tolist(),batch_encoding['input_ids']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_data.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "article_emb       False\n",
       "user_beh_emb      False\n",
       "user_emb          False\n",
       "totalClapCount    False\n",
       "label             False\n",
       "example_age       False\n",
       "dtype: bool"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.isna().any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "# average the histrical article embedding \n",
    "# average user_beh_embedding \n",
    "# user_emb \n",
    "\n",
    "# multi-class problem"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data['article_emb_bert'],train_data['article_emb_inputid'] = get_bert_emb(train_data,'article_emb')\n",
    "train_data['user_beh_emb_bert'],train_data['user_beh_emb_inputid'] = get_bert_emb(train_data,'user_beh_emb')\n",
    "train_data['user_emb_bert'],train_data['user_emb_inputid'] = get_bert_emb(train_data,'user_emb')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data['example_age_square'] = train_data['example_age'].map(np.square)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['article_emb', 'user_beh_emb', 'user_emb', 'totalClapCount', 'label',\n",
       "       'example_age', 'article_emb_bert', 'article_emb_inputid',\n",
       "       'user_beh_emb_bert', 'user_beh_emb_inputid', 'user_emb_bert',\n",
       "       'user_emb_inputid', 'example_age_square'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>article_emb</th>\n",
       "      <th>user_beh_emb</th>\n",
       "      <th>user_emb</th>\n",
       "      <th>totalClapCount</th>\n",
       "      <th>label</th>\n",
       "      <th>example_age</th>\n",
       "      <th>article_emb_bert</th>\n",
       "      <th>article_emb_inputid</th>\n",
       "      <th>user_beh_emb_bert</th>\n",
       "      <th>user_beh_emb_inputid</th>\n",
       "      <th>user_emb_bert</th>\n",
       "      <th>user_emb_inputid</th>\n",
       "      <th>example_age_square</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>Making the SwitchThe Painstakingly Slow Beginn...</td>\n",
       "      <td></td>\n",
       "      <td>Entrepreneurial Techie who loves helping start...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>210</td>\n",
       "      <td>0.039334</td>\n",
       "      <td>[0.22904324531555176, 0.02765187993645668, 0.1...</td>\n",
       "      <td>[101, 10697, 1103, 156, 18548, 1942, 4638, 133...</td>\n",
       "      <td>[-0.17436088621616364, -0.2535916566848755, 0....</td>\n",
       "      <td>[101, 102, 0, 0, 0, 0, 0, 0, 0, 0]</td>\n",
       "      <td>[0.33958497643470764, 0.15296240150928497, 0.0...</td>\n",
       "      <td>[101, 13832, 7877, 1643, 16717, 19700, 1233, 7...</td>\n",
       "      <td>0.001547</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>Code with Eager Execution, Run with Graphs: Op...</td>\n",
       "      <td></td>\n",
       "      <td>Tech data Providing Courses Like SaS Training ...</td>\n",
       "      <td>0.007078</td>\n",
       "      <td>386</td>\n",
       "      <td>0.077156</td>\n",
       "      <td>[0.10684485733509064, 0.19496707618236542, 0.2...</td>\n",
       "      <td>[101, 6741, 1114, 142, 12344, 16409, 10294, 12...</td>\n",
       "      <td>[-0.17436088621616364, -0.2535916566848755, 0....</td>\n",
       "      <td>[101, 102, 0, 0, 0, 0, 0, 0, 0, 0]</td>\n",
       "      <td>[0.2618952691555023, 0.13922862708568573, -0.1...</td>\n",
       "      <td>[101, 7882, 2233, 5096, 18312, 1158, 10176, 11...</td>\n",
       "      <td>0.005953</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>Code with Eager Execution, Run with Graphs: Op...</td>\n",
       "      <td></td>\n",
       "      <td>Computer Science Undergrad trying to figure st...</td>\n",
       "      <td>0.007078</td>\n",
       "      <td>386</td>\n",
       "      <td>0.077156</td>\n",
       "      <td>[0.10684485733509064, 0.19496707618236542, 0.2...</td>\n",
       "      <td>[101, 6741, 1114, 142, 12344, 16409, 10294, 12...</td>\n",
       "      <td>[-0.17436088621616364, -0.2535916566848755, 0....</td>\n",
       "      <td>[101, 102, 0, 0, 0, 0, 0, 0, 0, 0]</td>\n",
       "      <td>[0.15719716250896454, 0.2271394282579422, 0.09...</td>\n",
       "      <td>[101, 6701, 2444, 2831, 20561, 1774, 1106, 248...</td>\n",
       "      <td>0.005953</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>Application of Random Forest Algorithm in Remo...</td>\n",
       "      <td>OBIZCOIN Classification Evolution Mechanical M...</td>\n",
       "      <td>Obizcoin Io18 Classification Evolution Mechan...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>226</td>\n",
       "      <td>0.240545</td>\n",
       "      <td>[0.02249579131603241, 0.12434376776218414, -0....</td>\n",
       "      <td>[101, 22491, 1104, 19441, 4089, 2586, 18791, 7...</td>\n",
       "      <td>[0.13140153884887695, -0.12303154170513153, -0...</td>\n",
       "      <td>[101, 152, 24963, 5301, 15678, 11607, 19295, 1...</td>\n",
       "      <td>[0.5636561512947083, -0.11189024150371552, 0.1...</td>\n",
       "      <td>[101, 152, 5567, 1584, 2528, 1394, 146, 1186, ...</td>\n",
       "      <td>0.057862</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>Application of Random Forest Algorithm in Remo...</td>\n",
       "      <td>Tech Data</td>\n",
       "      <td>Freelance writer and content marketer. Mainly ...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>226</td>\n",
       "      <td>0.240545</td>\n",
       "      <td>[0.02249579131603241, 0.12434376776218414, -0....</td>\n",
       "      <td>[101, 22491, 1104, 19441, 4089, 2586, 18791, 7...</td>\n",
       "      <td>[0.06514380872249603, -0.1176924854516983, 0.3...</td>\n",
       "      <td>[101, 7882, 7154, 102, 0, 0, 0, 0, 0, 0]</td>\n",
       "      <td>[0.4467066824436188, -0.12890012562274933, 0.0...</td>\n",
       "      <td>[101, 4299, 13831, 2432, 1105, 3438, 2319, 120...</td>\n",
       "      <td>0.057862</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         article_emb  \\\n",
       "0  Making the SwitchThe Painstakingly Slow Beginn...   \n",
       "1  Code with Eager Execution, Run with Graphs: Op...   \n",
       "2  Code with Eager Execution, Run with Graphs: Op...   \n",
       "3  Application of Random Forest Algorithm in Remo...   \n",
       "4  Application of Random Forest Algorithm in Remo...   \n",
       "\n",
       "                                        user_beh_emb  \\\n",
       "0                                                      \n",
       "1                                                      \n",
       "2                                                      \n",
       "3  OBIZCOIN Classification Evolution Mechanical M...   \n",
       "4                                          Tech Data   \n",
       "\n",
       "                                            user_emb  totalClapCount  label  \\\n",
       "0  Entrepreneurial Techie who loves helping start...        0.000000    210   \n",
       "1  Tech data Providing Courses Like SaS Training ...        0.007078    386   \n",
       "2  Computer Science Undergrad trying to figure st...        0.007078    386   \n",
       "3   Obizcoin Io18 Classification Evolution Mechan...        0.000000    226   \n",
       "4  Freelance writer and content marketer. Mainly ...        0.000000    226   \n",
       "\n",
       "   example_age                                   article_emb_bert  \\\n",
       "0     0.039334  [0.22904324531555176, 0.02765187993645668, 0.1...   \n",
       "1     0.077156  [0.10684485733509064, 0.19496707618236542, 0.2...   \n",
       "2     0.077156  [0.10684485733509064, 0.19496707618236542, 0.2...   \n",
       "3     0.240545  [0.02249579131603241, 0.12434376776218414, -0....   \n",
       "4     0.240545  [0.02249579131603241, 0.12434376776218414, -0....   \n",
       "\n",
       "                                 article_emb_inputid  \\\n",
       "0  [101, 10697, 1103, 156, 18548, 1942, 4638, 133...   \n",
       "1  [101, 6741, 1114, 142, 12344, 16409, 10294, 12...   \n",
       "2  [101, 6741, 1114, 142, 12344, 16409, 10294, 12...   \n",
       "3  [101, 22491, 1104, 19441, 4089, 2586, 18791, 7...   \n",
       "4  [101, 22491, 1104, 19441, 4089, 2586, 18791, 7...   \n",
       "\n",
       "                                   user_beh_emb_bert  \\\n",
       "0  [-0.17436088621616364, -0.2535916566848755, 0....   \n",
       "1  [-0.17436088621616364, -0.2535916566848755, 0....   \n",
       "2  [-0.17436088621616364, -0.2535916566848755, 0....   \n",
       "3  [0.13140153884887695, -0.12303154170513153, -0...   \n",
       "4  [0.06514380872249603, -0.1176924854516983, 0.3...   \n",
       "\n",
       "                                user_beh_emb_inputid  \\\n",
       "0                 [101, 102, 0, 0, 0, 0, 0, 0, 0, 0]   \n",
       "1                 [101, 102, 0, 0, 0, 0, 0, 0, 0, 0]   \n",
       "2                 [101, 102, 0, 0, 0, 0, 0, 0, 0, 0]   \n",
       "3  [101, 152, 24963, 5301, 15678, 11607, 19295, 1...   \n",
       "4           [101, 7882, 7154, 102, 0, 0, 0, 0, 0, 0]   \n",
       "\n",
       "                                       user_emb_bert  \\\n",
       "0  [0.33958497643470764, 0.15296240150928497, 0.0...   \n",
       "1  [0.2618952691555023, 0.13922862708568573, -0.1...   \n",
       "2  [0.15719716250896454, 0.2271394282579422, 0.09...   \n",
       "3  [0.5636561512947083, -0.11189024150371552, 0.1...   \n",
       "4  [0.4467066824436188, -0.12890012562274933, 0.0...   \n",
       "\n",
       "                                    user_emb_inputid  example_age_square  \n",
       "0  [101, 13832, 7877, 1643, 16717, 19700, 1233, 7...            0.001547  \n",
       "1  [101, 7882, 2233, 5096, 18312, 1158, 10176, 11...            0.005953  \n",
       "2  [101, 6701, 2444, 2831, 20561, 1774, 1106, 248...            0.005953  \n",
       "3  [101, 152, 5567, 1584, 2528, 1394, 146, 1186, ...            0.057862  \n",
       "4  [101, 4299, 13831, 2432, 1105, 3438, 2319, 120...            0.057862  "
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# values = train_data[['article_emb_inputid','user_beh_emb_inputid','user_emb_inputid']].values\n",
    "# values = train_data[['article_emb_inputid','user_beh_emb_inputid','user_emb_inputid']].values\\\n",
    "#         .reshape(values.shape[0]*values.shape[1],)\n",
    "# values = list(set([item for sublist in values for item in sublist]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import preprocessing\n",
    "lb = preprocessing.LabelBinarizer()\n",
    "label_trans = lb.fit_transform(train_data['label'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(568, 187)"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label_trans.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "#### wrong: the feature shape is (example_num, feature_amount(768)), it can not be embeeding, \n",
    "### where embedding is for each word, built a vector "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "# num_tags = 12  # Number of unique issue tags\n",
    "# num_words = 10000  # Size of vocabulary obtained when preprocessing text data\n",
    "# num_departments = 4  # Number of departments for predictions\n",
    "\n",
    "# article_input = tf.keras.Input(\n",
    "#     shape=(None,), name=\"article\"\n",
    "# )  \n",
    "# userbeh_input = tf.keras.Input(shape=(None,), name=\"userbehaviour\")  \n",
    "\n",
    "# user_input = tf.keras.Input(\n",
    "#     shape=(None,), name=\"user\"\n",
    "# )  \n",
    "\n",
    "# article_features = tf.keras.layers.Embedding(np.array(train_data['article_emb_bert'].values.tolist()).shape[0], \n",
    "#                                              np.array(train_data['article_emb_bert'].values.tolist()).shape[1], \n",
    "#                                     weights=[np.array(train_data['article_emb_bert'].values.tolist())], \n",
    "#                                     input_length=4, trainable=True)(article_input)\n",
    "# user_beh_features = tf.keras.layers.Embedding(np.array(train_data['user_beh_emb_bert'].values.tolist()).shape[0],\n",
    "#                                               np.array(train_data['user_beh_emb_bert'].values.tolist()).shape[1], \n",
    "#                                     weights=[np.array(train_data['user_beh_emb_bert'].values.tolist())], \n",
    "#                                     input_length=4, trainable=True)(userbeh_input)\n",
    "# user_features = tf.keras.layers.Embedding(np.array(train_data['user_emb_bert'].values.tolist()).shape[0], \n",
    "#                                           np.array(train_data['user_emb_bert'].values.tolist()).shape[1], \n",
    "#                                     weights=[np.array(train_data['user_emb_bert'].values.tolist())], \n",
    "#                                     input_length=4, trainable=True)(user_input)\n",
    "\n",
    "# # Merge all available features into a single large vector via concatenation\n",
    "# x = tf.keras.layers.concatenate([article_features, user_beh_features, user_features])\n",
    "\n",
    "# # Stick a logistic regression for priority prediction on top of the features\n",
    "# hidden_layer = tf.keras.layers.Dense(256, activation='relu', name=\"relu\")(x)\n",
    "# priority_pred = tf.keras.layers.Dense(187, activation= 'softmax', name=\"priority\")(hidden_layer)\n",
    "\n",
    "# # Instantiate an end-to-end model predicting both priority and department\n",
    "# model = tf.keras.Model(\n",
    "#     inputs=[article_input, userbeh_input, user_input],\n",
    "#     outputs=priority_pred,\n",
    "# )\n",
    "# model.compile(optimizer='adam',\n",
    "#             loss=\"categorical_crossentropy\",\n",
    "#             metrics=['accuracy'])\n",
    "# model.fit([tf.convert_to_tensor(train_data['article_emb_inputid']),\n",
    "#            tf.convert_to_tensor(train_data['user_beh_emb_inputid']),\n",
    "#            tf.convert_to_tensor(train_data['user_emb_inputid'])],\n",
    "#           label_trans, epochs=50, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "#### set these features as fixed input \n",
    "article_input = tf.keras.Input(\n",
    "    shape=(768,), name=\"article\"\n",
    ")  \n",
    "userbeh_input = tf.keras.Input(shape=(768,), name=\"userbehaviour\")  \n",
    "\n",
    "user_input = tf.keras.Input(\n",
    "    shape=(768,), name=\"user\"\n",
    ") \n",
    "x = tf.keras.layers.concatenate([article_input, userbeh_input, user_input])\n",
    "\n",
    "hidden_layer = tf.keras.layers.Dense(256, activation='relu', name=\"relu\")(x)\n",
    "priority_pred = tf.keras.layers.Dense(187, activation= 'softmax', name=\"priority\")(hidden_layer)\n",
    "\n",
    "model = tf.keras.Model(\n",
    "    inputs=[article_input, userbeh_input, user_input],\n",
    "    outputs=priority_pred,\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(optimizer='adam',\n",
    "            loss=\"categorical_crossentropy\",\n",
    "            metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 568 samples\n",
      "Epoch 1/50\n",
      "568/568 [==============================] - 0s 571us/sample - loss: 3.7320 - accuracy: 0.4718\n",
      "Epoch 2/50\n",
      "568/568 [==============================] - 0s 83us/sample - loss: 3.0865 - accuracy: 0.4947\n",
      "Epoch 3/50\n",
      "568/568 [==============================] - 0s 79us/sample - loss: 2.7668 - accuracy: 0.4982\n",
      "Epoch 4/50\n",
      "568/568 [==============================] - 0s 81us/sample - loss: 2.4772 - accuracy: 0.5317\n",
      "Epoch 5/50\n",
      "568/568 [==============================] - 0s 82us/sample - loss: 2.1959 - accuracy: 0.5493\n",
      "Epoch 6/50\n",
      "568/568 [==============================] - 0s 79us/sample - loss: 1.9373 - accuracy: 0.5687\n",
      "Epoch 7/50\n",
      "568/568 [==============================] - 0s 81us/sample - loss: 1.7172 - accuracy: 0.5951\n",
      "Epoch 8/50\n",
      "568/568 [==============================] - 0s 84us/sample - loss: 1.4470 - accuracy: 0.6215\n",
      "Epoch 9/50\n",
      "568/568 [==============================] - 0s 86us/sample - loss: 1.1957 - accuracy: 0.6655\n",
      "Epoch 10/50\n",
      "568/568 [==============================] - 0s 89us/sample - loss: 0.9951 - accuracy: 0.7077\n",
      "Epoch 11/50\n",
      "568/568 [==============================] - 0s 85us/sample - loss: 0.8220 - accuracy: 0.7570\n",
      "Epoch 12/50\n",
      "568/568 [==============================] - 0s 83us/sample - loss: 0.7021 - accuracy: 0.8204\n",
      "Epoch 13/50\n",
      "568/568 [==============================] - 0s 86us/sample - loss: 0.5773 - accuracy: 0.8680\n",
      "Epoch 14/50\n",
      "568/568 [==============================] - 0s 82us/sample - loss: 0.4861 - accuracy: 0.8838\n",
      "Epoch 15/50\n",
      "568/568 [==============================] - 0s 82us/sample - loss: 0.4210 - accuracy: 0.9067\n",
      "Epoch 16/50\n",
      "568/568 [==============================] - 0s 84us/sample - loss: 0.3388 - accuracy: 0.9401\n",
      "Epoch 17/50\n",
      "568/568 [==============================] - 0s 88us/sample - loss: 0.2905 - accuracy: 0.9507\n",
      "Epoch 18/50\n",
      "568/568 [==============================] - 0s 89us/sample - loss: 0.2518 - accuracy: 0.9613\n",
      "Epoch 19/50\n",
      "568/568 [==============================] - 0s 91us/sample - loss: 0.2319 - accuracy: 0.9630\n",
      "Epoch 20/50\n",
      "568/568 [==============================] - 0s 90us/sample - loss: 0.2131 - accuracy: 0.9683\n",
      "Epoch 21/50\n",
      "568/568 [==============================] - 0s 89us/sample - loss: 0.1899 - accuracy: 0.9824\n",
      "Epoch 22/50\n",
      "568/568 [==============================] - 0s 87us/sample - loss: 0.1695 - accuracy: 0.9771\n",
      "Epoch 23/50\n",
      "568/568 [==============================] - 0s 81us/sample - loss: 0.1507 - accuracy: 0.9789\n",
      "Epoch 24/50\n",
      "568/568 [==============================] - 0s 83us/sample - loss: 0.1417 - accuracy: 0.9789\n",
      "Epoch 25/50\n",
      "568/568 [==============================] - 0s 83us/sample - loss: 0.1308 - accuracy: 0.9806\n",
      "Epoch 26/50\n",
      "568/568 [==============================] - 0s 83us/sample - loss: 0.1293 - accuracy: 0.9806\n",
      "Epoch 27/50\n",
      "568/568 [==============================] - 0s 85us/sample - loss: 0.1172 - accuracy: 0.9806\n",
      "Epoch 28/50\n",
      "568/568 [==============================] - 0s 79us/sample - loss: 0.1127 - accuracy: 0.9842\n",
      "Epoch 29/50\n",
      "568/568 [==============================] - 0s 82us/sample - loss: 0.1040 - accuracy: 0.9806\n",
      "Epoch 30/50\n",
      "568/568 [==============================] - 0s 79us/sample - loss: 0.0947 - accuracy: 0.9824\n",
      "Epoch 31/50\n",
      "568/568 [==============================] - 0s 82us/sample - loss: 0.0901 - accuracy: 0.9842\n",
      "Epoch 32/50\n",
      "568/568 [==============================] - 0s 84us/sample - loss: 0.0937 - accuracy: 0.9842\n",
      "Epoch 33/50\n",
      "568/568 [==============================] - 0s 84us/sample - loss: 0.0858 - accuracy: 0.9824\n",
      "Epoch 34/50\n",
      "568/568 [==============================] - 0s 88us/sample - loss: 0.0805 - accuracy: 0.9859\n",
      "Epoch 35/50\n",
      "568/568 [==============================] - 0s 88us/sample - loss: 0.0851 - accuracy: 0.9806\n",
      "Epoch 36/50\n",
      "568/568 [==============================] - 0s 83us/sample - loss: 0.0738 - accuracy: 0.9824\n",
      "Epoch 37/50\n",
      "568/568 [==============================] - 0s 80us/sample - loss: 0.0790 - accuracy: 0.9806\n",
      "Epoch 38/50\n",
      "568/568 [==============================] - 0s 82us/sample - loss: 0.0753 - accuracy: 0.9842\n",
      "Epoch 39/50\n",
      "568/568 [==============================] - 0s 82us/sample - loss: 0.0710 - accuracy: 0.9859\n",
      "Epoch 40/50\n",
      "568/568 [==============================] - 0s 77us/sample - loss: 0.0754 - accuracy: 0.9824\n",
      "Epoch 41/50\n",
      "568/568 [==============================] - 0s 79us/sample - loss: 0.0709 - accuracy: 0.9789\n",
      "Epoch 42/50\n",
      "568/568 [==============================] - 0s 85us/sample - loss: 0.0661 - accuracy: 0.9824\n",
      "Epoch 43/50\n",
      "568/568 [==============================] - 0s 83us/sample - loss: 0.0630 - accuracy: 0.9842\n",
      "Epoch 44/50\n",
      "568/568 [==============================] - 0s 81us/sample - loss: 0.0573 - accuracy: 0.9859\n",
      "Epoch 45/50\n",
      "568/568 [==============================] - 0s 79us/sample - loss: 0.0537 - accuracy: 0.9877\n",
      "Epoch 46/50\n",
      "568/568 [==============================] - 0s 85us/sample - loss: 0.0501 - accuracy: 0.9877\n",
      "Epoch 47/50\n",
      "568/568 [==============================] - 0s 88us/sample - loss: 0.0644 - accuracy: 0.9824\n",
      "Epoch 48/50\n",
      "568/568 [==============================] - 0s 85us/sample - loss: 0.0541 - accuracy: 0.9859\n",
      "Epoch 49/50\n",
      "568/568 [==============================] - 0s 89us/sample - loss: 0.0495 - accuracy: 0.9824\n",
      "Epoch 50/50\n",
      "568/568 [==============================] - 0s 85us/sample - loss: 0.0482 - accuracy: 0.9877\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.keras.callbacks.History at 0x1a4aced160>"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.fit([tf.convert_to_tensor(train_data['article_emb_bert']),\n",
    "           tf.convert_to_tensor(train_data['user_beh_emb_bert']),\n",
    "           tf.convert_to_tensor(train_data['user_emb_bert'])],\n",
    "          label_trans, epochs=50, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TensorShape([568, 768])"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.convert_to_tensor(train_data['article_emb_bert']).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "dim_len = len(train_data['article_emb'].head(1).values.tolist()[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "article_emb_cols = ['article_emb_'+str(i) for i in range(0,dim_len)]\n",
    "user_beh_emb_cols = ['user_beh_emb_'+str(i) for i in range(0,dim_len)]\n",
    "user_emb_cols = ['user_emb'+str(i) for i in range(0,dim_len)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = pd.concat([pd.DataFrame(train_data.article_emb.tolist(),index= train_data.index,columns=article_emb_cols),\n",
    "                       pd.DataFrame(train_data.user_beh_emb.tolist(),index= train_data.index,columns=user_beh_emb_cols),\n",
    "                       pd.DataFrame(train_data.user_emb.tolist(),index= train_data.index,columns=user_emb_cols),\n",
    "                       train_data[['totalClapCount','example_age_square','example_age','label']]\n",
    "                      ],axis=1, sort=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(571, 2308)"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "train, test = train_test_split(train_data, test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1a52736518>"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYQAAAEKCAYAAAASByJ7AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi40LCBodHRwOi8vbWF0cGxvdGxpYi5vcmcv7US4rQAAIABJREFUeJzt3Xl8VNX9//HXZ5YkJEAIEPYgqyCgoiAKLlVQFqliXQrUWr9uaKvW37etVrvZ2q+2dtOqVYu4UG0L7lIEFcStqEAE2UXCIoQECBBZs83M+f1xLxpCIAMkmSzv5+Mxj8zce+6dcy/DvOeee+655pxDREQkkOgKiIhI3aBAEBERQIEgIiI+BYKIiAAKBBER8SkQREQEUCCIiIhPgSAiIoACQUREfKF4CpnZSOCvQBCY5Jz7fYX5ycA/gAHAdmCsc269mbUCXgROA55xzt1SbpkBwDNAE2AGcJur4rLp1q1buy5dusS3ZSIiwieffLLNOZcZT9kqA8HMgsDfgAuAXGCBmU1zzq0oV+w6oNA518PMxgH3A2OBYuCXQD//Ud5jwATgY7xAGAnMPFxdunTpQnZ2djzbJSIigJl9EW/ZeJqMBgE5zrm1zrlSYAowpkKZMcBk//mLwDAzM+fcXufcf/GCoXwF2wPNnXMf+UcF/wAuibfSIiJS/eIJhI7AxnKvc/1plZZxzkWAnUCrKtaZW8U6RUSkFsUTCFbJtIpt/fGUOaryZjbBzLLNLLugoOAwqxQRkWMRTyDkAlnlXncC8g5VxsxCQDqwo4p1dqpinQA45yY65wY65wZmZsZ1XkRERI5CPIGwAOhpZl3NLAkYB0yrUGYacLX//HJgzuF6DDnn8oHdZnaGmRnwPeC1I669iIhUmyp7GTnnImZ2C/AmXrfTp5xzy83sHiDbOTcNeBJ41sxy8I4Mxu1f3szWA82BJDO7BBju91D6Pl93O51JFT2MRESkZll9umPawIEDnbqdiojEz8w+cc4NjKesrlQWERFAgSAiIr64hq6Qhm3iJxOrfZ0TBkyo9nWKSM3SEYKIiAAKBBER8SkQREQEUCCIiIhPgSAiIoACQUREfAoEEREBFAgiIuJTIIiICKBAEBERnwJBREQABYKIiPgUCCIiAigQRETEp0AQERFAgSAiIj4FgoiIAAoEERHxKRBERARQIIiIiE+BICIigAJBRER8CgQREQEUCCIi4lMgiIgIoEAQERGfAkFERAAFgoiI+BQIIiICKBBERMSnQBARESDOQDCzkWa2ysxyzOzOSuYnm9lUf/48M+tSbt5d/vRVZjai3PT/NbPlZrbMzP5tZinVsUEiInJ0qgwEMwsCfwNGAX2A8WbWp0Kx64BC51wP4AHgfn/ZPsA4oC8wEnjUzIJm1hH4ITDQOdcPCPrlREQkQeI5QhgE5Djn1jrnSoEpwJgKZcYAk/3nLwLDzMz86VOccyXOuXVAjr8+gBDQxMxCQCqQd2ybIiIixyKeQOgIbCz3OtefVmkZ51wE2Am0OtSyzrlNwJ+ADUA+sNM599bRbICIiFSPeALBKpnm4ixT6XQzy8A7eugKdADSzOy7lb652QQzyzaz7IKCgjiqKyIiRyOeQMgFssq97sTBzTtflfGbgNKBHYdZ9nxgnXOuwDlXBrwMDKnszZ1zE51zA51zAzMzM+OoroiIHI14AmEB0NPMuppZEt7J32kVykwDrvafXw7Mcc45f/o4vxdSV6AnMB+vqegMM0v1zzUMA1Ye++aIiMjRClVVwDkXMbNbgDfxegM95Zxbbmb3ANnOuWnAk8CzZpaDd2Qwzl92uZk9D6wAIsDNzrkoMM/MXgQW+tMXAROrf/NERCReVQYCgHNuBjCjwrRflXteDFxxiGXvBe6tZPrdwN1HUlkREak5ulJZREQABYKIiPgUCCIiAigQRETEp0AQERFAgSAiIj4FgoiIAAoEERHxKRBERARQIIiIiE+BICIigAJBRER8CgQREQEUCCIi4lMgiIgIoEAQERGfAkFERAAFgoiI+BQIIiICKBBERMSnQBAREUCBICIiPgWCiIgACgQREfEpEEREBFAgiIiIT4EgIiKAAkFERHwKBBERARQIIiLiUyCIiAigQBAREZ8CQUREAAWCiIj44goEMxtpZqvMLMfM7qxkfrKZTfXnzzOzLuXm3eVPX2VmI8pNb2FmL5rZZ2a20swGV8cGiYjI0akyEMwsCPwNGAX0AcabWZ8Kxa4DCp1zPYAHgPv9ZfsA44C+wEjgUX99AH8F3nDO9QZOBlYe++aIiMjRiucIYRCQ45xb65wrBaYAYyqUGQNM9p+/CAwzM/OnT3HOlTjn1gE5wCAzaw6cAzwJ4Jwrdc59eeybIyIiRyueQOgIbCz3OtefVmkZ51wE2Am0Osyy3YAC4GkzW2Rmk8ws7ai2QEREqkU8gWCVTHNxljnU9BBwKvCYc+4UYC9w0LkJADObYGbZZpZdUFAQR3VFRORoxBMIuUBWudedgLxDlTGzEJAO7DjMsrlArnNunj/9RbyAOIhzbqJzbqBzbmBmZmYc1RURkaMRTyAsAHqaWVczS8I7STytQplpwNX+88uBOc45508f5/dC6gr0BOY75zYDG82sl7/MMGDFMW6LiIgcg1BVBZxzETO7BXgTCAJPOeeWm9k9QLZzbhreyeFnzSwH78hgnL/scjN7Hu/LPgLc7JyL+qu+FfinHzJrgWuqedtEROQIVBkIAM65GcCMCtN+Ve55MXDFIZa9F7i3kumfAgOPpLIiIlJzdKWyiIgACgQREfEpEEREBFAgiIiIT4EgIiKAAkFERHwKBBERARQIIiLiUyCIiAigQBAREZ8CQUREAAWCiIj4FAgiIgIoEERExKdAEBERQIEgIiI+BYKIiAAKBBER8SkQREQEiPOeytLAffB+9a/zk2pe34QJ1bxCEalIRwgiIgIoEATY60rIiWxlT6wk0VURkQRSk1EjtmzrMsa/NJ5lu5YBkEyIc5OPZ0RSH9ICyQmunYjUNgVCI/XWmre44oUrSAuncWlKf9oGmpNd9gVvlaxgadkm7mg6nCaWlOhqikgtUpNRIzR/03xG/2s0XVp0Yd718xiR3Jf+4SyuTz2LH6YOZXNsF4/v/YCIiya6qiJSixQIjUxxpJhrXruGtmlteffqd8lKzzpgfp9we77X5Aw+i25manF1dxUSkbpMgdDI3PPePawoWMETFz1BRpOMSssMTurGsKTevF+6mvWR7bVcQxFJFAVCI7Js6zLun3s/1/a/llE9Rx227EUpJ9HMUphanI1zrpZqKCKJpEBoRO774D5Sw6n84YI/VFm2iYX5Vkp/1ka3saBsfc1XTkQSToHQSOTsyGHq8ql8f+D3aZXaKq5lBoe70TnYkleLFxN1sRquoYgkmgKhkbj/v/cTDoT50eAfxb1MwIzRyf3Y7vayqGxjDdZOROoCBUIjkLsrl8mLJ3PdKdfRrmm7I1r2pFBH2gSaMat0pc4liDRwCoRGYNLCSURiEX4y5CdHvGzAAgxL6s366HbWRAtqoHYiUlcoEBq4aCzK058+zQXdL6BrRtejWseQpG6kWRKzSlZWc+1EpC6JKxDMbKSZrTKzHDO7s5L5yWY21Z8/z8y6lJt3lz99lZmNqLBc0MwWmdn0Y90QqdycdXPYsHMD1/a/9qjXkWQhzk7qweLIJr6M7avG2olIXVJlIJhZEPgbMAroA4w3sz4Vil0HFDrnegAPAPf7y/YBxgF9gZHAo/769rsN0M/OGvTUp0/RsklLLul9yTGtZ0i4Ow7Hx2XrqqlmIlLXxHOEMAjIcc6tdc6VAlOAMRXKjAEm+89fBIaZmfnTpzjnSpxz64Acf32YWSdgNDDp2DdDKrOjaAevrHyFK0+8kuTQsY1e2jbYnB7BTD4sXaOTyyINVDyB0BEo3+cw159WaRnnXATYCbSqYtkHgTuAw3ZwN7MJZpZtZtkFBTqpeSSmLptKSbSEa085+uai8oYkdWdLbDdro9uqZX0iUrfEEwhWybSKPxEPVabS6Wb2TWCrc67K0dOccxOdcwOdcwMzMzOrrq185fkVz3NC6xPo365/taxvQLgzyYSYW7qmWtYnInVLPIGQC5QfErMTkHeoMmYWAtKBHYdZ9kzgYjNbj9cENdTMnjuK+sshbN6zmfe/eJ8r+lxRbetMsTCnhjuTXfYFpS5SbesVkbohnkBYAPQ0s65mloR3knhahTLTgKv955cDc5zX0DwNGOf3QuoK9ATmO+fucs51cs518dc3xzn33WrYHvG9vPJlYi7GFX2rLxAATk/qSgkRlkUq/iYQkfquyjumOeciZnYL8CYQBJ5yzi03s3uAbOfcNOBJ4Fkzy8E7MhjnL7vczJ4HVgAR4GbndNeV2vDCihc4ofUJ9M3sW63rPT7YhmaWwoLSLzg13Lla1y0iiRXXLTSdczOAGRWm/arc82Kg0p+izrl7gXsPs+53gXfjqYfEZ39z0S/O/gVeZ6/qE7QAA8KdmVu6hmJXRoqFq3X9IpI4ulK5AXpl5Ss10ly032nh4ygjyuKy3BpZv4gkhgKhAXpt1Wv0bNmz2puL9usWzCTDUsku+6JG1i8iiaFAaGB2l+zmnfXvcHGvi6u9uWi/gBkDwp1ZHslnnyutkfcQkdqnQGhg3lrzFqXRUi46/qIafZ8B4eOIEmOJmo1EGgwFQgPzn8//Q0ZKBmd2PrNG36dLsBUtrAkLdeMckQZDgdCARGNRpn8+ndHHjyYUiKsD2VELmHFKuDPLI3kUu7IafS8RqR0KhAbko9yP2F60vcabi/Y7NZxFhBhLyzbVyvuJSM1SIDQg0z+fTjgQZkT3EVUXrgY9gpk0sxQWRdRsJNIQKBAakBmrZ3D2cWeTnpJeK+8XsACnhDuxrCxPYxuJNAAKhAYid1cuS7cuZVSPUbX6vqeEOlNChBWR/Fp9XxGpfjV75lFqzczVMwGOLBBWrYJXX2X0pNmk7i4iXBKhLCnEtqyWbOnSmjX9j6OkacphV9Er1JZUS2Jh2Qb6h7MOW1ZE6jYFQgMxM2cmWc2z6JNZ8e6mlVi2DH7yE3jzTQCSO2ZQ2K4FZckhkveV0mH1Fnp+sp4zpi1i1aBufDqsL3sz0ipdVdACnBzqxKdlG4m4KKED7pAqIvWJAqEBKI2WMnvtbMb3G3/4q5NLSuD22+Fvf4PmzeH3v4fvfIeXX7rroKIZ+V9y4nuf0fvjNfTMXsfHF5/KZ4N7QCXrPzWcxUdla/ksspl+4Yo30xOR+kLnEBqADzd+yO7S3YzqeZjmorw8OPdcePhh+P73IScHfvpTyKq8maewfQveH3cGU392EQWdW3HOC/MZ+cS7JBUdPFTFCaH2pBDSRWoi9ZwCoQGYuXom4UCYYV2HVV5g2TIYOBCWLoUXXoBHHoFWreJa956WTXn9pmHMvXQgHT/fzMUPvUVa4d4DyoQtyInhjnwaySXqDnuLbBGpwxQIDcDMnJmc1fksmiU3O3jmZ5/BsGFeU89HH8Hllx/5GwSM5Wf3YuaN59H0y31c8uCbZOR/eUCRU8Od2etKWB3depRbISKJpkCo5w7b3XT1ahg61AuDOXPgxBOP6b3yerbjtR8OB4PRj71N+tZdX83rF+pAEkEWlm04pvcQkcRRINRzb+S8AXDw+YMdO2DUKCgrg7ffhl69quX9Ctu3YPr3h2HOMfqxt2m6Yw8ASRaiX6gDn5blEnOuWt5LRGqXAqGem5kzk07NOx14M5xIBMaOhY0b4bXXoG/13ihnZ9t0Xr9pGOGSCBc+PofkvSUAnBLuzE5XxNpoQbW+n4jUDgVCPVYWLWP22tmM6jHqwO6mt98Os2fDY4/BkCE18t47OmbwxvXfoNmOvZw/+QMsGuPEcEdCBNTbSKSeUiDUYx9u/JBdJbsOPH/w8svw4INw661w7bU1+v5burXh/bGn03H1Fs56aQFNCHFCqD2Lyjbg1GwkUu8oEOqxmTkzCQVCDOvmdzfNzYXrr/e6mP7pT7VSh9WndWPR+X054aMcen+Uw6nhLHa4fbrfskg9pECox/Z3N22e3ByiUbjqKigthX/9C5KSaq0eC0adzMZe7RnySjbfKGhKAOOl4oW19v4iUj0UCPXUxp0bWbJlCRf2uNCb8Je/wLvvehed9exZu5UJGO9cOYSStGS+NXkeJ1gbXipepGYjkXpGgVBPzVg9A4DRx4/2Ri395S/hkkvg6qsTUp/iZim8fdWZNN+2h/GLysiJbmVpRHdSE6lPFAj11IycGXRp0YUTWvaC666D1FR49NFKB5+rLZu7tyX7wpO4YdYOAk7NRiL1jQKhHiqOFDN77WxG9xyNPfoozJ3r9Sxq3z7RVePToX0pzmrP2RscL+2en+jqiMgRUCDUQ++tf499Zfu4sOUg+NnPYORI74RyXeCfT7hsbROWWwGr9qxPdI1EJE4KhHpoxuoZNAk14byHp3tDUzzySEKbiioqaZrCpadfjTmYuvDZRFdHROKkQKhnnHO8vvp1hjY/mSb/egHuuAO6d090tQ7SsfspfGN3K55rmYubPy/R1RGROCgQ6pnPt3/OmsI1XDj7CzjuOLjzzkRX6ZCu6jiK1a1g/pznoEDjG4nUdQqEeub11a8DMPq9fHjgAa93UR11eepAUgjxbL8oPPmkd/GciNRZCoR6ZsbyV+m7LcBxZ4zwrjuow5oHmjAmpT9T+oco3bAOpk9PdJVE5DDiCgQzG2lmq8wsx8wOaqMws2Qzm+rPn2dmXcrNu8ufvsrMRvjTsszsHTNbaWbLzey26tqghmx3yW7ez53LhauBhx6qUyeSD+W7TU5ne7CEN77ZG2bOhM8/T3SVROQQqgwEMwsCfwNGAX2A8WbWp0Kx64BC51wP4AHgfn/ZPsA4oC8wEnjUX18E+LFz7gTgDODmStYpFcya8TBlFmP0qePg+OMTXZ24jEjuS5tAM54aFIbMTHjqKdi7t+oFRaTWxXOEMAjIcc6tdc6VAlOAMRXKjAEm+89fBIaZN0D/GGCKc67EObcOyAEGOefynXMLAZxzu4GVQMdj35wGLBplxn/+QnqJMeSnjyS6NnELW5BrmgxhetlyNl33bdi1C557DjTOkUidE08gdATK3/Ekl4O/vL8q45yLADuBVvEs6zcvnQKob+JhxP7+ODNabmd469MJp2ckujpH5IbUs4kS4+lWG2DMGFi40Lu6WkTqlHgCobKG6oo/7w5V5rDLmllT4CXg/znndlVSFjObYGbZZpZd0Fi7Lm7bxrxH7iK/GYw5/+ZE1+aIdQ9lcn7SCTxR9F+i5w+DE06AqVMhLy/RVRORcuIJhFwgq9zrTkDF/8lflTGzEJAO7DjcsmYWxguDfzrnXj7UmzvnJjrnBjrnBmZmZsZR3QboZz/jlaw9hCzE6OO/mejaHJUJqWezIbqDt8o+g2uugZQUePxxKCpKdNVExBdPICwAeppZVzNLwjtJPK1CmWnA/nGXLwfmOG8w/GnAOL8XUlegJzDfP7/wJLDSOfeX6tiQBmvBAtykJ3jl9HSGdRtGi5QWia7RURmTcjJtA815eN87kJ4ON9zgXaz2zDMQiyW6eiJCHIHgnxO4BXgT7+Tv88655WZ2j5ld7Bd7EmhlZjnAj4A7/WWXA88DK4A3gJudc1HgTOAqYKiZfeo/Lqzmbav/YjG4+WaW925FTvBLvtX7W4mu0VFLshC3pJ7LzJJlLC/L83pJXXYZfPopvPVWoqsnIkAonkLOuRnAjArTflXueTFwxSGWvRe4t8K0/1L5+QUp76mnYMECXnnoMmzHy4zpXbFzV/3y/bRv8Lu9b/DnvbN4qsXVMGwYrF8Pr77qDcNxwgmJrqJIo6YrleuqHTu8cYrOPptXktYyOGsw7Zq2S3StjkmrQFOubTKE54rmkR/d6V1Y993vevdxeOIJ2L490VUUadQUCHXVz38OhYXk/P4OFm1exKW9L010jarF/6adT5QYf937tjchJQVuuskb5+jxx6G0NLEVFGnEFAh10fz58Pe/ww9/yJTIpwB8u++3E1yp6tEtlMnYlIE8vO8dNkd3ehPbtoVrr4WNG2HSJJ1kFkkQBUJdE4l4v5jbt4d77mHKsimc1fksstKzql62nrin2cWUugj/t6fcaamTT4axY2HxYu8aBV3JLFLrFAh1zaOPwqJF8OCDLN23nuUFyxnfb3yia1WteoTacH3qWfx93/usjZS72PC88+CCC+Ddd72B8ESkVikQ6pK8PPjFL2DECLj8cqYsm0LQglze5/JE16za/bLpaMIE+dnuVw+ccemlMGgQvPYazJ6dmMqJNFIKhLrkRz/yTqo+8ggOmLJ8CsO6DaNNWptE16zadQi24I6mI5hanM0bxcu+nhEIwP/8D5x6KrzwArz3XsLqKNLYxHUdgtSCt97y2s5/8xvo0YMPN8xlbeFafnnOLxNdsxpzV9ORTC3K5qZd/2JZ0q9oGkjxZgSDcN11UFYG//oXFBfDhAmJrWx9N3FiomtQNf0bJ5yOEOqC4mK4+Wbo2RN++lMAnlr0FE2TmjbI5qL9ki3MpBZX8UV0Oz/f/dqBM0Mh7+T6aafByy/DHXeo95FIDVMg1AW//jXk5HgnlJOT2V2ym6nLpzK271iaJjVNdO1q1JlJPbg19Twe2jeHF4s+OXBmKOR1R/3GN+CPf4Rvfxv27ElMRRuiWMxrolTQik9NRom2YIH3ZXfddXD++QC8sOIF9pbt5dpTrk1w5WrHH5tfxoKy9fzPzsn0DrWjX7jcLTMCARg/Hi6+GG6/3bsF5yuvQPfuiatwXVZUBJ99BsuXw9q1kJ8Pmzd73Xl37fKORsvKvO7N5YMgORmaNPEuFGzSBDIyoFUr7xqRjh2hQwdvnjRoCoREKinxhoJu3x7+/OevJj+56El6t+7N4E6DE1i52pNsYV7KuIkB2+7l4sJHebfVj+kcbPl1ATPvhPuJJ3rXKvTvDw8+6B091IP7SteIkhJYtcr74l+2zPu7PwTKf9G3auV9vlJSvFuYpqZCOOwdfe1/lJV5QVJc7D327YNNm2DJEi84wNvPHTpAt27emFN9+njBIQ2KAiGRfvMb7z/x6697Q0IDKwpW8OHGD/nD+X/AGtGXXYdgC6Zl3MwFOx7knO1/4u2W/0v30Nf3v5j4yURoCWnP3sG5v36GjtdfzxeT/8qHPxnH7o6tj+o9Jwyo+ycxJ338KOkbtpKxNo+MNXm0XJNHxto8mucWEIh6X/yxYICdndtS2K09heddyI7uHSjs1p5dWW2Ihb3/4nvfm8Xa6DY2Rb9kc2wX22N7+NIVsdeVUOIiRIkRJECYIE0DyTS3JmRaFp1LUjhpR4jT10foteZL2iz4mOQPPiAWMPK7tWFDn44MHjAG2tXvcbbEY64eXRE6cOBAl52dnehqVI/334dzz/WOEJ588qvJN02/icmLJ7Ph/20gM612bgg08cHvVvs6J6Sec1TLLSzbwAXbHyTJgjzX4lqGJXsjoE4cUK5QLMaJ/57DwMdew2Ixllw1nMVXDaesaQJ/sX7w/lFvM+CN5VRQ4F2LsmmT19STl0dsy2YCMe//aMyMXa2bUtiuBYXt0ilsl86OdunsbNOcWCh4wOp2x4pZFskjJ7KVNdEC8mNf35CwhTWhdaApLQKppFkSyYQJmhHDUeqi7HHF7IwVsTW2my/d1zcw6hhowQnBtgzb1pwLluyh64o8Wub7w4906QJDhnidAFJTj24fqJdRjTCzT5xzA+Mqq0BIgC+/9IZqCIe9+wE09U4cb9+3nawHsrjyxCt54uInaq06dSkQAJaX5XFZ4eOsim7hf9OG8cumo3nhtLSDyqVtKWTQwy/T8435lDRLZdm481g2diglLSo/Ee+co7C4kG37trGzZCe7S3azs2Qne0v3EnP+r20Xw8xIDafSLKkZzZKb0TypOZlpmWSmZhIMBCtdd9yBUFoKW7Z47fr72/c3b/amlW+ead0aOnRgUWbkqy//L9ukEw1X/v7OObbEdrE4ksvisk2sjRbggFRLonuwNd2CmfQIZZIVzKCJJVVdT1+RK2VDdAdrIttYFdlCTnQrEWK0tFQGhrvwjaI23PVZa+zDj7wgC4fhlFPgrLO8e14cyVGuAqFGHEkgqMmotjkHP/iB959n7tyvwgC8ZpGiSBG3nXFbAiuYeH3DHViY+Qt+sutFHtj7NpP2zWXwynM4rcNpZDXP+qopbW/bDN75v+tYNn4Y/Z+ZyYAnXqf/M2+y8rwT+WBEb5Z2SyOvuIDNezaTvyefLXu2UBItOeC9AhagaVJTAgQOaKLbU7qHsljZAWVDgRBt0trQoVkHsppn0S2jG8elH0dyKPnADYjFvNDftu3AL/z8fG9Y8/0/wvZ/8bdrB337em30HTp4bf5J3pf2gn3vH3I/RV2MNdEClpRtYnEkl62x3QB0DrRkdPKJnBzuRKdABoFjaHpsYkn0CrWjV6gdF9KPIlfG4rKNLCj7gtmlK3kruIJX+3fhx0NGcVlBa0JzP/IGZ5w/H7KyYPhwGDDAu7ZE6jwdIdS2iRPhxhvht7/1hqnwlUXL6PLXLvTJ7MOsq2bVbpXq2BFCeYvLNnLfnpm8ULwQh6N5cnM6NutIZlomKcEUAoEApZFS9pXtY1dhPtt2bmZr+MAv/baxVNqntqFVZhfapncgMy2T9OR00lPSSQ2nErDKe18XR4rZXbKbXSW72Lp3K3l78sjfnU/+rk1sK94BQACjZySdU/IdozelMTinmG7rvsSi5U7shsPel/7+R/v23t82bbx5hzGxQiDsihWzIpLHskgeKyL57HWlhAjQK9SWk0OdOCnciYzAUTbZHKE9sWIWlH3BJ2UbWB3dSudgS25LHcr1oUE0z14Ks2Z5YdiypXczpLPOOnxPJR0h1Ag1GdVV2dlw5pkwdKh3Ijnw9RfRM58+wzWvXcP08dMZffzoWq1WXQ6E/f7UbxfLti5j5baVbN2zlYJ9BZRGS4m6KMnBZFLDqWQ0ySAzNZO2Ka3plVfCqUu2cfZ762m/0fvydmbsbt+SXZ3asC8znaKWzShNa0IsFCQWCuKCAWKhIMGSMsL7igkXlRDeW0LS3iKa7NhF6rZdpBZ8SfLPhNDqAAAROUlEQVSeIralwsedvMdHnWB+J9jjt8S0LgtzemkmpweOY1DTXvTNOJ6OoZZH3EnAOcf9e99gXXQ76yLb+Dy6hQ3RHTigmaXQN9Sek8Od6BNqT4odPlhqUsw5lkY2MbtkJZ9Ht5JmyYxK7su5oZ50X7mZk95dSYc1WylOTWLxeX1YfvbxRJIrqe/Z3memPpzsr08UCHXR9u3eobNzsHCh1x3QVxYto/ffepOenM4nEz6p9d5F9SEQDjipfIRStxbSZvl6Wn2+kfQNW2m+sYAmO3bRZMcuQqWRQy4XSQ5TmpZCWWoKRS2bs691Ovta+38zW/iv09nTriXFi+YzJKkHH5WuZV7ZOuaVrWNFJP+rdTWzFPqE2tM71I5OwRa0C6TT3FJI9dvzI8TYGStiW2wPG2OFrI0UsCSyiS3+yeAQAY4LtqJvqD39wh3ICrQ8pqagmrIuso1pJUtYEcknw1L5ZsqJDA53o92GHQx4cymdV+ZR1DSZT4f1ZcWQnkSTyrVaKxBqhM4h1DWlpd4N5fPz4b//PSAMAP6x+B+sLVzLf8b/p1F1NT0iHxy6Lb0q+4D1IVjfpwX0aQEc781wDos5AtEYga/+xoiEgkSSQ7jA4S7kj0BkO2zeDpvXErAAJ4W9Jpsb8b7YdsaKWFi2gZWRfFZE8lkZyWdWyUo2x3YS49A/xFpZGt1CmYxI7kOZi9Il2IpOwRaErO63w3cNtea20FBWRbbwSvGnPFs0j1klKxnbYSAFE86jzfoCBs5YwuDXFnLSOyv59Py+rBzc46BeUpIYCoSa5pzXNvree/DPf3rd8sopjZby2/d/y2kdTmN0z9ptKmr0zHBBIxoMEK2B1acHmnBeci/OS+51wPSoi7E9tofdroR9rhQDQhakuaXQKpBGcrnmn4rnEOqLXqG2/DRtOIsjubxYvJC/7ptD/1AWV3Q+la0/GEb7nC0MnLmYM1/O5uQ5K1g4/ERWDT4Tp2BIKAVCTbvvPpg82Ruv6DvfOWj2Ywse44udX/DY6Md0dNBIBC1Am2BzGt6g5gcyM/qHs+gb6sCskpXMLFnGr3fnMSK5DyO69yH/lgvo+PlmBs5czDnPz+Pkj9aRfeNFcMr1B5xfk9qjQKhJjz7q9SS68kr41a8Omr15z2Z+9e6vGN59OCN7jExABUVqXtiCXJjSjzOSuvJS8SKmlyzlo9K1XNFkAO74Tmw6vh2dl2/itPdyGPaLJ+H5BXDvvTB6dOMdmiRBFMM1ZfJkb0jriy+Gp5+u9IN9x6w7KCor4uFRD+voQBq8loE0bkg9ix+lDSPZQjy+730e2vcOW2K72dCvEy/96xe8/X/Xwd69cNFFXjdV3SCpVikQasLTT3sDr11wgXfTm0r6mr+3/j2eXfIstw+5neNbHZ+ASookRq9QO37R9EK+nTKAtZFt3LPndV4p/pSSWBlrRg6ClSvh73+HL77whncZMQI++aTK9cqxU7fT6vanP3nDNA8f7t3YJe3gIRcKiwrp//f+hAIhlty0hLSkg8vUpnrR7bSenlyVw9sZK+Ll4kV8XLaOjJQMvt3321+fTysq8ppdf/c7r9v2ZZd5A0L27ZvoatcrR9LtVEcI1SUS8YZovv1270Yu//lPpWHgnOOG/9xA3u48/n3ZvxMeBiKJlB5owjWpQ7g97QLSwmn8/ZO/M+K5EazatsobXvvHP/aG9L77bnjzTejXDy69VEcMNUSBUB22b4dRo+CBB+DWW737ACdVPoDYowse5aWVL3Hv0HsZ1HFQLVdUpG7qEWrDz87+GWP7jmXepnmc+NiJ3Dn7TnaX7Ibmzb1eeuvXe50z3nkHBg6EkSO963qk2igQjtXcud6H8/33vWGsH3rokAN5vfbZa/zwjR/yzeO/yU+G/KSWKypStwUDQYZ2Hcrnt3zOd078DvfPvZ/uD3Xn4XkPUxot9S7o/M1vvHMLv/udd8X/2WfDGWd4P8JKSxO9CfWeziEcrZIS78N5//1w3HHeB/KMMw5ZfO6GuZz/7Pmc1PYk5nxvTp1qKqqJcwgiR6zC0BXzcudx59t38u76d+mW0Y3fnvdbxvUb9/VghPv2wVNPeT/CVq/2Bg286SZv8Mi2bRO1FXWOziHUtBkzvNs5/u533g1uFi8+bBi8/vnrDH9uOFnNs5g+fnqdCgORuur0Tqcz53tzmHnlTJolNePKl6+k36P9eHrR05RESrwb8dxyi3cP6RkzvHuM3H23N+z2pZfCq6/qqOEIKRCOxLx53rmC/RfMvPEGTJoEzZpVWtw5x2MLHmPMlDH0bt2bD675oNbugibSEJgZI3uMZOGNC/n3Zf8mKZjEtdOupdtD3fjD3D+ws3ind1XzqFEwc6YXDrfeCh9+CN/6lnd/iVtvhY8/PvBe01IpBUJVolHv18eIEd5RwIIF8Mc/wtKl3rRDKNhbwCVTL+EHM37ABd0v4N2r36VtUx3GihyNgAUY128ci25cxJvffZM+mX346eyf0v7P7bnqlat4e+3b3l3vevWCP/8ZcnO9IebPPx+eeAIGD4aOHeGGG2DaNK+5SQ6ioSsOZeVKeP55eOYZr3dD27be+YLvf/+QRwTg3VTlkfmPcN8H97G3bC9/Gf4XbjvjtkPehEVE4mdmDO8+nOHdh7MwfyFPfPIE/172b55b8hyd0zvznX7f4aJeF3F6x9MJXnghXHihd/e611/3guD5572j+pQUOOcc73H22TBo0OFv3tNIxHVS2cxGAn8FgsAk59zvK8xPBv4BDAC2A2Odc+v9eXcB1wFR4IfOuTfjWWdlavSkcmGhd1g5a5bX33nFCq9Z6NxzvRNVl1xyyK6kABt2bmDSwklMWjiJ/D35jOwxkj9d8Cf6tqn7F9HopLLUCUd5P4SisiKmrZrG058+zey1s4m6KC2btGRkj5GM7D6SIVlD6JbRzbvYrbQUPvjAC4c5c2DZMm8lSUneSMSnnQYnneSdj+jTp0GERLXeIMfMgsDnwAVALrAAGO+cW1GuzA+Ak5xzN5nZOOBbzrmxZtYH+DcwCOgAzOarwegPv87KVEsg7NkDGzbAunXeDe4XLoRFi7zXAMnJ3i+GMWO8KyPbt690NbtKdjF/03zmbpjL9NXTyc7LxjBG9RzFjwf/mKFdhx5bPWuRAkHqhGq4QU5hUSFvrXmLGTkzmLl6JgX7CgBo2aQlp3U4jdM6nMYJmSfQs2VPerbqSYt9Ma/r+AcfeI/Fi70rpMHrPn788dCjB3TrBl27fv1o187rBlsPRmWt7hvkDAJynHNr/ZVPAcYA5b+8xwC/9p+/CDxi3mhtY4ApzrkSYJ2Z5fjrI451Vo9YzDu59MUXXhAUFh44v0cP71fBjTd6dzQ780zvCsnDWLplKSc/fjLOv8nJ6R1P576h9zGu3zi6ZnSt9k0QkfhkNMlgbL+xjO03lpiLsXTLUuZvmu898uZz33/v8841+FqntqZT8060Pakt7Yb0om3qWWTsc6RtKaRpXgFpGzaTtmkJaW/NImV3MQ6IGTgDFwjgWrYg2jKDXa2asjMtxK7UIF82MfJSysgNF3FN2ll8q9WZ3nhmoZD3CAYPfB6JeEcuZWUH/i0t9Qb627XLK3v33TW+/+IJhI7AxnKvc4HTD1XGORcxs51AK3/6xxWW7eg/r2qd1SMQgN27va5oZ54JnTt//ejXD9LTj3iVvVv35u5v3M0Znc7g9E6n0yKlRQ1UXESORcACnNzuZE5udzI3DLgB8M7xrS1cy+rtq1m9YzWrt68mb08eW/ZsYUXBCrbs3eJdBAdel5su/qNSMWCH/zhQq33QqQD2frgSljxxbBtiBl261JlAqGxc5ortTIcqc6jplR1nVdp2ZWYTgP3HkHvMbNUh6nmkWgPbqmld9Z32xde0Lw5Ui/vjnwDcyI2183ZHLu59sd1/XOU/jolzXpP20Q+Rf1y8BeMJhFwgq9zrTkDeIcrkmlkISMeLzcMtW9U6AXDOTQQmxlHPI2Jm2fG2qzV02hdf0744kPbH1xrDvojnjMgCoKeZdTWzJGAcMK1CmWnA1f7zy4E5zjtbPQ0YZ2bJZtYV6AnMj3OdIiJSi6o8QvDPCdwCvInXRfQp59xyM7sHyHbOTQOeBJ71TxrvwPuCxy/3PN7J4ghws3MuClDZOqt/80REJF71anC76mRmE/zmqEZP++Jr2hcH0v74WmPYF402EERE5EB1/6oKERGpFQ0+EMzsj2b2mZktMbNXzKxFuXl3mVmOma0ysxHlpo/0p+WY2Z2JqXntaEzbCmBmWWb2jpmtNLPlZnabP72lmc0ys9X+3wx/upnZQ/7+WWJmpyZ2C6qfmQXNbJGZTfdfdzWzef6+mOp3/MDvHDLV3xfzzKxLIutdE8yshZm96H9nrDSzwY3ps9HgAwGYBfRzzp2EN1zGXQD+sBrjgL7ASOBR/z9GEPgbMAroA4z3yzY4jWlby4kAP3bOnQCcAdzsb/OdwNvOuZ7A2/5r8PZNT/8xAXis9qtc424DVpZ7fT/wgL8vCvHGIsP/W+ic6wE84JdraP4KvOGc6w2cjLdfGs1no8EHgnPuLedcxH/5Md41D1BuWA3n3Dpg/7AaXw3V4ZwrBfYPq9EQNaZtBcA5l++cW+g/3433H74j3nZP9otNBi7xn48B/uE8HwMtzKzyAa7qITPrBIwGJvmvDRiKNwQNHLwv9u+jF4FhfvkGwcyaA+fg9ZrEOVfqnPuSRvTZaPCBUMG1wEz/eWVDcnQ8zPSGqDFt60H8Jo9TgHlAW+dcPnihAbTxizX0ffQgcAfeOAzgDTnzZbkfUeW394AhaoD9Q9Q0FN2AAuBpvwltkpml0Yg+Gw0iEMxstpktq+QxplyZn+M1F/xz/6RKVnW44TYaosa0rQcws6bAS8D/c87tOlzRSqY1iH1kZt8EtjrnPik/uZKiLo55DUEIOBV4zDl3CrCXr5uHKtPg9keDuEGOc+78w803s6uBbwLD3Nf9bI95WI0GIJ5hSRocMwvjhcE/nXMv+5O3mFl751y+f9i/1Z/ekPfRmcDFZnYhkAI0xztiaGFmIf8ooPz2HmqImoYiF8h1zs3zX7+IFwiN5rPRII4QDse8G/H8FLjYOVf+vnkaVqNxbSvwVRv5k8BK59xfys0qP/zK1cBr5aZ/z+9Rcgawc3/zQX3nnLvLOdfJOdcF799+jnPuSuAdvCFo4OB9UdkQNQ2Cc24zsNHMevmThuGNstB4PhvOuQb9wDtZvBH41H88Xm7ez4E1wCpgVLnpF+L1SFoD/DzR21DD+6fRbKu/vWfhHdYvKfeZuBCvLfxtYLX/t6Vf3vB6Yq0BlgIDE70NNbRfzgWm+8+74f04ygFeAJL96Sn+6xx/frdE17sG9kN/INv/fLwKZDSmz4auVBYREaARNBmJiEh8FAgiIgIoEERExKdAEBERQIEgIiI+BYLIIZjZnirmdzGzZUe4zmfM7PKqS4rUPgWCiIgACgSRKplZUzN728wWmtnS8mNkASEzm+yPh/+imaX6ywwws/fM7BMze7O+j4IpjYMCQaRqxcC3nHOnAucBfy437HMvYKLz7rexC/iBP1bSw8DlzrkBwFPAvQmot8gRaRCD24nUMAPuM7Nz8IaJ7gi09edtdM7N9Z8/B/wQeAPoB8zycyMI1O8xbqRRUCCIVO1KIBMY4JwrM7P1eOP6wMHDHe8fQn25c25w7VVR5NipyUikaul49w0oM7PzgOPKzetsZvu/+McD/8UbLDFz/3QzC5tZ31qtschRUCCIVO2fwEAzy8Y7Wvis3LyVwNVmtgRoiXdzlVK84aHvN7PFeCOqDqnlOoscMY12KiIigI4QRETEp0AQERFAgSAiIj4FgoiIAAoEERHxKRBERARQIIiIiE+BICIiAPx/VjLe2I8llU8AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.distplot(test['label'], color='red')\n",
    "sns.distplot(train['label'], color='green')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:There are non-GPU devices in `tf.distribute.Strategy`, not using nccl allreduce.\n",
      "INFO:tensorflow:Using MirroredStrategy with devices ('/job:localhost/replica:0/task:0/device:CPU:0',)\n"
     ]
    }
   ],
   "source": [
    "strategy = tf.distribute.MirroredStrategy()\n",
    "\n",
    "BUFFER_SIZE = train.shape[0]\n",
    "\n",
    "BATCH_SIZE_PER_REPLICA = 64\n",
    "GLOBAL_BATCH_SIZE = BATCH_SIZE_PER_REPLICA * strategy.num_replicas_in_sync\n",
    "\n",
    "EPOCHS = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## recommend based on the most watched video content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(571, 2308)"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['article_emb_0', 'article_emb_1', 'article_emb_2', 'article_emb_3',\n",
       "       'article_emb_4', 'article_emb_5', 'article_emb_6', 'article_emb_7',\n",
       "       'article_emb_8', 'article_emb_9',\n",
       "       ...\n",
       "       'user_emb762', 'user_emb763', 'user_emb764', 'user_emb765',\n",
       "       'user_emb766', 'user_emb767', 'totalClapCount', 'example_age_square',\n",
       "       'example_age', 'label'],\n",
       "      dtype='object', length=2308)"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "item_num = len(articles['aid'].unique())\n",
    "embedding_size = 256\n",
    "\n",
    "initializer = tf.random_uniform_initializer(minval=-0.5 / embedding_size, maxval=0.5 / embedding_size)\n",
    "partitioner = tf.compat.v1.fixed_size_partitioner(num_shards=embedding_size)\n",
    "item_embedding = tf.compat.v1.get_variable(\"item_embedding\", \n",
    "                                 [item_num, embedding_size],\n",
    "                                 tf.float32, \n",
    "                                 initializer=initializer, \n",
    "                                 partitioner=partitioner)\n",
    "\n",
    "\n",
    "def mapped_function(train, label):\n",
    "    return train, tf.nn.embedding_lookup(item_embedding, label)\n",
    "\n",
    "train_dataset =tf.data.Dataset.from_tensor_slices((train.drop(columns={'label'}).values, train['label'].values))\\\n",
    "                                .map(mapped_function)\\\n",
    "                                .shuffle(BUFFER_SIZE)\\\n",
    "                                .batch(GLOBAL_BATCH_SIZE)\\\n",
    "                                .prefetch(buffer_size =tf.data.experimental.AUTOTUNE)\n",
    "\n",
    "test_dataset = tf.data.Dataset.from_tensor_slices((test.drop(columns={'label'}).values, test['label'].values))\\\n",
    "                                .map(mapped_function)\\\n",
    "                                .batch(GLOBAL_BATCH_SIZE)\\\n",
    "                                .prefetch(buffer_size =tf.data.experimental.AUTOTUNE)\n",
    "\n",
    "# train_dist_dataset = strategy.experimental_distribute_dataset(train_dataset)\n",
    "# test_dist_dataset = strategy.experimental_distribute_dataset(test_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import OneHotEncoder\n",
    "enc = OneHotEncoder(handle_unknown='ignore').fit(train_data[['label']].values)\n",
    "\n",
    "train_dataset =tf.data.Dataset.from_tensor_slices((train.drop(columns={'label'}).values, enc.transform(train[['label']].values).toarray()))\\\n",
    "                                .shuffle(BUFFER_SIZE)\\\n",
    "                                .batch(GLOBAL_BATCH_SIZE)\\\n",
    "                                .prefetch(buffer_size =tf.data.experimental.AUTOTUNE)\n",
    "\n",
    "test_dataset = tf.data.Dataset.from_tensor_slices((test.drop(columns={'label'}).values, enc.transform(test[['label']].values).toarray()))\\\n",
    "                                .batch(GLOBAL_BATCH_SIZE)\\\n",
    "                                .prefetch(buffer_size =tf.data.experimental.AUTOTUNE)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(\n",
      "[[1. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [1. 0. 0. ... 0. 0. 0.]\n",
      " ...\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [1. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]], shape=(64, 187), dtype=float64)\n"
     ]
    }
   ],
   "source": [
    "for x,y in train_dataset.take(1):\n",
    "    print(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train for 9 steps, validate for 3 steps\n",
      "Epoch 1/50\n",
      "9/9 [==============================] - 1s 89ms/step - loss: 3.5859 - accuracy: 0.5018 - val_loss: 2.7330 - val_accuracy: 0.6496\n",
      "Epoch 2/50\n",
      "9/9 [==============================] - 0s 33ms/step - loss: 2.9317 - accuracy: 0.5607 - val_loss: 3.1179 - val_accuracy: 0.6496\n",
      "Epoch 3/50\n",
      "9/9 [==============================] - 0s 34ms/step - loss: 2.8668 - accuracy: 0.5607 - val_loss: 2.9151 - val_accuracy: 0.6496\n",
      "Epoch 4/50\n",
      "9/9 [==============================] - 0s 37ms/step - loss: 2.6779 - accuracy: 0.5607 - val_loss: 2.9388 - val_accuracy: 0.6496\n",
      "Epoch 5/50\n",
      "9/9 [==============================] - 0s 36ms/step - loss: 2.5724 - accuracy: 0.5607 - val_loss: 2.9145 - val_accuracy: 0.6496\n",
      "Epoch 6/50\n",
      "9/9 [==============================] - 0s 35ms/step - loss: 2.2681 - accuracy: 0.5625 - val_loss: 2.9996 - val_accuracy: 0.6496\n",
      "Epoch 7/50\n",
      "9/9 [==============================] - 0s 34ms/step - loss: 2.0493 - accuracy: 0.5846 - val_loss: 3.2214 - val_accuracy: 0.6350\n",
      "Epoch 8/50\n",
      "9/9 [==============================] - 0s 36ms/step - loss: 1.7802 - accuracy: 0.6140 - val_loss: 3.2859 - val_accuracy: 0.6569\n",
      "Epoch 9/50\n",
      "9/9 [==============================] - 0s 34ms/step - loss: 1.5731 - accuracy: 0.6415 - val_loss: 3.4829 - val_accuracy: 0.6715\n",
      "Epoch 10/50\n",
      "9/9 [==============================] - 0s 34ms/step - loss: 1.2814 - accuracy: 0.6673 - val_loss: 3.7544 - val_accuracy: 0.6496\n",
      "Epoch 11/50\n",
      "9/9 [==============================] - 0s 35ms/step - loss: 1.0680 - accuracy: 0.7206 - val_loss: 4.5432 - val_accuracy: 0.5620\n",
      "Epoch 12/50\n",
      "9/9 [==============================] - 0s 34ms/step - loss: 0.9042 - accuracy: 0.7555 - val_loss: 4.3586 - val_accuracy: 0.6496\n",
      "Epoch 13/50\n",
      "9/9 [==============================] - 0s 34ms/step - loss: 0.6624 - accuracy: 0.8033 - val_loss: 4.7694 - val_accuracy: 0.5912\n",
      "Epoch 14/50\n",
      "9/9 [==============================] - 0s 37ms/step - loss: 0.5475 - accuracy: 0.8180 - val_loss: 5.2154 - val_accuracy: 0.5839\n",
      "Epoch 15/50\n",
      "9/9 [==============================] - 0s 34ms/step - loss: 0.4514 - accuracy: 0.8640 - val_loss: 5.4870 - val_accuracy: 0.6131\n",
      "Epoch 16/50\n",
      "9/9 [==============================] - 0s 35ms/step - loss: 0.3552 - accuracy: 0.8842 - val_loss: 5.3623 - val_accuracy: 0.5620\n",
      "Epoch 17/50\n",
      "9/9 [==============================] - 0s 34ms/step - loss: 0.2751 - accuracy: 0.9191 - val_loss: 6.1142 - val_accuracy: 0.5985\n",
      "Epoch 18/50\n",
      "9/9 [==============================] - 0s 37ms/step - loss: 0.2456 - accuracy: 0.9393 - val_loss: 6.1018 - val_accuracy: 0.5766\n",
      "Epoch 19/50\n",
      "9/9 [==============================] - 0s 36ms/step - loss: 0.2292 - accuracy: 0.9412 - val_loss: 6.0662 - val_accuracy: 0.6131\n",
      "Epoch 20/50\n",
      "9/9 [==============================] - 0s 35ms/step - loss: 0.1474 - accuracy: 0.9632 - val_loss: 6.1147 - val_accuracy: 0.5985\n",
      "Epoch 21/50\n",
      "9/9 [==============================] - 0s 33ms/step - loss: 0.1482 - accuracy: 0.9632 - val_loss: 6.2281 - val_accuracy: 0.5985\n",
      "Epoch 22/50\n",
      "9/9 [==============================] - 0s 36ms/step - loss: 0.1260 - accuracy: 0.9724 - val_loss: 6.4135 - val_accuracy: 0.6058\n",
      "Epoch 23/50\n",
      "9/9 [==============================] - 0s 36ms/step - loss: 0.0990 - accuracy: 0.9706 - val_loss: 6.7906 - val_accuracy: 0.5839\n",
      "Epoch 24/50\n",
      "9/9 [==============================] - 0s 36ms/step - loss: 0.0843 - accuracy: 0.9724 - val_loss: 7.0275 - val_accuracy: 0.6058\n",
      "Epoch 25/50\n",
      "9/9 [==============================] - 0s 37ms/step - loss: 0.0727 - accuracy: 0.9761 - val_loss: 7.1266 - val_accuracy: 0.6058\n",
      "Epoch 26/50\n",
      "9/9 [==============================] - 0s 37ms/step - loss: 0.0945 - accuracy: 0.9798 - val_loss: 7.1083 - val_accuracy: 0.5766\n",
      "Epoch 27/50\n",
      "9/9 [==============================] - 0s 36ms/step - loss: 0.0628 - accuracy: 0.9835 - val_loss: 6.9021 - val_accuracy: 0.5693\n",
      "Epoch 28/50\n",
      "9/9 [==============================] - 0s 34ms/step - loss: 0.0679 - accuracy: 0.9761 - val_loss: 7.1652 - val_accuracy: 0.5839\n",
      "Epoch 29/50\n",
      "9/9 [==============================] - 0s 35ms/step - loss: 0.0826 - accuracy: 0.9779 - val_loss: 7.0476 - val_accuracy: 0.5693\n",
      "Epoch 30/50\n",
      "9/9 [==============================] - 0s 35ms/step - loss: 0.0626 - accuracy: 0.9835 - val_loss: 7.2258 - val_accuracy: 0.6204\n",
      "Epoch 31/50\n",
      "9/9 [==============================] - 0s 35ms/step - loss: 0.0418 - accuracy: 0.9890 - val_loss: 7.1962 - val_accuracy: 0.6058\n",
      "Epoch 32/50\n",
      "9/9 [==============================] - 0s 32ms/step - loss: 0.0503 - accuracy: 0.9853 - val_loss: 7.4187 - val_accuracy: 0.5839\n",
      "Epoch 33/50\n",
      "9/9 [==============================] - 0s 36ms/step - loss: 0.0491 - accuracy: 0.9871 - val_loss: 7.4101 - val_accuracy: 0.5985\n",
      "Epoch 34/50\n",
      "9/9 [==============================] - 0s 34ms/step - loss: 0.0448 - accuracy: 0.9871 - val_loss: 7.1293 - val_accuracy: 0.5766\n",
      "Epoch 35/50\n",
      "9/9 [==============================] - 0s 34ms/step - loss: 0.0491 - accuracy: 0.9890 - val_loss: 7.4816 - val_accuracy: 0.6131\n",
      "Epoch 36/50\n",
      "9/9 [==============================] - 0s 37ms/step - loss: 0.0340 - accuracy: 0.9871 - val_loss: 7.3650 - val_accuracy: 0.5912\n",
      "Epoch 37/50\n",
      "9/9 [==============================] - 0s 31ms/step - loss: 0.0290 - accuracy: 0.9890 - val_loss: 7.3871 - val_accuracy: 0.5985\n",
      "Epoch 38/50\n",
      "9/9 [==============================] - 0s 33ms/step - loss: 0.0343 - accuracy: 0.9853 - val_loss: 7.6955 - val_accuracy: 0.5766\n",
      "Epoch 39/50\n",
      "9/9 [==============================] - 0s 37ms/step - loss: 0.0336 - accuracy: 0.9890 - val_loss: 7.8028 - val_accuracy: 0.6204\n",
      "Epoch 40/50\n",
      "9/9 [==============================] - 0s 36ms/step - loss: 0.0460 - accuracy: 0.9890 - val_loss: 7.3304 - val_accuracy: 0.5766\n",
      "Epoch 41/50\n",
      "9/9 [==============================] - 0s 34ms/step - loss: 0.0438 - accuracy: 0.9853 - val_loss: 7.6142 - val_accuracy: 0.6058\n",
      "Epoch 42/50\n",
      "9/9 [==============================] - 0s 36ms/step - loss: 0.0443 - accuracy: 0.9890 - val_loss: 7.3544 - val_accuracy: 0.5912\n",
      "Epoch 43/50\n",
      "9/9 [==============================] - 0s 35ms/step - loss: 0.0277 - accuracy: 0.9908 - val_loss: 7.2168 - val_accuracy: 0.6131\n",
      "Epoch 44/50\n",
      "9/9 [==============================] - 0s 39ms/step - loss: 0.0263 - accuracy: 0.9908 - val_loss: 7.3460 - val_accuracy: 0.5985\n",
      "Epoch 45/50\n",
      "9/9 [==============================] - 0s 35ms/step - loss: 0.0217 - accuracy: 0.9908 - val_loss: 7.6405 - val_accuracy: 0.5985\n",
      "Epoch 46/50\n",
      "9/9 [==============================] - 0s 38ms/step - loss: 0.0264 - accuracy: 0.9890 - val_loss: 7.8215 - val_accuracy: 0.5839\n",
      "Epoch 47/50\n",
      "9/9 [==============================] - 0s 37ms/step - loss: 0.0244 - accuracy: 0.9908 - val_loss: 7.8508 - val_accuracy: 0.5912\n",
      "Epoch 48/50\n",
      "9/9 [==============================] - 0s 37ms/step - loss: 0.0297 - accuracy: 0.9871 - val_loss: 7.7864 - val_accuracy: 0.5766\n",
      "Epoch 49/50\n",
      "9/9 [==============================] - 0s 36ms/step - loss: 0.0325 - accuracy: 0.9871 - val_loss: 7.9261 - val_accuracy: 0.6058\n",
      "Epoch 50/50\n",
      "9/9 [==============================] - 0s 37ms/step - loss: 0.0329 - accuracy: 0.9908 - val_loss: 7.5064 - val_accuracy: 0.5766\n"
     ]
    }
   ],
   "source": [
    "\n",
    "model = tf.keras.models.Sequential()\n",
    "\n",
    "model.add(tf.keras.layers.Flatten(input_shape=(train.shape[1]-1,)))\n",
    "\n",
    "model.add(tf.keras.layers.Dense(units=2048, activation='relu'))\n",
    "model.add(tf.keras.layers.Dense(units=1024, activation='relu'))\n",
    "model.add(tf.keras.layers.Dense(units=512, activation='relu'))\n",
    "model.add(tf.keras.layers.Dense(units=187, activation='softmax'))\n",
    "\n",
    "model.compile(optimizer='adam',\n",
    "            loss=\"categorical_crossentropy\",\n",
    "            metrics=['accuracy'])\n",
    " \n",
    "\n",
    "history = model.fit(train_dataset, epochs=50, shuffle=True, validation_data=test_dataset)\n",
    "\n",
    "# To track the progression of training, gather a snapshot\n",
    "# of the model's metrics at each epoch. \n",
    "epochs = history.epoch\n",
    "hist = pd.DataFrame(history.history)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['article_emb_0', 'article_emb_1', 'article_emb_2', 'article_emb_3',\n",
       "       'article_emb_4', 'article_emb_5', 'article_emb_6', 'article_emb_7',\n",
       "       'article_emb_8', 'article_emb_9',\n",
       "       ...\n",
       "       'user_emb762', 'user_emb763', 'user_emb764', 'user_emb765',\n",
       "       'user_emb766', 'user_emb767', 'totalClapCount', 'example_age_square',\n",
       "       'example_age', 'label'],\n",
       "      dtype='object', length=2308)"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# use the embedding \n",
    "train_data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}