HFModel_Boosting.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# High Freq Model\n",
    "\n",
    "在这个notebook中，我们用训练高频模型。这里的代码做为模型更新线程的基础。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle\n",
    "import matplotlib.pyplot as plt\n",
    "import time\n",
    "\n",
    "pd.options.mode.chained_assignment = None\n",
    "\n",
    "%run Functions.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def to_pm(s):\n",
    "    '''\n",
    "    提取序列中的正负号 ( e.g. [-3, 2, -2, 3] -> [-1, 1, -1, 1] )\n",
    "    s : array\n",
    "    '''\n",
    "    s_pm = np.zeros(s.shape)\n",
    "    for i in range(len(s)):\n",
    "        if s[i] > 0:\n",
    "            s_pm[i] = 1\n",
    "        if s[i] < 0:\n",
    "            s_pm[i] = -1\n",
    "    return s_pm\n",
    "\n",
    "def calc_accuracy(pred, real):\n",
    "    return (1 + np.sum(to_pm(pred) * to_pm(real))/len(pred))/2\n",
    "\n",
    "def plotTVT(tr, trp, va, vap, te, tep):\n",
    "\n",
    "    plt.figure(figsize = (15, 5))\n",
    "\n",
    "    plt.subplot(1, 3, 1)\n",
    "    plt.plot(trp, tr, 'g.')\n",
    "    plt.plot([-1,1], [-1,1], 'grey', ls = '--')\n",
    "    plt.plot([-1,1], np.poly1d(np.polyfit(trp, tr, 1))([-1,1]), 'g--')\n",
    "    plt.xlabel('pred')\n",
    "    plt.ylabel('real')\n",
    "    plt.title('training set')\n",
    "    plt.grid()\n",
    "    \n",
    "    plt.subplot(1, 3, 2)\n",
    "    plt.plot(vap, va, 'g.')\n",
    "    plt.plot([-1,1], [-1,1], 'grey', ls = '--')\n",
    "    plt.plot([-1,1], np.poly1d(np.polyfit(vap, va, 1))([-1,1]), 'g--')\n",
    "    plt.xlabel('pred')\n",
    "    plt.ylabel('real')\n",
    "    plt.title('validation set')\n",
    "    plt.grid()\n",
    "    \n",
    "    plt.subplot(1, 3, 3)\n",
    "    plt.plot(tep, te, 'g.')\n",
    "    plt.plot([-1,1], [-1,1], 'grey', ls = '--')\n",
    "    plt.plot([-1,1], np.poly1d(np.polyfit(tep, te, 1))([-1,1]), 'g--')\n",
    "    plt.xlabel('pred')\n",
    "    plt.ylabel('real')\n",
    "    plt.title('test set')\n",
    "    plt.grid()\n",
    "    \n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "alphas23 = pd.read_csv('data/alphas23.csv').set_index('Unnamed: 0')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tick</th>\n",
       "      <th>stock</th>\n",
       "      <th>close</th>\n",
       "      <th>SOIR1</th>\n",
       "      <th>SOIR2</th>\n",
       "      <th>SOIR3</th>\n",
       "      <th>SOIR4</th>\n",
       "      <th>SOIR5</th>\n",
       "      <th>SOIR6</th>\n",
       "      <th>SOIR7</th>\n",
       "      <th>...</th>\n",
       "      <th>MPC4</th>\n",
       "      <th>MPC5</th>\n",
       "      <th>MPC6</th>\n",
       "      <th>MPC7</th>\n",
       "      <th>MPC8</th>\n",
       "      <th>MPC9</th>\n",
       "      <th>MPC10</th>\n",
       "      <th>momentum5</th>\n",
       "      <th>momentum10</th>\n",
       "      <th>momentum20</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>4000</td>\n",
       "      <td>1042.170</td>\n",
       "      <td>-0.777778</td>\n",
       "      <td>-0.977778</td>\n",
       "      <td>-0.230769</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>4001</td>\n",
       "      <td>98.586</td>\n",
       "      <td>-0.750006</td>\n",
       "      <td>0.166711</td>\n",
       "      <td>0.954291</td>\n",
       "      <td>-0.729765</td>\n",
       "      <td>0.157954</td>\n",
       "      <td>0.285731</td>\n",
       "      <td>0.063828</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>4002</td>\n",
       "      <td>66.660</td>\n",
       "      <td>-0.454545</td>\n",
       "      <td>0.800000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>4003</td>\n",
       "      <td>22.668</td>\n",
       "      <td>0.760005</td>\n",
       "      <td>-0.885748</td>\n",
       "      <td>0.612910</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.916678</td>\n",
       "      <td>0.090923</td>\n",
       "      <td>-0.975316</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>4004</td>\n",
       "      <td>399.928</td>\n",
       "      <td>0.200009</td>\n",
       "      <td>0.569893</td>\n",
       "      <td>-0.837899</td>\n",
       "      <td>-0.942880</td>\n",
       "      <td>-0.826150</td>\n",
       "      <td>-0.833391</td>\n",
       "      <td>-0.785788</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1559995</th>\n",
       "      <td>3119</td>\n",
       "      <td>4495</td>\n",
       "      <td>1449.497</td>\n",
       "      <td>0.513539</td>\n",
       "      <td>0.588708</td>\n",
       "      <td>0.555731</td>\n",
       "      <td>0.766875</td>\n",
       "      <td>0.710178</td>\n",
       "      <td>-0.375896</td>\n",
       "      <td>-0.932339</td>\n",
       "      <td>...</td>\n",
       "      <td>0.004005</td>\n",
       "      <td>0.004675</td>\n",
       "      <td>0.005481</td>\n",
       "      <td>0.006759</td>\n",
       "      <td>0.008041</td>\n",
       "      <td>0.010208</td>\n",
       "      <td>0.011430</td>\n",
       "      <td>0.004674</td>\n",
       "      <td>0.011429</td>\n",
       "      <td>0.016898</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1559996</th>\n",
       "      <td>3119</td>\n",
       "      <td>4496</td>\n",
       "      <td>16.070</td>\n",
       "      <td>-0.646018</td>\n",
       "      <td>0.406990</td>\n",
       "      <td>0.189655</td>\n",
       "      <td>-0.168831</td>\n",
       "      <td>-0.285107</td>\n",
       "      <td>0.803403</td>\n",
       "      <td>0.412338</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.005884</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1559997</th>\n",
       "      <td>3119</td>\n",
       "      <td>4497</td>\n",
       "      <td>510.817</td>\n",
       "      <td>0.916710</td>\n",
       "      <td>0.666792</td>\n",
       "      <td>0.896105</td>\n",
       "      <td>0.421691</td>\n",
       "      <td>-0.290783</td>\n",
       "      <td>-0.135140</td>\n",
       "      <td>0.080703</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000135</td>\n",
       "      <td>0.000269</td>\n",
       "      <td>0.000740</td>\n",
       "      <td>0.000472</td>\n",
       "      <td>0.000539</td>\n",
       "      <td>0.000942</td>\n",
       "      <td>0.001077</td>\n",
       "      <td>0.000337</td>\n",
       "      <td>0.001347</td>\n",
       "      <td>0.001682</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1559998</th>\n",
       "      <td>3119</td>\n",
       "      <td>4498</td>\n",
       "      <td>107.099</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.652373</td>\n",
       "      <td>0.763163</td>\n",
       "      <td>0.784623</td>\n",
       "      <td>0.972815</td>\n",
       "      <td>0.976767</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.002008</td>\n",
       "      <td>0.001862</td>\n",
       "      <td>0.003734</td>\n",
       "      <td>0.004025</td>\n",
       "      <td>0.002874</td>\n",
       "      <td>0.003162</td>\n",
       "      <td>0.004314</td>\n",
       "      <td>0.001431</td>\n",
       "      <td>0.004314</td>\n",
       "      <td>-0.000718</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1559999</th>\n",
       "      <td>3119</td>\n",
       "      <td>4499</td>\n",
       "      <td>108.134</td>\n",
       "      <td>-0.894746</td>\n",
       "      <td>0.272732</td>\n",
       "      <td>-0.932585</td>\n",
       "      <td>0.600023</td>\n",
       "      <td>0.945951</td>\n",
       "      <td>-0.058835</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.003059</td>\n",
       "      <td>0.005182</td>\n",
       "      <td>0.006370</td>\n",
       "      <td>0.007560</td>\n",
       "      <td>0.008507</td>\n",
       "      <td>0.007792</td>\n",
       "      <td>0.005894</td>\n",
       "      <td>0.004711</td>\n",
       "      <td>0.004711</td>\n",
       "      <td>0.004711</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1560000 rows × 26 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            tick  stock     close     SOIR1     SOIR2     SOIR3     SOIR4  \\\n",
       "Unnamed: 0                                                                  \n",
       "0              0   4000  1042.170 -0.777778 -0.977778 -0.230769  1.000000   \n",
       "1              0   4001    98.586 -0.750006  0.166711  0.954291 -0.729765   \n",
       "2              0   4002    66.660 -0.454545  0.800000 -1.000000 -1.000000   \n",
       "3              0   4003    22.668  0.760005 -0.885748  0.612910  0.000000   \n",
       "4              0   4004   399.928  0.200009  0.569893 -0.837899 -0.942880   \n",
       "...          ...    ...       ...       ...       ...       ...       ...   \n",
       "1559995     3119   4495  1449.497  0.513539  0.588708  0.555731  0.766875   \n",
       "1559996     3119   4496    16.070 -0.646018  0.406990  0.189655 -0.168831   \n",
       "1559997     3119   4497   510.817  0.916710  0.666792  0.896105  0.421691   \n",
       "1559998     3119   4498   107.099  0.000000 -0.652373  0.763163  0.784623   \n",
       "1559999     3119   4499   108.134 -0.894746  0.272732 -0.932585  0.600023   \n",
       "\n",
       "               SOIR5     SOIR6     SOIR7  ...      MPC4      MPC5      MPC6  \\\n",
       "Unnamed: 0                                ...                                 \n",
       "0           1.000000  0.000000  0.000000  ...       NaN       NaN       NaN   \n",
       "1           0.157954  0.285731  0.063828  ...       NaN       NaN       NaN   \n",
       "2           0.000000  0.000000  0.000000  ...       NaN       NaN       NaN   \n",
       "3          -0.916678  0.090923 -0.975316  ...       NaN       NaN       NaN   \n",
       "4          -0.826150 -0.833391 -0.785788  ...       NaN       NaN       NaN   \n",
       "...              ...       ...       ...  ...       ...       ...       ...   \n",
       "1559995     0.710178 -0.375896 -0.932339  ...  0.004005  0.004675  0.005481   \n",
       "1559996    -0.285107  0.803403  0.412338  ...  0.000000  0.000000  0.000000   \n",
       "1559997    -0.290783 -0.135140  0.080703  ...  0.000135  0.000269  0.000740   \n",
       "1559998     0.972815  0.976767  1.000000  ...  0.002008  0.001862  0.003734   \n",
       "1559999     0.945951 -0.058835 -1.000000  ...  0.003059  0.005182  0.006370   \n",
       "\n",
       "                MPC7      MPC8      MPC9     MPC10  momentum5  momentum10  \\\n",
       "Unnamed: 0                                                                  \n",
       "0                NaN       NaN       NaN       NaN        NaN         NaN   \n",
       "1                NaN       NaN       NaN       NaN        NaN         NaN   \n",
       "2                NaN       NaN       NaN       NaN        NaN         NaN   \n",
       "3                NaN       NaN       NaN       NaN        NaN         NaN   \n",
       "4                NaN       NaN       NaN       NaN        NaN         NaN   \n",
       "...              ...       ...       ...       ...        ...         ...   \n",
       "1559995     0.006759  0.008041  0.010208  0.011430   0.004674    0.011429   \n",
       "1559996     0.000000  0.000000  0.000000  0.000000   0.000000    0.000000   \n",
       "1559997     0.000472  0.000539  0.000942  0.001077   0.000337    0.001347   \n",
       "1559998     0.004025  0.002874  0.003162  0.004314   0.001431    0.004314   \n",
       "1559999     0.007560  0.008507  0.007792  0.005894   0.004711    0.004711   \n",
       "\n",
       "            momentum20  \n",
       "Unnamed: 0              \n",
       "0                  NaN  \n",
       "1                  NaN  \n",
       "2                  NaN  \n",
       "3                  NaN  \n",
       "4                  NaN  \n",
       "...                ...  \n",
       "1559995       0.016898  \n",
       "1559996       0.005884  \n",
       "1559997       0.001682  \n",
       "1559998      -0.000718  \n",
       "1559999       0.004711  \n",
       "\n",
       "[1560000 rows x 26 columns]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "alphas23"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tick</th>\n",
       "      <th>stock</th>\n",
       "      <th>close</th>\n",
       "      <th>SOIR1</th>\n",
       "      <th>SOIR2</th>\n",
       "      <th>SOIR3</th>\n",
       "      <th>SOIR4</th>\n",
       "      <th>SOIR5</th>\n",
       "      <th>SOIR6</th>\n",
       "      <th>SOIR7</th>\n",
       "      <th>...</th>\n",
       "      <th>MPC6</th>\n",
       "      <th>MPC7</th>\n",
       "      <th>MPC8</th>\n",
       "      <th>MPC9</th>\n",
       "      <th>MPC10</th>\n",
       "      <th>momentum5</th>\n",
       "      <th>momentum10</th>\n",
       "      <th>momentum20</th>\n",
       "      <th>obj10</th>\n",
       "      <th>obj5</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>10000</th>\n",
       "      <td>20</td>\n",
       "      <td>4000</td>\n",
       "      <td>1014.758</td>\n",
       "      <td>0.629630</td>\n",
       "      <td>-0.066667</td>\n",
       "      <td>0.272727</td>\n",
       "      <td>0.809524</td>\n",
       "      <td>-0.785714</td>\n",
       "      <td>0.617021</td>\n",
       "      <td>0.217391</td>\n",
       "      <td>...</td>\n",
       "      <td>0.002418</td>\n",
       "      <td>-0.001445</td>\n",
       "      <td>0.003874</td>\n",
       "      <td>0.001933</td>\n",
       "      <td>-0.001445</td>\n",
       "      <td>0.003874</td>\n",
       "      <td>-0.001445</td>\n",
       "      <td>-0.026303</td>\n",
       "      <td>-0.012014</td>\n",
       "      <td>0.001441</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10001</th>\n",
       "      <td>20</td>\n",
       "      <td>4001</td>\n",
       "      <td>98.414</td>\n",
       "      <td>-0.600000</td>\n",
       "      <td>0.333333</td>\n",
       "      <td>-0.941199</td>\n",
       "      <td>-0.777778</td>\n",
       "      <td>0.906443</td>\n",
       "      <td>-0.075474</td>\n",
       "      <td>0.103451</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.001158</td>\n",
       "      <td>-0.001158</td>\n",
       "      <td>-0.000001</td>\n",
       "      <td>-0.000001</td>\n",
       "      <td>-0.000001</td>\n",
       "      <td>-0.001157</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.001745</td>\n",
       "      <td>-0.002904</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10002</th>\n",
       "      <td>20</td>\n",
       "      <td>4002</td>\n",
       "      <td>67.371</td>\n",
       "      <td>-0.500000</td>\n",
       "      <td>0.272727</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.008969</td>\n",
       "      <td>0.009530</td>\n",
       "      <td>0.007839</td>\n",
       "      <td>0.005582</td>\n",
       "      <td>0.006709</td>\n",
       "      <td>0.010105</td>\n",
       "      <td>0.006709</td>\n",
       "      <td>0.010666</td>\n",
       "      <td>-0.003885</td>\n",
       "      <td>0.001661</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10003</th>\n",
       "      <td>20</td>\n",
       "      <td>4003</td>\n",
       "      <td>23.207</td>\n",
       "      <td>-0.822795</td>\n",
       "      <td>-0.764710</td>\n",
       "      <td>-0.684213</td>\n",
       "      <td>-0.676468</td>\n",
       "      <td>0.869568</td>\n",
       "      <td>-0.951022</td>\n",
       "      <td>-0.798663</td>\n",
       "      <td>...</td>\n",
       "      <td>0.008255</td>\n",
       "      <td>0.008255</td>\n",
       "      <td>0.009654</td>\n",
       "      <td>0.009654</td>\n",
       "      <td>0.009654</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.009658</td>\n",
       "      <td>0.023778</td>\n",
       "      <td>0.016331</td>\n",
       "      <td>0.021804</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10004</th>\n",
       "      <td>20</td>\n",
       "      <td>4004</td>\n",
       "      <td>400.608</td>\n",
       "      <td>-0.333359</td>\n",
       "      <td>0.111145</td>\n",
       "      <td>0.904800</td>\n",
       "      <td>-0.023247</td>\n",
       "      <td>-0.780838</td>\n",
       "      <td>-0.593228</td>\n",
       "      <td>-0.911117</td>\n",
       "      <td>...</td>\n",
       "      <td>0.001700</td>\n",
       "      <td>0.000425</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.000848</td>\n",
       "      <td>-0.000848</td>\n",
       "      <td>0.002126</td>\n",
       "      <td>-0.000848</td>\n",
       "      <td>0.001700</td>\n",
       "      <td>-0.002540</td>\n",
       "      <td>-0.000423</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1537995</th>\n",
       "      <td>3075</td>\n",
       "      <td>4495</td>\n",
       "      <td>1415.776</td>\n",
       "      <td>-0.176480</td>\n",
       "      <td>0.181781</td>\n",
       "      <td>-0.414606</td>\n",
       "      <td>0.400030</td>\n",
       "      <td>0.692420</td>\n",
       "      <td>-0.222927</td>\n",
       "      <td>0.245991</td>\n",
       "      <td>...</td>\n",
       "      <td>0.003414</td>\n",
       "      <td>0.003414</td>\n",
       "      <td>0.003414</td>\n",
       "      <td>0.003414</td>\n",
       "      <td>0.003551</td>\n",
       "      <td>0.001362</td>\n",
       "      <td>0.003414</td>\n",
       "      <td>0.005473</td>\n",
       "      <td>0.000340</td>\n",
       "      <td>-0.000340</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1537996</th>\n",
       "      <td>3075</td>\n",
       "      <td>4496</td>\n",
       "      <td>16.070</td>\n",
       "      <td>-0.234657</td>\n",
       "      <td>0.480041</td>\n",
       "      <td>0.086705</td>\n",
       "      <td>-0.240084</td>\n",
       "      <td>-0.100720</td>\n",
       "      <td>0.312349</td>\n",
       "      <td>0.071719</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.005884</td>\n",
       "      <td>-0.005849</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1537997</th>\n",
       "      <td>3075</td>\n",
       "      <td>4497</td>\n",
       "      <td>509.959</td>\n",
       "      <td>0.254230</td>\n",
       "      <td>0.192994</td>\n",
       "      <td>-0.085700</td>\n",
       "      <td>-0.438591</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>0.585187</td>\n",
       "      <td>0.021583</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.000135</td>\n",
       "      <td>-0.000538</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1537998</th>\n",
       "      <td>3075</td>\n",
       "      <td>4498</td>\n",
       "      <td>107.559</td>\n",
       "      <td>0.437510</td>\n",
       "      <td>0.771444</td>\n",
       "      <td>0.528540</td>\n",
       "      <td>-0.694586</td>\n",
       "      <td>0.966974</td>\n",
       "      <td>0.587791</td>\n",
       "      <td>0.714297</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.005526</td>\n",
       "      <td>-0.005103</td>\n",
       "      <td>-0.003124</td>\n",
       "      <td>-0.002133</td>\n",
       "      <td>-0.002703</td>\n",
       "      <td>-0.005667</td>\n",
       "      <td>-0.002846</td>\n",
       "      <td>-0.004259</td>\n",
       "      <td>-0.007131</td>\n",
       "      <td>-0.002854</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1537999</th>\n",
       "      <td>3075</td>\n",
       "      <td>4499</td>\n",
       "      <td>106.359</td>\n",
       "      <td>0.928578</td>\n",
       "      <td>0.249982</td>\n",
       "      <td>0.142846</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.023197</td>\n",
       "      <td>0.023197</td>\n",
       "      <td>0.023197</td>\n",
       "      <td>0.023197</td>\n",
       "      <td>0.023197</td>\n",
       "      <td>0.024416</td>\n",
       "      <td>0.024416</td>\n",
       "      <td>0.024416</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001187</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1528000 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            tick  stock     close     SOIR1     SOIR2     SOIR3     SOIR4  \\\n",
       "Unnamed: 0                                                                  \n",
       "10000         20   4000  1014.758  0.629630 -0.066667  0.272727  0.809524   \n",
       "10001         20   4001    98.414 -0.600000  0.333333 -0.941199 -0.777778   \n",
       "10002         20   4002    67.371 -0.500000  0.272727  1.000000  1.000000   \n",
       "10003         20   4003    23.207 -0.822795 -0.764710 -0.684213 -0.676468   \n",
       "10004         20   4004   400.608 -0.333359  0.111145  0.904800 -0.023247   \n",
       "...          ...    ...       ...       ...       ...       ...       ...   \n",
       "1537995     3075   4495  1415.776 -0.176480  0.181781 -0.414606  0.400030   \n",
       "1537996     3075   4496    16.070 -0.234657  0.480041  0.086705 -0.240084   \n",
       "1537997     3075   4497   509.959  0.254230  0.192994 -0.085700 -0.438591   \n",
       "1537998     3075   4498   107.559  0.437510  0.771444  0.528540 -0.694586   \n",
       "1537999     3075   4499   106.359  0.928578  0.249982  0.142846  0.000000   \n",
       "\n",
       "               SOIR5     SOIR6     SOIR7  ...      MPC6      MPC7      MPC8  \\\n",
       "Unnamed: 0                                ...                                 \n",
       "10000      -0.785714  0.617021  0.217391  ...  0.002418 -0.001445  0.003874   \n",
       "10001       0.906443 -0.075474  0.103451  ... -0.001158 -0.001158 -0.000001   \n",
       "10002       1.000000  1.000000  1.000000  ...  0.008969  0.009530  0.007839   \n",
       "10003       0.869568 -0.951022 -0.798663  ...  0.008255  0.008255  0.009654   \n",
       "10004      -0.780838 -0.593228 -0.911117  ...  0.001700  0.000425  0.000000   \n",
       "...              ...       ...       ...  ...       ...       ...       ...   \n",
       "1537995     0.692420 -0.222927  0.245991  ...  0.003414  0.003414  0.003414   \n",
       "1537996    -0.100720  0.312349  0.071719  ...  0.000000  0.000000  0.000000   \n",
       "1537997     0.500000  0.585187  0.021583  ...  0.000000 -0.000135 -0.000538   \n",
       "1537998     0.966974  0.587791  0.714297  ... -0.005526 -0.005103 -0.003124   \n",
       "1537999    -1.000000 -1.000000 -1.000000  ...  0.023197  0.023197  0.023197   \n",
       "\n",
       "                MPC9     MPC10  momentum5  momentum10  momentum20     obj10  \\\n",
       "Unnamed: 0                                                                    \n",
       "10000       0.001933 -0.001445   0.003874   -0.001445   -0.026303 -0.012014   \n",
       "10001      -0.000001 -0.000001  -0.001157    0.000000   -0.001745 -0.002904   \n",
       "10002       0.005582  0.006709   0.010105    0.006709    0.010666 -0.003885   \n",
       "10003       0.009654  0.009654   0.000000    0.009658    0.023778  0.016331   \n",
       "10004      -0.000848 -0.000848   0.002126   -0.000848    0.001700 -0.002540   \n",
       "...              ...       ...        ...         ...         ...       ...   \n",
       "1537995     0.003414  0.003551   0.001362    0.003414    0.005473  0.000340   \n",
       "1537996     0.000000  0.000000   0.000000    0.000000    0.005884 -0.005849   \n",
       "1537997     0.000000  0.000000   0.000000    0.000000    0.000000  0.000000   \n",
       "1537998    -0.002133 -0.002703  -0.005667   -0.002846   -0.004259 -0.007131   \n",
       "1537999     0.023197  0.023197   0.024416    0.024416    0.024416  0.000000   \n",
       "\n",
       "                obj5  \n",
       "Unnamed: 0            \n",
       "10000       0.001441  \n",
       "10001       0.000000  \n",
       "10002       0.001661  \n",
       "10003       0.021804  \n",
       "10004      -0.000423  \n",
       "...              ...  \n",
       "1537995    -0.000340  \n",
       "1537996     0.000000  \n",
       "1537997     0.000000  \n",
       "1537998    -0.002854  \n",
       "1537999     0.001187  \n",
       "\n",
       "[1528000 rows x 28 columns]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub['obj10'] = sub[['stock', 'close']].groupby('stock') \\\n",
    "    .transform(lambda x: laggingF(abs2percF(x.values, 10), 1))\n",
    "\n",
    "sub['obj5'] = sub[['stock', 'close']].groupby('stock') \\\n",
    "    .transform(lambda x: laggingF(abs2percF(x.values, 5), 1))\n",
    "\n",
    "sub = sub.replace([np.inf, -np.inf], np.nan).dropna()\n",
    "\n",
    "sub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# with pd.option_context('display.max_rows', None, 'display.max_columns', None):\n",
    "#     print(alphasbasic.loc[: ,'obj1':].corr())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "y10_train = sub['obj10'].values\n",
    "y5_train = sub['obj5'].values\n",
    "X_train = sub.iloc[:, 3:-2].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      Iter       Train Loss   Remaining Time \n",
      "         1           0.0000            4.98m\n",
      "         2           0.0000            5.01m\n",
      "         3           0.0000            4.66m\n",
      "         4           0.0000            4.31m\n",
      "         5           0.0000            4.06m\n",
      "         6           0.0000            3.87m\n",
      "         7           0.0000            3.65m\n",
      "         8           0.0000            3.43m\n",
      "         9           0.0000            3.19m\n",
      "        10           0.0000            3.00m\n",
      "        20           0.0000            1.01m\n",
      "299.68202018737793\n",
      "      Iter       Train Loss   Remaining Time \n",
      "         1           0.0000            4.77m\n",
      "         2           0.0000            4.63m\n",
      "         3           0.0000            4.36m\n",
      "         4           0.0000            4.41m\n",
      "         5           0.0000            4.20m\n",
      "         6           0.0000            3.92m\n",
      "         7           0.0000            3.68m\n",
      "         8           0.0000            3.45m\n",
      "         9           0.0000            3.24m\n",
      "        10           0.0000            3.02m\n",
      "        20           0.0000            1.02m\n",
      "601.1286909580231\n"
     ]
    }
   ],
   "source": [
    "from sklearn.ensemble import GradientBoostingRegressor\n",
    "\n",
    "t = time.time()\n",
    "\n",
    "gb10 = GradientBoostingRegressor(loss = 'huber', \n",
    "                       learning_rate = 0.25, \n",
    "                       n_estimators = 25, \n",
    "                       verbose = True, \n",
    "                       random_state = 2020)\n",
    "gb10.fit(X_train, y10_train)\n",
    "\n",
    "print(time.time()- t)\n",
    "\n",
    "gb5 = GradientBoostingRegressor(loss = 'huber', \n",
    "                       learning_rate = 0.25, \n",
    "                       n_estimators = 25, \n",
    "                       verbose = True, \n",
    "                       random_state = 2020)\n",
    "gb5.fit(X_train, y5_train)\n",
    "\n",
    "print(time.time()- t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pickle.dump(gb10, open('Models/gb10_final.sav', 'wb'))\n",
    "# pickle.dump(gb5, open('Models/gb5_final.sav', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training MSE: 0.00039549005537695256\n",
      "Training accuracy: 0.5323399869109948\n"
     ]
    }
   ],
   "source": [
    "y10_train_pred = gb10.predict(X_train)\n",
    "print(\"Training MSE: \" + str(mean_squared_error(y10_train_pred, y10_train)))\n",
    "print(\"Training accuracy: \" + str(calc_accuracy(y10_train_pred, y10_train)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training MSE: 0.00020046882640963143\n",
      "Training accuracy: 0.5301917539267016\n"
     ]
    }
   ],
   "source": [
    "y5_train_pred = gb5.predict(X_train)\n",
    "print(\"Training MSE: \" + str(mean_squared_error(y5_train_pred, y5_train)))\n",
    "print(\"Training accuracy: \" + str(calc_accuracy(y5_train_pred, y5_train)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "45840.0"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "1528000 * 15/500"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.0033679528065970886, -0.0019388365383932796)"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(np.mean(y10_train[y10_train_pred.argsort()[-45840:]]), \\\n",
    "np.mean(y10_train[y10_train_pred.argsort()[:45840]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.001793683396226249, -0.0011919373740673748)"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "(np.mean(y5_train[y5_train_pred.argsort()[-45840:]]), \\\n",
    "np.mean(y5_train[y5_train_pred.argsort()[:45840]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from sklearn.metrics import mean_squared_error\n",
    "# from sklearn.utils import shuffle\n",
    "\n",
    "# data_shuf = shuffle(sub, random_state = 2020)\n",
    "\n",
    "# y10_train = data_shuf.iloc[:900000, 0].values\n",
    "# y5_train = data_shuf.iloc[:900000, 1].values\n",
    "# X_train = data_shuf.iloc[:900000, 2:].values\n",
    "# y10_validate = data_shuf.iloc[900000:1300000, 0].values\n",
    "# y5_validate = data_shuf.iloc[900000:1300000, 1].values\n",
    "# X_validate = data_shuf.iloc[900000:1300000, 2:].values\n",
    "# y10_test = data_shuf.iloc[1300000:, 0].values\n",
    "# y5_test = data_shuf.iloc[1300000:, 1].values\n",
    "# X_test = data_shuf.iloc[1300000:, 2:].values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "# y10 = data_shuf.iloc[:, 0].values\n",
    "# y5 = data_shuf.iloc[:, 1].values\n",
    "# X = data_shuf.iloc[:, 2:].values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# SKLEARN gradient boosting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import GradientBoostingRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "gb10 = GradientBoostingRegressor(loss = 'huber', \n",
    "                               learning_rate = 0.25, \n",
    "                               n_estimators = 25, \n",
    "                               verbose = True, \n",
    "                               random_state = 2020)\n",
    "\n",
    "gb10.fit(X_train, y10_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y10_train_pred = gb10.predict(X_train)\n",
    "print(\"Training MSE: \" + str(mean_squared_error(y10_train_pred, y10_train)))\n",
    "print(\"Training accuracy: \" + str(calc_accuracy(y10_train_pred, y10_train)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y10_validate_pred = gb10.predict(X_validate)\n",
    "print(\"Validation MSE: \" + str(mean_squared_error(y10_validate_pred, y10_validate)))\n",
    "print(\"Validation accuracy: \" + str(calc_accuracy(y10_validate_pred, y10_validate)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y10_test_pred = gb10.predict(X_test)\n",
    "print(\"Test MSE: \" + str(mean_squared_error(y10_test_pred, y10_test)))\n",
    "print(\"Test accuracy: \" + str(calc_accuracy(y10_test_pred, y10_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "plotTVT(y10_train, y10_train_pred, \n",
    "        y10_validate, y10_validate_pred,\n",
    "        y10_test, y10_test_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# (1544500 - 1300000) * 15/500 = 7335\n",
    "\n",
    "(np.mean(y10_test[y10_test_pred.argsort()[-7335:]]), \\\n",
    "np.mean(y10_test[y10_test_pred.argsort()[:7335]]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "gb5 = GradientBoostingRegressor(loss = 'huber', \n",
    "                               learning_rate = 0.25, \n",
    "                               n_estimators = 25, \n",
    "                               verbose = True, \n",
    "                               random_state = 2020)\n",
    "\n",
    "gb5.fit(X_train, y5_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "y5_train_pred = gb5.predict(X_train)\n",
    "print(\"Training MSE: \" + str(mean_squared_error(y5_train_pred, y5_train)))\n",
    "print(\"Training accuracy: \" + str(calc_accuracy(y5_train_pred, y5_train)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y5_validate_pred = gb5.predict(X_validate)\n",
    "print(\"Validation MSE: \" + str(mean_squared_error(y5_validate_pred, y5_validate)))\n",
    "print(\"Validation accuracy: \" + str(calc_accuracy(y5_validate_pred, y5_validate)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y5_test_pred = gb5.predict(X_test)\n",
    "print(\"Test MSE: \" + str(mean_squared_error(y5_test_pred, y5_test)))\n",
    "print(\"Test accuracy: \" + str(calc_accuracy(y5_test_pred, y5_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plotTVT(y5_train, y5_train_pred, \n",
    "        y5_validate, y5_validate_pred,\n",
    "        y5_test, y5_test_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# (1544500 - 1300000) * 15/500 = 7335\n",
    "\n",
    "(np.mean(y5_test[y5_test_pred.argsort()[-7335:]]), \\\n",
    "np.mean(y5_test[y5_test_pred.argsort()[:7335]]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LIGHTGBM gradient boosting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import lightgbm as lgb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # grid search\n",
    "\n",
    "# import itertools\n",
    "\n",
    "# l_num_leaves = [40,60,80,100,120]\n",
    "# l_learning_rate = [0.05, 0.1, 0.2, 0.4, 0.8]\n",
    "# l_num_boost_round = [100, 300, 500]\n",
    "# l_feature_fraction = [0.5, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# l_acc = np.zeros(0)\n",
    "\n",
    "# for x in itertools.product(l_num_leaves, l_learning_rate, l_num_boost_round, l_feature_fraction):\n",
    "    \n",
    "#     params = {\n",
    "#         'boosting_type': 'gbdt',\n",
    "#         'objective': 'regression',\n",
    "#         'metric': {'huber'},\n",
    "#         'num_leaves': x[0],\n",
    "#         'learning_rate': x[1],\n",
    "#         'num_boost_round' : x[2],\n",
    "#         'feature_fraction': x[3],\n",
    "#         'bagging_fraction': 1,\n",
    "#         'bagging_freq': 1,\n",
    "#         'verbose': -1\n",
    "#     }\n",
    "#     lgb10_train = lgb.Dataset(X_train, y10_train)\n",
    "#     lgb10_eval = lgb.Dataset(X_validate, y10_validate, reference = lgb10_train)\n",
    "#     lgbm10 = lgb.train(params, lgb10_train, valid_sets = lgb10_eval)\n",
    "#     y10_test_lgbmpred = lgbm10.predict(X_test)\n",
    "    \n",
    "#     l_acc = np.append(l_acc, calc_accuracy(y10_test_lgbmpred, y10_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# np.max(l_acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# list(itertools.product(l_num_leaves, l_learning_rate, l_num_boost_round, l_feature_fraction))[np.where(l_acc == max(l_acc))[0][0]]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## hf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# futher parameter tuning\n",
    "\n",
    "params = {\n",
    "    'boosting_type': 'gbdt',\n",
    "    'objective': 'regression',\n",
    "    'metric': {'huber'},\n",
    "    'num_leaves': 100,\n",
    "    'learning_rate': 0.2,\n",
    "    'num_boost_round' : 1000,\n",
    "    'feature_fraction': 0.5,\n",
    "    'bagging_fraction': 1,\n",
    "    'bagging_freq': 1,\n",
    "    'verbose': -1\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "lgb10_train = lgb.Dataset(X_train, y10_train)\n",
    "lgb10_eval = lgb.Dataset(X_validate, y10_validate, reference = lgb10_train)\n",
    "lgbm10 = lgb.train(params, lgb10_train, valid_sets = lgb10_eval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "y10_train_lgbmpred = lgbm10.predict(X_train)\n",
    "print(\"Training MSE: \" + str(mean_squared_error(y10_train_lgbmpred, y10_train)))\n",
    "print(\"Training accuracy: \" + str(calc_accuracy(y10_train_lgbmpred, y10_train)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y10_validate_lgbmpred = lgbm10.predict(X_validate)\n",
    "print(\"Validation MSE: \" + str(mean_squared_error(y10_validate_lgbmpred, y10_validate)))\n",
    "print(\"Validation accuracy: \" + str(calc_accuracy(y10_validate_lgbmpred, y10_validate)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y10_test_lgbmpred = lgbm10.predict(X_test)\n",
    "print(\"Test MSE: \" + str(mean_squared_error(y10_test_lgbmpred, y10_test)))\n",
    "print(\"Test accuracy: \" + str(calc_accuracy(y10_test_lgbmpred, y10_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "plotTVT(y10_train, y10_train_lgbmpred, \n",
    "        y10_validate, y10_validate_lgbmpred,\n",
    "        y10_test, y10_test_lgbmpred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# (1544500 - 1300000) * 15/500 = 7335\n",
    "\n",
    "(np.mean(y10_test[y10_test_lgbmpred.argsort()[-7335:]]), \\\n",
    "np.mean(y10_test[y10_test_lgbmpred.argsort()[:7335]]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# hhf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# futher parameter tuning\n",
    "\n",
    "params = {\n",
    "    'boosting_type': 'gbdt',\n",
    "    'objective': 'regression',\n",
    "    'metric': {'huber'},\n",
    "    'num_leaves': 100,\n",
    "    'learning_rate': 0.2,\n",
    "    'num_boost_round' : 100,\n",
    "    'feature_fraction': 0.5,\n",
    "    'bagging_fraction': 1,\n",
    "    'bagging_freq': 1, \n",
    "    'verbose': -1\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "lgb5_train = lgb.Dataset(X_train, y5_train)\n",
    "lgb5_eval = lgb.Dataset(X_validate, y5_validate, reference = lgb5_train)\n",
    "lgbm5 = lgb.train(params, lgb10_train, valid_sets = lgb5_eval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "y5_train_lgbmpred = lgbm5.predict(X_train)\n",
    "print(\"Training MSE: \" + str(mean_squared_error(y5_train_lgbmpred, y5_train)))\n",
    "print(\"Training accuracy: \" + str(calc_accuracy(y5_train_lgbmpred, y5_train)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y5_validate_lgbmpred = lgbm5.predict(X_validate)\n",
    "print(\"Validation MSE: \" + str(mean_squared_error(y5_validate_lgbmpred, y5_validate)))\n",
    "print(\"Validation accuracy: \" + str(calc_accuracy(y5_validate_lgbmpred, y5_validate)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y5_test_lgbmpred = lgbm5.predict(X_test)\n",
    "print(\"Test MSE: \" + str(mean_squared_error(y5_test_lgbmpred, y5_test)))\n",
    "print(\"Test accuracy: \" + str(calc_accuracy(y5_test_lgbmpred, y5_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "plotTVT(y5_train, y5_train_lgbmpred, \n",
    "        y5_validate, y5_validate_lgbmpred,\n",
    "        y5_test, y5_test_lgbmpred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# (1544500 - 1300000) * 15/500 = 7335\n",
    "\n",
    "(np.mean(y5_test[y5_test_lgbmpred.argsort()[-7335:]]), \\\n",
    "np.mean(y5_test[y5_test_lgbmpred.argsort()[:7335]]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 速度对比"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = time.time()\n",
    "lgbm10.predict(X_test[:500])\n",
    "print(time.time() - t)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = time.time()\n",
    "gb10.predict(X_test[:500])\n",
    "print(time.time() - t)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 模型保存"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%run Functions.ipynb\n",
    "\n",
    "alphasbasic = pd.read_csv('data/alphas23.csv').set_index('Unnamed: 0')\n",
    "\n",
    "alphasbasic['obj10'] = alphasbasic[['stock', 'close']].groupby('stock') \\\n",
    "    .transform(lambda x: laggingF(abs2percF(x.values, 10), 1)) \\\n",
    "    .replace([np.inf, -np.inf], np.nan).dropna()\n",
    "\n",
    "alphasbasic['obj5'] = alphasbasic[['stock', 'close']].groupby('stock') \\\n",
    "    .transform(lambda x: laggingF(abs2percF(x.values, 5), 1)) \\\n",
    "    .replace([np.inf, -np.inf], np.nan).dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y10_train = alphasbasic['obj10'].iloc[:600000].values\n",
    "y5_train = alphasbasic['obj5'].iloc[:600000].values\n",
    "X_train = alphasbasic.iloc[:600000, 3:-2].values\n",
    "\n",
    "y10_validate = alphasbasic['obj10'].iloc[600000:720000].values\n",
    "y5_validate = alphasbasic['obj5'].iloc[600000:720000].values\n",
    "X_validate = alphasbasic.iloc[600000:720000, 3:-2].values\n",
    "\n",
    "params10 = {\n",
    "    'boosting_type': 'gbdt',\n",
    "    'objective': 'regression',\n",
    "    'metric': {'huber'},\n",
    "    'num_leaves': 100,\n",
    "    'learning_rate': 0.2,\n",
    "    'num_boost_round' : 500,\n",
    "    'feature_fraction': 0.5,\n",
    "    'bagging_fraction': 1,\n",
    "    'bagging_freq': 1,\n",
    "    'verbose': -1\n",
    "}\n",
    "\n",
    "lgb10_train = lgb.Dataset(X_train, y10_train)\n",
    "lgb10_eval = lgb.Dataset(X_validate, y10_validate, reference = lgb10_train)\n",
    "lgbm10 = lgb.train(params10, lgb10_train, valid_sets = lgb10_eval, verbose_eval=False)\n",
    "pickle.dump(lgbm10, open('Models/lgbm10_partial.sav', 'wb'))\n",
    "\n",
    "params5 = {\n",
    "    'boosting_type': 'gbdt',\n",
    "    'objective': 'regression',\n",
    "    'metric': {'huber'},\n",
    "    'num_leaves': 100,\n",
    "    'learning_rate': 0.2,\n",
    "    'num_boost_round' : 500,\n",
    "    'feature_fraction': 0.5,\n",
    "    'bagging_fraction': 1,\n",
    "    'bagging_freq': 1,\n",
    "    'verbose': -1\n",
    "}\n",
    "\n",
    "lgb5_train = lgb.Dataset(X_train, y5_train)\n",
    "lgb5_eval = lgb.Dataset(X_validate, y5_validate, reference = lgb5_train)\n",
    "lgbm5 = lgb.train(params5, lgb5_train, valid_sets = lgb5_eval, verbose_eval=False)\n",
    "pickle.dump(lgbm5, open('Models/lgbm5_partial.sav', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 保存模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "y10_train = alphasbasic.replace([np.inf, -np.inf], np.nan).dropna()['obj10'].iloc[:600000].values\n",
    "y5_train = alphasbasic.replace([np.inf, -np.inf], np.nan).dropna()['obj5'].iloc[:600000].values\n",
    "X_train = alphasbasic.replace([np.inf, -np.inf], np.nan).dropna().iloc[:600000, 3:-2].values\n",
    "\n",
    "gb5_par = GradientBoostingRegressor(loss = 'huber', \n",
    "                               learning_rate = 0.25, \n",
    "                               n_estimators = 25, \n",
    "                               verbose = True, \n",
    "                               random_state = 2020)\n",
    "\n",
    "gb5_par.fit(X_train, y5_train)\n",
    "\n",
    "gb10_par = GradientBoostingRegressor(loss = 'huber', \n",
    "                               learning_rate = 0.25, \n",
    "                               n_estimators = 25, \n",
    "                               verbose = True, \n",
    "                               random_state = 2020)\n",
    "\n",
    "gb10_par.fit(X_train, y10_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pickle.dump(gb5_par, open('Models/gb5_par.sav', 'wb'))\n",
    "# pickle.dump(gb10_par, open('Models/gb10_par.sav', 'wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}