fast-lgb-augmentation-without-kfolds.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "8846f3b662b6cee6b0b4250c60be525b779452ac"
   },
   "source": [
    "Big thanks to:\n",
    "* Jiwei Liu for [Augment insight](https://www.kaggle.com/jiweiliu/lgb-2-leaves-augment)\n",
    "* [Ole Morten Grodås, Lightgbm with data augmentation](https://www.kaggle.com/omgrodas/lightgbm-with-data-augmentation)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [],
   "source": [
    "import lightgbm as lgb\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [],
   "source": [
    "train=pd.read_csv(\"../input/train.csv\").drop(\"ID_code\",axis=1)\n",
    "test=pd.read_csv(\"../input/test.csv\").drop(\"ID_code\",axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "_uuid": "9f9c47a50c46d136c629c6aa36ecdf812dbc1532"
   },
   "outputs": [],
   "source": [
    "# use sample instead of all data\n",
    "sample = 0\n",
    "if(sample == 1):\n",
    "    sample_train = train[train.target == 0].sample(n = 4, random_state = 1573456)\n",
    "    sample_train_2 = train[train.target == 1].sample(n = 3, random_state = 1573456)\n",
    "    sample_train = sample_train.append(sample_train_2)\n",
    "    sample_y = sample_train['target']\n",
    "    sample_X = sample_train.drop(['target'], axis = 1)\n",
    "    \n",
    "    train = sample_train\n",
    "    y = sample_y\n",
    "    X = sample_X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "_uuid": "30518f7d088ab7150c0e203a3d3b75b3016bec3e"
   },
   "outputs": [],
   "source": [
    "# Augmentation. Inspiration from [2]\n",
    "def augment(train, num_n = 1, num_p = 2):\n",
    "    newtrain=[train]\n",
    "    \n",
    "    n = train[train.target == 0]\n",
    "    for i in range(num_n):\n",
    "        newtrain.append( n.apply( lambda x:x.values.take(np.random.permutation(len(n))) ) )\n",
    "    \n",
    "    p = train[train.target == 1]\n",
    "    for i in range(num_p):\n",
    "        newtrain.append( p.apply( lambda x:x.values.take(np.random.permutation(len(p))) ) )\n",
    "    \n",
    "    return pd.concat(newtrain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "_uuid": "ee57304897e67a222551bdeb2e5d53553441c927"
   },
   "outputs": [],
   "source": [
    "param = {\n",
    "    #'bagging_freq': 5,\n",
    "    #'bagging_fraction': 0.335,\n",
    "    'boost_from_average':'false',\n",
    "    'boost': 'gbdt',\n",
    "    'feature_fraction': 0.041,\n",
    "    'learning_rate': 0.0083,\n",
    "    'max_depth': -1,\n",
    "    'metric':'auc',\n",
    "    'min_data_in_leaf': 80,\n",
    "    'min_sum_hessian_in_leaf': 10.0,\n",
    "    'num_leaves': 13,\n",
    "    'num_threads': 8,\n",
    "    'tree_learner': 'serial',\n",
    "    'objective': 'binary', \n",
    "    'verbosity': -1\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "_uuid": "4449a9288cb03b34336924ca23ad6d0a90923821"
   },
   "outputs": [],
   "source": [
    "train_augmented = augment(train)\n",
    "train_data = lgb.Dataset(train_augmented.drop('target', axis = 1), train_augmented['target'])\n",
    "# 11500 -> ?(just run); 12985 -> 0.900; 14500 -> 0.900; 16000 -> 0.900; 17500 -> 0.900; 19000 -> 0.900; 23000 -> 0.899\n",
    "model = lgb.train(param, train_data, 11500)\n",
    "model.save_model('lgbm_model_11500')\n",
    "prediction = model.predict(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "_uuid": "43e556c61f35a8e75dbb36495521b06573f85884"
   },
   "outputs": [],
   "source": [
    "submission = pd.read_csv('../input/sample_submission.csv')\n",
    "submission['target'] = prediction\n",
    "filename = \"sub_{:%Y-%m-%d_%H_%M}.csv\".format(datetime.now())\n",
    "submission.to_csv(filename, index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "_uuid": "b5823e42beb60c08c1d6debbe8d9d85a8d9a498d"
   },
   "source": [
    "# Inspired by:\n",
    "\n",
    "[1] [Ole Morten Grodås, Lightgbm with data augmentation](https://www.kaggle.com/omgrodas/lightgbm-with-data-augmentation)\n",
    "\n",
    "[2] [Jiwei Liu, LGB 2 leaves + augment](https://www.kaggle.com/jiweiliu/lgb-2-leaves-augment)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}