CCFraud-clustering.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "!unzip data/data.zip -d data/\n",
    "!pip install --upgrade pip\n",
    "!pip install numpy pandas matplotlib seaborn scikit-learn imblearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load in \n",
    "\n",
    "# Imported Libraries\n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.manifold import TSNE\n",
    "from sklearn.decomposition import PCA, TruncatedSVD\n",
    "import matplotlib.patches as mpatches\n",
    "import time\n",
    "\n",
    "# Classifier Libraries\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "import collections\n",
    "\n",
    "\n",
    "# Other Libraries\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from imblearn.pipeline import make_pipeline as imbalanced_make_pipeline\n",
    "from imblearn.over_sampling import SMOTE\n",
    "from imblearn.under_sampling import NearMiss\n",
    "from imblearn.metrics import classification_report_imbalanced\n",
    "from sklearn.metrics import precision_score, recall_score, f1_score, roc_auc_score, accuracy_score, classification_report\n",
    "from collections import Counter\n",
    "from sklearn.model_selection import KFold, StratifiedKFold\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "\n",
    "df = pd.read_csv('data/creditcard.csv')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "376ce881-463a-4a09-9ac0-c63f85577eec",
    "_kg_hide-input": true,
    "_uuid": "93031e732e5aca3a2b4984799d6bf58d76e4b52d"
   },
   "outputs": [],
   "source": [
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "03ddb929-5bc8-4af4-90cd-21dcbb57560d",
    "_kg_hide-input": true,
    "_uuid": "38bec67888aa534e9739e95ef9fac62d27a87021"
   },
   "outputs": [],
   "source": [
    "# Good No Null Values!\n",
    "df.isnull().sum().max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "6a526b6c-8463-4f6f-92b0-e8a3a21cbb2e",
    "_kg_hide-input": true,
    "_uuid": "479a5f12d3dd68262316a17b4b7b3499e0a2cbe0"
   },
   "outputs": [],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "01c007fa-0fcc-4eea-84ff-0861a2f8c533",
    "_kg_hide-input": true,
    "_uuid": "f6b96ff34855e3bf7af1f6979342b01c473e4e07"
   },
   "outputs": [],
   "source": [
    "# The classes are heavily skewed we need to solve this issue later.\n",
    "print('No Frauds', round(df['Class'].value_counts()[0]/len(df) * 100,2), '% of the dataset')\n",
    "print('Frauds', round(df['Class'].value_counts()[1]/len(df) * 100,2), '% of the dataset')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "cee315f2-325f-42b6-a640-736f10c272cc",
    "_kg_hide-input": true,
    "_uuid": "cfa51792bf6f8a6b318ae1bffcff4e922b1d1917"
   },
   "outputs": [],
   "source": [
    "fig, ax = plt.subplots(1, 2, figsize=(18,4))\n",
    "\n",
    "amount_val = df['Amount'].values\n",
    "time_val = df['Time'].values\n",
    "\n",
    "sns.distplot(amount_val, ax=ax[0], color='r')\n",
    "ax[0].set_title('Distribution of Transaction Amount', fontsize=14)\n",
    "ax[0].set_xlim([min(amount_val), max(amount_val)])\n",
    "\n",
    "sns.distplot(time_val, ax=ax[1], color='b')\n",
    "ax[1].set_title('Distribution of Transaction Time', fontsize=14)\n",
    "ax[1].set_xlim([min(time_val), max(time_val)])\n",
    "\n",
    "\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "d5d64bf0-2fbb-4096-a265-f68887bf2fde",
    "_kg_hide-input": true,
    "_uuid": "1501ec379c9b5c39c3857ba0febd0aedee9c30d5"
   },
   "outputs": [],
   "source": [
    "# Since most of our data has already been scaled we should scale the columns that are left to scale (Amount and Time)\n",
    "from sklearn.preprocessing import StandardScaler, RobustScaler\n",
    "\n",
    "# RobustScaler is less prone to outliers.\n",
    "\n",
    "std_scaler = StandardScaler()\n",
    "rob_scaler = RobustScaler()\n",
    "\n",
    "df['scaled_amount'] = rob_scaler.fit_transform(df['Amount'].values.reshape(-1,1))\n",
    "df['scaled_time'] = rob_scaler.fit_transform(df['Time'].values.reshape(-1,1))\n",
    "\n",
    "df.drop(['Time','Amount'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "cdb9bb1e-9fab-4fd1-a409-468ba8bc36ee",
    "_kg_hide-input": true,
    "_uuid": "a33d701247ab45d849c5e94735346a738a6c6970"
   },
   "outputs": [],
   "source": [
    "scaled_amount = df['scaled_amount']\n",
    "scaled_time = df['scaled_time']\n",
    "\n",
    "df.drop(['scaled_amount', 'scaled_time'], axis=1, inplace=True)\n",
    "df.insert(0, 'scaled_amount', scaled_amount)\n",
    "df.insert(1, 'scaled_time', scaled_time)\n",
    "\n",
    "# Amount and Time are Scaled!\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "657bc987-4b15-4cfa-b290-c39a2632e2ac",
    "_kg_hide-input": true,
    "_uuid": "337caaf6ed3f65beedb24a74eebb22d97ff52ba4",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "colors = [\"#0101DF\", \"#DF0101\"]\n",
    "\n",
    "sns.countplot(x='Class', data=df, palette=colors)\n",
    "plt.title('Class Distributions \\n (0: No Fraud || 1: Fraud)', fontsize=14)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "c6c962cc-6f38-4a00-bcd7-ce9d91db954c",
    "_kg_hide-input": true,
    "_uuid": "9f7b5d920703b3a3c8c0f62bc6042e4615bc8324"
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import StratifiedShuffleSplit\n",
    "\n",
    "print('No Frauds', round(df['Class'].value_counts()[0]/len(df) * 100,2), '% of the dataset')\n",
    "print('Frauds', round(df['Class'].value_counts()[1]/len(df) * 100,2), '% of the dataset')\n",
    "\n",
    "X = df.drop('Class', axis=1)\n",
    "y = df['Class']\n",
    "\n",
    "sss = StratifiedKFold(n_splits=5, random_state=None, shuffle=False)\n",
    "\n",
    "for train_index, test_index in sss.split(X, y):\n",
    "    print(\"Train:\", train_index, \"Test:\", test_index)\n",
    "    original_Xtrain, original_Xtest = X.iloc[train_index], X.iloc[test_index]\n",
    "    original_ytrain, original_ytest = y.iloc[train_index], y.iloc[test_index]\n",
    "\n",
    "# We already have X_train and y_train for undersample data thats why I am using original to distinguish and to not overwrite these variables.\n",
    "# original_Xtrain, original_Xtest, original_ytrain, original_ytest = train_test_split(X, y, test_size=0.2, random_state=42)\n",
    "\n",
    "# Check the Distribution of the labels\n",
    "\n",
    "\n",
    "# Turn into an array\n",
    "original_Xtrain = original_Xtrain.values\n",
    "original_Xtest = original_Xtest.values\n",
    "original_ytrain = original_ytrain.values\n",
    "original_ytest = original_ytest.values\n",
    "\n",
    "# See if both the train and test label distribution are similarly distributed\n",
    "train_unique_label, train_counts_label = np.unique(original_ytrain, return_counts=True)\n",
    "test_unique_label, test_counts_label = np.unique(original_ytest, return_counts=True)\n",
    "print('-' * 100)\n",
    "\n",
    "print('Label Distributions: \\n')\n",
    "print(train_counts_label/ len(original_ytrain))\n",
    "print(test_counts_label/ len(original_ytest))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "f0acfc44-eb2a-4356-ad03-d0c12807acd7",
    "_kg_hide-input": true,
    "_uuid": "e3a2b89752681164f14c8273452fc66734d7f41b"
   },
   "outputs": [],
   "source": [
    "# Since our classes are highly skewed we should make them equivalent in order to have a normal distribution of the classes.\n",
    "\n",
    "# Lets shuffle the data before creating the subsamples\n",
    "\n",
    "df = df.sample(frac=1)\n",
    "\n",
    "# amount of fraud classes 492 rows.\n",
    "fraud_df = df.loc[df['Class'] == 1]\n",
    "non_fraud_df = df.loc[df['Class'] == 0][:492]\n",
    "\n",
    "normal_distributed_df = pd.concat([fraud_df, non_fraud_df])\n",
    "\n",
    "# Shuffle dataframe rows\n",
    "new_df = normal_distributed_df.sample(frac=1, random_state=42)\n",
    "\n",
    "new_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "73454100-dc69-49fd-b1b2-f72e326bca5d",
    "_kg_hide-input": true,
    "_uuid": "68b42e92df59f10fbd3ba700389796c4506af604"
   },
   "outputs": [],
   "source": [
    "print('Distribution of the Classes in the subsample dataset')\n",
    "print(new_df['Class'].value_counts()/len(new_df))\n",
    "\n",
    "\n",
    "\n",
    "sns.countplot(x='Class', data=new_df, palette=colors)\n",
    "plt.title('Equally Distributed Classes', fontsize=14)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "9f353623-9435-4bb2-b854-b4a201ec7dd9",
    "_kg_hide-input": true,
    "_uuid": "e2f417c5d7c633a1e3cdfaa78acd6bd77a38400e"
   },
   "outputs": [],
   "source": [
    "# Make sure we use the subsample in our correlation\n",
    "\n",
    "f, (ax1, ax2) = plt.subplots(2, 1, figsize=(24,20))\n",
    "\n",
    "# Entire DataFrame\n",
    "corr = df.corr()\n",
    "sns.heatmap(corr, cmap='coolwarm_r', annot_kws={'size':20}, ax=ax1)\n",
    "ax1.set_title(\"Imbalanced Correlation Matrix \\n (don't use for reference)\", fontsize=14)\n",
    "\n",
    "\n",
    "sub_sample_corr = new_df.corr()\n",
    "sns.heatmap(sub_sample_corr, cmap='coolwarm_r', annot_kws={'size':20}, ax=ax2)\n",
    "ax2.set_title('SubSample Correlation Matrix \\n (use for reference)', fontsize=14)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "2f02c21f-daa3-4251-a8e9-acad09a5ce0f",
    "_kg_hide-input": true,
    "_uuid": "318d0e7e0443f99139be21c00a7abc663be26385"
   },
   "outputs": [],
   "source": [
    "f, axes = plt.subplots(ncols=4, figsize=(20,4))\n",
    "\n",
    "# Negative Correlations with our Class (The lower our feature value the more likely it will be a fraud transaction)\n",
    "sns.boxplot(x=\"Class\", y=\"V17\", data=new_df, palette=colors, ax=axes[0])\n",
    "axes[0].set_title('V17 vs Class Negative Correlation')\n",
    "\n",
    "sns.boxplot(x=\"Class\", y=\"V14\", data=new_df, palette=colors, ax=axes[1])\n",
    "axes[1].set_title('V14 vs Class Negative Correlation')\n",
    "\n",
    "\n",
    "sns.boxplot(x=\"Class\", y=\"V12\", data=new_df, palette=colors, ax=axes[2])\n",
    "axes[2].set_title('V12 vs Class Negative Correlation')\n",
    "\n",
    "\n",
    "sns.boxplot(x=\"Class\", y=\"V10\", data=new_df, palette=colors, ax=axes[3])\n",
    "axes[3].set_title('V10 vs Class Negative Correlation')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "b457b10e-c17c-4cb2-9719-6d4128377c9f",
    "_kg_hide-input": true,
    "_uuid": "7bfc46c028f8602ee949de83629082633aa47b2c"
   },
   "outputs": [],
   "source": [
    "f, axes = plt.subplots(ncols=4, figsize=(20,4))\n",
    "\n",
    "# Positive correlations (The higher the feature the probability increases that it will be a fraud transaction)\n",
    "sns.boxplot(x=\"Class\", y=\"V11\", data=new_df, palette=colors, ax=axes[0])\n",
    "axes[0].set_title('V11 vs Class Positive Correlation')\n",
    "\n",
    "sns.boxplot(x=\"Class\", y=\"V4\", data=new_df, palette=colors, ax=axes[1])\n",
    "axes[1].set_title('V4 vs Class Positive Correlation')\n",
    "\n",
    "\n",
    "sns.boxplot(x=\"Class\", y=\"V2\", data=new_df, palette=colors, ax=axes[2])\n",
    "axes[2].set_title('V2 vs Class Positive Correlation')\n",
    "\n",
    "\n",
    "sns.boxplot(x=\"Class\", y=\"V19\", data=new_df, palette=colors, ax=axes[3])\n",
    "axes[3].set_title('V19 vs Class Positive Correlation')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "9c690dfa-8fed-44e5-99f5-ff4eb6f87f16",
    "_kg_hide-input": true,
    "_uuid": "b6963900379db5b0d4adf92f8c7f959164e9119f"
   },
   "outputs": [],
   "source": [
    "from scipy.stats import norm\n",
    "\n",
    "f, (ax1, ax2, ax3) = plt.subplots(1,3, figsize=(20, 6))\n",
    "\n",
    "v14_fraud_dist = new_df['V14'].loc[new_df['Class'] == 1].values\n",
    "sns.distplot(v14_fraud_dist,ax=ax1, fit=norm, color='#FB8861')\n",
    "ax1.set_title('V14 Distribution \\n (Fraud Transactions)', fontsize=14)\n",
    "\n",
    "v12_fraud_dist = new_df['V12'].loc[new_df['Class'] == 1].values\n",
    "sns.distplot(v12_fraud_dist,ax=ax2, fit=norm, color='#56F9BB')\n",
    "ax2.set_title('V12 Distribution \\n (Fraud Transactions)', fontsize=14)\n",
    "\n",
    "\n",
    "v10_fraud_dist = new_df['V10'].loc[new_df['Class'] == 1].values\n",
    "sns.distplot(v10_fraud_dist,ax=ax3, fit=norm, color='#C5B3F9')\n",
    "ax3.set_title('V10 Distribution \\n (Fraud Transactions)', fontsize=14)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "2e19fe33-f85a-4ffd-8e4a-807d0e0fb992",
    "_kg_hide-input": true,
    "_uuid": "21e43406e62a9561fba2f065ce15a8d87a1bf389"
   },
   "outputs": [],
   "source": [
    "# # -----> V14 Removing Outliers (Highest Negative Correlated with Labels)\n",
    "v14_fraud = new_df['V14'].loc[new_df['Class'] == 1].values\n",
    "q25, q75 = np.percentile(v14_fraud, 25), np.percentile(v14_fraud, 75)\n",
    "print('Quartile 25: {} | Quartile 75: {}'.format(q25, q75))\n",
    "v14_iqr = q75 - q25\n",
    "print('iqr: {}'.format(v14_iqr))\n",
    "\n",
    "v14_cut_off = v14_iqr * 1.5\n",
    "v14_lower, v14_upper = q25 - v14_cut_off, q75 + v14_cut_off\n",
    "print('Cut Off: {}'.format(v14_cut_off))\n",
    "print('V14 Lower: {}'.format(v14_lower))\n",
    "print('V14 Upper: {}'.format(v14_upper))\n",
    "\n",
    "outliers = [x for x in v14_fraud if x < v14_lower or x > v14_upper]\n",
    "print('Feature V14 Outliers for Fraud Cases: {}'.format(len(outliers)))\n",
    "print('V10 outliers:{}'.format(outliers))\n",
    "\n",
    "new_df = new_df.drop(new_df[(new_df['V14'] > v14_upper) | (new_df['V14'] < v14_lower)].index)\n",
    "print('----' * 44)\n",
    "\n",
    "# -----> V12 removing outliers from fraud transactions\n",
    "v12_fraud = new_df['V12'].loc[new_df['Class'] == 1].values\n",
    "q25, q75 = np.percentile(v12_fraud, 25), np.percentile(v12_fraud, 75)\n",
    "v12_iqr = q75 - q25\n",
    "\n",
    "v12_cut_off = v12_iqr * 1.5\n",
    "v12_lower, v12_upper = q25 - v12_cut_off, q75 + v12_cut_off\n",
    "print('V12 Lower: {}'.format(v12_lower))\n",
    "print('V12 Upper: {}'.format(v12_upper))\n",
    "outliers = [x for x in v12_fraud if x < v12_lower or x > v12_upper]\n",
    "print('V12 outliers: {}'.format(outliers))\n",
    "print('Feature V12 Outliers for Fraud Cases: {}'.format(len(outliers)))\n",
    "new_df = new_df.drop(new_df[(new_df['V12'] > v12_upper) | (new_df['V12'] < v12_lower)].index)\n",
    "print('Number of Instances after outliers removal: {}'.format(len(new_df)))\n",
    "print('----' * 44)\n",
    "\n",
    "\n",
    "# Removing outliers V10 Feature\n",
    "v10_fraud = new_df['V10'].loc[new_df['Class'] == 1].values\n",
    "q25, q75 = np.percentile(v10_fraud, 25), np.percentile(v10_fraud, 75)\n",
    "v10_iqr = q75 - q25\n",
    "\n",
    "v10_cut_off = v10_iqr * 1.5\n",
    "v10_lower, v10_upper = q25 - v10_cut_off, q75 + v10_cut_off\n",
    "print('V10 Lower: {}'.format(v10_lower))\n",
    "print('V10 Upper: {}'.format(v10_upper))\n",
    "outliers = [x for x in v10_fraud if x < v10_lower or x > v10_upper]\n",
    "print('V10 outliers: {}'.format(outliers))\n",
    "print('Feature V10 Outliers for Fraud Cases: {}'.format(len(outliers)))\n",
    "new_df = new_df.drop(new_df[(new_df['V10'] > v10_upper) | (new_df['V10'] < v10_lower)].index)\n",
    "print('Number of Instances after outliers removal: {}'.format(len(new_df)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "66e44398-7c91-4cce-9778-4512cb838973",
    "_kg_hide-input": true,
    "_uuid": "ac80d9cfb07f1865094a8d460ae801750e93d694"
   },
   "outputs": [],
   "source": [
    "f,(ax1, ax2, ax3) = plt.subplots(1, 3, figsize=(20,6))\n",
    "\n",
    "colors = ['#B3F9C5', '#f9c5b3']\n",
    "# Boxplots with outliers removed\n",
    "# Feature V14\n",
    "sns.boxplot(x=\"Class\", y=\"V14\", data=new_df,ax=ax1, palette=colors)\n",
    "ax1.set_title(\"V14 Feature \\n Reduction of outliers\", fontsize=14)\n",
    "ax1.annotate('Fewer extreme \\n outliers', xy=(0.98, -17.5), xytext=(0, -12),\n",
    "            arrowprops=dict(facecolor='black'),\n",
    "            fontsize=14)\n",
    "\n",
    "# Feature 12\n",
    "sns.boxplot(x=\"Class\", y=\"V12\", data=new_df, ax=ax2, palette=colors)\n",
    "ax2.set_title(\"V12 Feature \\n Reduction of outliers\", fontsize=14)\n",
    "ax2.annotate('Fewer extreme \\n outliers', xy=(0.98, -17.3), xytext=(0, -12),\n",
    "            arrowprops=dict(facecolor='black'),\n",
    "            fontsize=14)\n",
    "\n",
    "# Feature V10\n",
    "sns.boxplot(x=\"Class\", y=\"V10\", data=new_df, ax=ax3, palette=colors)\n",
    "ax3.set_title(\"V10 Feature \\n Reduction of outliers\", fontsize=14)\n",
    "ax3.annotate('Fewer extreme \\n outliers', xy=(0.95, -16.5), xytext=(0, -12),\n",
    "            arrowprops=dict(facecolor='black'),\n",
    "            fontsize=14)\n",
    "\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "f83cde6b-90d0-4e9d-ac63-fb69780431b2",
    "_kg_hide-input": true,
    "_uuid": "af3027e7df67b75c92c88d597003632e285c9bff"
   },
   "outputs": [],
   "source": [
    "# New_df is from the random undersample data (fewer instances)\n",
    "X = new_df.drop('Class', axis=1)\n",
    "y = new_df['Class']\n",
    "\n",
    "\n",
    "# T-SNE Implementation\n",
    "t0 = time.time()\n",
    "X_reduced_tsne = TSNE(n_components=2, random_state=42).fit_transform(X.values)\n",
    "t1 = time.time()\n",
    "print(\"T-SNE took {:.2} s\".format(t1 - t0))\n",
    "\n",
    "# PCA Implementation\n",
    "t0 = time.time()\n",
    "X_reduced_pca = PCA(n_components=2, random_state=42).fit_transform(X.values)\n",
    "t1 = time.time()\n",
    "print(\"PCA took {:.2} s\".format(t1 - t0))\n",
    "\n",
    "# TruncatedSVD\n",
    "t0 = time.time()\n",
    "X_reduced_svd = TruncatedSVD(n_components=2, algorithm='randomized', random_state=42).fit_transform(X.values)\n",
    "t1 = time.time()\n",
    "print(\"Truncated SVD took {:.2} s\".format(t1 - t0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "07015ae5-f7ac-4d64-8f41-1e4b7c9dd2ac",
    "_kg_hide-input": true,
    "_uuid": "084f2a7421c2212082491d2a90e65d65c52b434a"
   },
   "outputs": [],
   "source": [
    "f, (ax1, ax2, ax3) = plt.subplots(1, 3, figsize=(24,6))\n",
    "# labels = ['No Fraud', 'Fraud']\n",
    "f.suptitle('Clusters using Dimensionality Reduction', fontsize=14)\n",
    "\n",
    "\n",
    "blue_patch = mpatches.Patch(color='#0A0AFF', label='No Fraud')\n",
    "red_patch = mpatches.Patch(color='#AF0000', label='Fraud')\n",
    "\n",
    "\n",
    "# t-SNE scatter plot\n",
    "ax1.scatter(X_reduced_tsne[:,0], X_reduced_tsne[:,1], c=(y == 0), cmap='coolwarm', label='No Fraud', linewidths=2)\n",
    "ax1.scatter(X_reduced_tsne[:,0], X_reduced_tsne[:,1], c=(y == 1), cmap='coolwarm', label='Fraud', linewidths=2)\n",
    "ax1.set_title('t-SNE', fontsize=14)\n",
    "\n",
    "ax1.grid(True)\n",
    "\n",
    "ax1.legend(handles=[blue_patch, red_patch])\n",
    "\n",
    "\n",
    "# PCA scatter plot\n",
    "ax2.scatter(X_reduced_pca[:,0], X_reduced_pca[:,1], c=(y == 0), cmap='coolwarm', label='No Fraud', linewidths=2)\n",
    "ax2.scatter(X_reduced_pca[:,0], X_reduced_pca[:,1], c=(y == 1), cmap='coolwarm', label='Fraud', linewidths=2)\n",
    "ax2.set_title('PCA', fontsize=14)\n",
    "\n",
    "ax2.grid(True)\n",
    "\n",
    "ax2.legend(handles=[blue_patch, red_patch])\n",
    "\n",
    "# TruncatedSVD scatter plot\n",
    "ax3.scatter(X_reduced_svd[:,0], X_reduced_svd[:,1], c=(y == 0), cmap='coolwarm', label='No Fraud', linewidths=2)\n",
    "ax3.scatter(X_reduced_svd[:,0], X_reduced_svd[:,1], c=(y == 1), cmap='coolwarm', label='Fraud', linewidths=2)\n",
    "ax3.set_title('Truncated SVD', fontsize=14)\n",
    "\n",
    "ax3.grid(True)\n",
    "\n",
    "ax3.legend(handles=[blue_patch, red_patch])\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "85ce8738-7599-4b06-a722-5c0ed073599b",
    "_kg_hide-input": true,
    "_uuid": "e3751d88766a982119e522e27a9c0c647f20af85"
   },
   "outputs": [],
   "source": [
    "# Undersampling before cross validating (prone to overfit)\n",
    "X = new_df.drop('Class', axis=1)\n",
    "y = new_df['Class']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "288a65b7-8b86-44b1-973d-38dbcfe82bbb",
    "_kg_hide-input": true,
    "_uuid": "fb0a479efaa7147d6702c2c24083f1118621863f"
   },
   "outputs": [],
   "source": [
    "# Our data is already scaled we should split our training and test sets\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# This is explicitly used for undersampling.\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "bccd5685-a979-451e-85b3-1cb968523540",
    "_kg_hide-input": true,
    "_uuid": "28f5178089d2d133b9e7478c1c7dc7a1f98aabee"
   },
   "outputs": [],
   "source": [
    "# Turn the values into an array for feeding the classification algorithms.\n",
    "X_train = X_train.values\n",
    "X_test = X_test.values\n",
    "y_train = y_train.values\n",
    "y_test = y_test.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "7810d0b9-b4e5-4b7f-909b-c127365b167c",
    "_kg_hide-input": true,
    "_uuid": "8dd4ea07fd60973fccabc2d46af28a09b0de9178"
   },
   "outputs": [],
   "source": [
    "# Let's implement simple classifiers\n",
    "\n",
    "classifiers = {\n",
    "    \"LogisiticRegression\": LogisticRegression(),\n",
    "    \"KNearest\": KNeighborsClassifier(),\n",
    "    \"Support Vector Classifier\": SVC(),\n",
    "    \"DecisionTreeClassifier\": DecisionTreeClassifier()\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "eb37c0f6-9cfe-48b6-92d3-475d5e6767a6",
    "_kg_hide-input": true,
    "_uuid": "fe129af379caccc5428cf1836e6c96bd32e68feb"
   },
   "outputs": [],
   "source": [
    "# Wow our scores are getting even high scores even when applying cross validation.\n",
    "from sklearn.model_selection import cross_val_score\n",
    "\n",
    "\n",
    "for key, classifier in classifiers.items():\n",
    "    classifier.fit(X_train, y_train)\n",
    "    training_score = cross_val_score(classifier, X_train, y_train, cv=5)\n",
    "    print(\"Classifiers: \", classifier.__class__.__name__, \"Has a training score of\", round(training_score.mean(), 2) * 100, \"% accuracy score\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "a1c35773-f4c7-4caf-9911-532784c9eae0",
    "_kg_hide-input": true,
    "_uuid": "d15b1ab16737358806e34c48dc57aa238cf0cfd2"
   },
   "outputs": [],
   "source": [
    "# Use GridSearchCV to find the best parameters.\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "\n",
    "# Logistic Regression \n",
    "log_reg_params = {\"penalty\": ['l1', 'l2'], 'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]}\n",
    "\n",
    "\n",
    "\n",
    "grid_log_reg = GridSearchCV(LogisticRegression(), log_reg_params)\n",
    "grid_log_reg.fit(X_train, y_train)\n",
    "# We automatically get the logistic regression with the best parameters.\n",
    "log_reg = grid_log_reg.best_estimator_\n",
    "\n",
    "knears_params = {\"n_neighbors\": list(range(2,5,1)), 'algorithm': ['auto', 'ball_tree', 'kd_tree', 'brute']}\n",
    "\n",
    "grid_knears = GridSearchCV(KNeighborsClassifier(), knears_params)\n",
    "grid_knears.fit(X_train, y_train)\n",
    "# KNears best estimator\n",
    "knears_neighbors = grid_knears.best_estimator_\n",
    "\n",
    "# Support Vector Classifier\n",
    "svc_params = {'C': [0.5, 0.7, 0.9, 1], 'kernel': ['rbf', 'poly', 'sigmoid', 'linear']}\n",
    "grid_svc = GridSearchCV(SVC(), svc_params)\n",
    "grid_svc.fit(X_train, y_train)\n",
    "\n",
    "# SVC best estimator\n",
    "svc = grid_svc.best_estimator_\n",
    "\n",
    "# DecisionTree Classifier\n",
    "tree_params = {\"criterion\": [\"gini\", \"entropy\"], \"max_depth\": list(range(2,4,1)), \n",
    "              \"min_samples_leaf\": list(range(5,7,1))}\n",
    "grid_tree = GridSearchCV(DecisionTreeClassifier(), tree_params)\n",
    "grid_tree.fit(X_train, y_train)\n",
    "\n",
    "# tree best estimator\n",
    "tree_clf = grid_tree.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "7f327bcd-335f-4e49-af07-fc4214dbcbdc",
    "_kg_hide-input": true,
    "_uuid": "1b2108bf377b924ed8a6efe580d9e162a132cd9e"
   },
   "outputs": [],
   "source": [
    "# Overfitting Case\n",
    "\n",
    "log_reg_score = cross_val_score(log_reg, X_train, y_train, cv=5)\n",
    "print('Logistic Regression Cross Validation Score: ', round(log_reg_score.mean() * 100, 2).astype(str) + '%')\n",
    "\n",
    "\n",
    "knears_score = cross_val_score(knears_neighbors, X_train, y_train, cv=5)\n",
    "print('Knears Neighbors Cross Validation Score', round(knears_score.mean() * 100, 2).astype(str) + '%')\n",
    "\n",
    "svc_score = cross_val_score(svc, X_train, y_train, cv=5)\n",
    "print('Support Vector Classifier Cross Validation Score', round(svc_score.mean() * 100, 2).astype(str) + '%')\n",
    "\n",
    "tree_score = cross_val_score(tree_clf, X_train, y_train, cv=5)\n",
    "print('DecisionTree Classifier Cross Validation Score', round(tree_score.mean() * 100, 2).astype(str) + '%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "38e430ef-0160-47a1-9b6f-11ff62c5ecc0",
    "_kg_hide-input": true,
    "_uuid": "eeb5736b279bb8fa3804689a175394f216ec4f72"
   },
   "outputs": [],
   "source": [
    "# We will undersample during cross validating\n",
    "undersample_X = df.drop('Class', axis=1)\n",
    "undersample_y = df['Class']\n",
    "\n",
    "for train_index, test_index in sss.split(undersample_X, undersample_y):\n",
    "    print(\"Train:\", train_index, \"Test:\", test_index)\n",
    "    undersample_Xtrain, undersample_Xtest = undersample_X.iloc[train_index], undersample_X.iloc[test_index]\n",
    "    undersample_ytrain, undersample_ytest = undersample_y.iloc[train_index], undersample_y.iloc[test_index]\n",
    "    \n",
    "undersample_Xtrain = undersample_Xtrain.values\n",
    "undersample_Xtest = undersample_Xtest.values\n",
    "undersample_ytrain = undersample_ytrain.values\n",
    "undersample_ytest = undersample_ytest.values \n",
    "\n",
    "undersample_accuracy = []\n",
    "undersample_precision = []\n",
    "undersample_recall = []\n",
    "undersample_f1 = []\n",
    "undersample_auc = []\n",
    "\n",
    "# Implementing NearMiss Technique \n",
    "# Distribution of NearMiss (Just to see how it distributes the labels we won't use these variables)\n",
    "#X_nearmiss, y_nearmiss = NearMiss().fit_sample(undersample_X.values, undersample_y.values)\n",
    "#print('NearMiss Label Distribution: {}'.format(Counter(y_nearmiss)))\n",
    "# Cross Validating the right way\n",
    "\n",
    "for train, test in sss.split(undersample_Xtrain, undersample_ytrain):\n",
    "    undersample_pipeline = imbalanced_make_pipeline(NearMiss(sampling_strategy='majority'), log_reg) # SMOTE happens during Cross Validation not before..\n",
    "    undersample_model = undersample_pipeline.fit(undersample_Xtrain[train], undersample_ytrain[train])\n",
    "    undersample_prediction = undersample_model.predict(undersample_Xtrain[test])\n",
    "    \n",
    "    undersample_accuracy.append(undersample_pipeline.score(original_Xtrain[test], original_ytrain[test]))\n",
    "    undersample_precision.append(precision_score(original_ytrain[test], undersample_prediction))\n",
    "    undersample_recall.append(recall_score(original_ytrain[test], undersample_prediction))\n",
    "    undersample_f1.append(f1_score(original_ytrain[test], undersample_prediction))\n",
    "    undersample_auc.append(roc_auc_score(original_ytrain[test], undersample_prediction))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "bb72803c-3ea3-40cd-8ac3-399540ab7f5a",
    "_kg_hide-input": true,
    "_uuid": "a12fb2f7e104931bb78e1bd6cfc5a516c970708b"
   },
   "outputs": [],
   "source": [
    "# Let's Plot LogisticRegression Learning Curve\n",
    "from sklearn.model_selection import ShuffleSplit\n",
    "from sklearn.model_selection import learning_curve\n",
    "\n",
    "def plot_learning_curve(estimator1, estimator2, estimator3, estimator4, X, y, ylim=None, cv=None,\n",
    "                        n_jobs=1, train_sizes=np.linspace(.1, 1.0, 5)):\n",
    "    f, ((ax1, ax2), (ax3, ax4)) = plt.subplots(2,2, figsize=(20,14), sharey=True)\n",
    "    if ylim is not None:\n",
    "        plt.ylim(*ylim)\n",
    "    # First Estimator\n",
    "    train_sizes, train_scores, test_scores = learning_curve(\n",
    "        estimator1, X, y, cv=cv, n_jobs=n_jobs, train_sizes=train_sizes)\n",
    "    train_scores_mean = np.mean(train_scores, axis=1)\n",
    "    train_scores_std = np.std(train_scores, axis=1)\n",
    "    test_scores_mean = np.mean(test_scores, axis=1)\n",
    "    test_scores_std = np.std(test_scores, axis=1)\n",
    "    ax1.fill_between(train_sizes, train_scores_mean - train_scores_std,\n",
    "                     train_scores_mean + train_scores_std, alpha=0.1,\n",
    "                     color=\"#ff9124\")\n",
    "    ax1.fill_between(train_sizes, test_scores_mean - test_scores_std,\n",
    "                     test_scores_mean + test_scores_std, alpha=0.1, color=\"#2492ff\")\n",
    "    ax1.plot(train_sizes, train_scores_mean, 'o-', color=\"#ff9124\",\n",
    "             label=\"Training score\")\n",
    "    ax1.plot(train_sizes, test_scores_mean, 'o-', color=\"#2492ff\",\n",
    "             label=\"Cross-validation score\")\n",
    "    ax1.set_title(\"Logistic Regression Learning Curve\", fontsize=14)\n",
    "    ax1.set_xlabel('Training size (m)')\n",
    "    ax1.set_ylabel('Score')\n",
    "    ax1.grid(True)\n",
    "    ax1.legend(loc=\"best\")\n",
    "    \n",
    "    # Second Estimator \n",
    "    train_sizes, train_scores, test_scores = learning_curve(\n",
    "        estimator2, X, y, cv=cv, n_jobs=n_jobs, train_sizes=train_sizes)\n",
    "    train_scores_mean = np.mean(train_scores, axis=1)\n",
    "    train_scores_std = np.std(train_scores, axis=1)\n",
    "    test_scores_mean = np.mean(test_scores, axis=1)\n",
    "    test_scores_std = np.std(test_scores, axis=1)\n",
    "    ax2.fill_between(train_sizes, train_scores_mean - train_scores_std,\n",
    "                     train_scores_mean + train_scores_std, alpha=0.1,\n",
    "                     color=\"#ff9124\")\n",
    "    ax2.fill_between(train_sizes, test_scores_mean - test_scores_std,\n",
    "                     test_scores_mean + test_scores_std, alpha=0.1, color=\"#2492ff\")\n",
    "    ax2.plot(train_sizes, train_scores_mean, 'o-', color=\"#ff9124\",\n",
    "             label=\"Training score\")\n",
    "    ax2.plot(train_sizes, test_scores_mean, 'o-', color=\"#2492ff\",\n",
    "             label=\"Cross-validation score\")\n",
    "    ax2.set_title(\"Knears Neighbors Learning Curve\", fontsize=14)\n",
    "    ax2.set_xlabel('Training size (m)')\n",
    "    ax2.set_ylabel('Score')\n",
    "    ax2.grid(True)\n",
    "    ax2.legend(loc=\"best\")\n",
    "    \n",
    "    # Third Estimator\n",
    "    train_sizes, train_scores, test_scores = learning_curve(\n",
    "        estimator3, X, y, cv=cv, n_jobs=n_jobs, train_sizes=train_sizes)\n",
    "    train_scores_mean = np.mean(train_scores, axis=1)\n",
    "    train_scores_std = np.std(train_scores, axis=1)\n",
    "    test_scores_mean = np.mean(test_scores, axis=1)\n",
    "    test_scores_std = np.std(test_scores, axis=1)\n",
    "    ax3.fill_between(train_sizes, train_scores_mean - train_scores_std,\n",
    "                     train_scores_mean + train_scores_std, alpha=0.1,\n",
    "                     color=\"#ff9124\")\n",
    "    ax3.fill_between(train_sizes, test_scores_mean - test_scores_std,\n",
    "                     test_scores_mean + test_scores_std, alpha=0.1, color=\"#2492ff\")\n",
    "    ax3.plot(train_sizes, train_scores_mean, 'o-', color=\"#ff9124\",\n",
    "             label=\"Training score\")\n",
    "    ax3.plot(train_sizes, test_scores_mean, 'o-', color=\"#2492ff\",\n",
    "             label=\"Cross-validation score\")\n",
    "    ax3.set_title(\"Support Vector Classifier \\n Learning Curve\", fontsize=14)\n",
    "    ax3.set_xlabel('Training size (m)')\n",
    "    ax3.set_ylabel('Score')\n",
    "    ax3.grid(True)\n",
    "    ax3.legend(loc=\"best\")\n",
    "    \n",
    "    # Fourth Estimator\n",
    "    train_sizes, train_scores, test_scores = learning_curve(\n",
    "        estimator4, X, y, cv=cv, n_jobs=n_jobs, train_sizes=train_sizes)\n",
    "    train_scores_mean = np.mean(train_scores, axis=1)\n",
    "    train_scores_std = np.std(train_scores, axis=1)\n",
    "    test_scores_mean = np.mean(test_scores, axis=1)\n",
    "    test_scores_std = np.std(test_scores, axis=1)\n",
    "    ax4.fill_between(train_sizes, train_scores_mean - train_scores_std,\n",
    "                     train_scores_mean + train_scores_std, alpha=0.1,\n",
    "                     color=\"#ff9124\")\n",
    "    ax4.fill_between(train_sizes, test_scores_mean - test_scores_std,\n",
    "                     test_scores_mean + test_scores_std, alpha=0.1, color=\"#2492ff\")\n",
    "    ax4.plot(train_sizes, train_scores_mean, 'o-', color=\"#ff9124\",\n",
    "             label=\"Training score\")\n",
    "    ax4.plot(train_sizes, test_scores_mean, 'o-', color=\"#2492ff\",\n",
    "             label=\"Cross-validation score\")\n",
    "    ax4.set_title(\"Decision Tree Classifier \\n Learning Curve\", fontsize=14)\n",
    "    ax4.set_xlabel('Training size (m)')\n",
    "    ax4.set_ylabel('Score')\n",
    "    ax4.grid(True)\n",
    "    ax4.legend(loc=\"best\")\n",
    "    return plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "780e485a-ea64-48a0-ad97-a7516b047f32",
    "_kg_hide-input": true,
    "_uuid": "fdd59bf2c7a8e61cfb401142570643e8a29cf86b"
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_curve\n",
    "from sklearn.model_selection import cross_val_predict\n",
    "# Create a DataFrame with all the scores and the classifiers names.\n",
    "\n",
    "log_reg_pred = cross_val_predict(log_reg, X_train, y_train, cv=5,\n",
    "                             method=\"decision_function\")\n",
    "\n",
    "knears_pred = cross_val_predict(knears_neighbors, X_train, y_train, cv=5)\n",
    "\n",
    "svc_pred = cross_val_predict(svc, X_train, y_train, cv=5,\n",
    "                             method=\"decision_function\")\n",
    "\n",
    "tree_pred = cross_val_predict(tree_clf, X_train, y_train, cv=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "57c211c6-e88f-4634-b321-4949df08815d",
    "_kg_hide-input": true,
    "_uuid": "cb2e4715e91e36f2029ef2a5c241991ff162cd9f"
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "print('Logistic Regression: ', roc_auc_score(y_train, log_reg_pred))\n",
    "print('KNears Neighbors: ', roc_auc_score(y_train, knears_pred))\n",
    "print('Support Vector Classifier: ', roc_auc_score(y_train, svc_pred))\n",
    "print('Decision Tree Classifier: ', roc_auc_score(y_train, tree_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "_cell_guid": "89b0b9b6-ef82-4b69-9517-e89a79696dbb",
    "_kg_hide-input": true,
    "_uuid": "9d57aad23f3f72f3c45bf80b089a65acbce2a9ab"
   },
   "outputs": [],
   "source": [
    "log_fpr, log_tpr, log_thresold = roc_curve(y_train, log_reg_pred)\n",
    "knear_fpr, knear_tpr, knear_threshold = roc_curve(y_train, knears_pred)\n",
    "svc_fpr, svc_tpr, svc_threshold = roc_curve(y_train, svc_pred)\n",
    "tree_fpr, tree_tpr, tree_threshold = roc_curve(y_train, tree_pred)\n",
    "\n",
    "\n",
    "def graph_roc_curve_multiple(log_fpr, log_tpr, knear_fpr, knear_tpr, svc_fpr, svc_tpr, tree_fpr, tree_tpr):\n",
    "    plt.figure(figsize=(16,8))\n",
    "    plt.title('ROC Curve \\n Top 4 Classifiers', fontsize=18)\n",
    "    plt.plot(log_fpr, log_tpr, label='Logistic Regression Classifier Score: {:.4f}'.format(roc_auc_score(y_train, log_reg_pred)))\n",
    "    plt.plot(knear_fpr, knear_tpr, label='KNears Neighbors Classifier Score: {:.4f}'.format(roc_auc_score(y_train, knears_pred)))\n",
    "    plt.plot(svc_fpr, svc_tpr, label='Support Vector Classifier Score: {:.4f}'.format(roc_auc_score(y_train, svc_pred)))\n",
    "    plt.plot(tree_fpr, tree_tpr, label='Decision Tree Classifier Score: {:.4f}'.format(roc_auc_score(y_train, tree_pred)))\n",
    "    plt.plot([0, 1], [0, 1], 'k--')\n",
    "    plt.axis([-0.01, 1, 0, 1])\n",
    "    plt.xlabel('False Positive Rate', fontsize=16)\n",
    "    plt.ylabel('True Positive Rate', fontsize=16)\n",
    "    plt.annotate('Minimum ROC Score of 50% \\n (This is the minimum score to get)', xy=(0.5, 0.5), xytext=(0.6, 0.3),\n",
    "                arrowprops=dict(facecolor='#6E726D', shrink=0.05),\n",
    "                )\n",
    "    plt.legend()\n",
    "    \n",
    "graph_roc_curve_multiple(log_fpr, log_tpr, knear_fpr, knear_tpr, svc_fpr, svc_tpr, tree_fpr, tree_tpr)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!rm data/creditcard.csv"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}