MNIST.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_data = pd.read_csv('mnist_train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4\n"
     ]
    }
   ],
   "source": [
    "images = raw_data.iloc[:,1:].values\n",
    "images = images.astype(float)\n",
    "images = np.multiply(images, 1.0/255.0)\n",
    "size = int(np.sqrt(len(images[0])))\n",
    "images = [img.reshape((size, size)) for img in images]\n",
    "\n",
    "labels = raw_data.iloc[:,0].values\n",
    "print(labels[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7fbff477df70>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAANSUlEQVR4nO3db4wc9X3H8c/Hx9mOnaD4TH11jAOU4Ae0Uo/qMFX4UypSRFAqgxJZsZTElVAvD2IpSHkApa1ClQclURMatRHSBdw4VQpKlCD8gKQYCxWhRI4P4mIb00KoXewYn1MnsgnGf799cEN0wO3seWd2Z33f90ta3e58d3a+GvnjmZ3f7v4cEQIw981rugEAvUHYgSQIO5AEYQeSIOxAEhf0cmPzvSAWanEvNwmk8qZ+o5NxwjPVKoXd9i2Svi5pQNKDEXFf2fMXarGu8U1VNgmgxLbY2rLW8Wm87QFJ35D0UUlXSlpn+8pOXw9Ad1V5z75a0ssR8UpEnJT0iKQ19bQFoG5Vwr5C0qvTHu8vlr2N7THbE7YnTulEhc0BqKLrV+MjYjwiRiNidFALur05AC1UCfsBSSunPb64WAagD1UJ+3ZJV9i+zPZ8SZ+UtLmetgDUreOht4g4bXuDpH/X1NDbxojYXVtnAGpVaZw9Ih6X9HhNvQDoIj4uCyRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKVZnEF+tlvPnFNy9qXv/JA6bpfWvuZ0npM7OqopyZVCrvtvZKOSToj6XREjNbRFID61XFk/9OI+GUNrwOgi3jPDiRRNewh6Qnbz9oem+kJtsdsT9ieOKUTFTcHoFNVT+Ovi4gDtpdJ2mL7xYh4evoTImJc0rgkXeihqLg9AB2qdGSPiAPF30lJj0paXUdTAOrXcdhtL7b9vrfuS7pZ0vk3HgEkUeU0fljSo7bfep1/i4gf1dJVFxxfU37ScXzpQGl9aONP6mwHPTA52vpY9qW9f97DTvpDx2GPiFck/WGNvQDoIobegCQIO5AEYQeSIOxAEoQdSCLNV1x/cUP5/2uLLv91+QtsrK8X1GRe+XBpfPB4y9pNy14sXXerP9xRS/2MIzuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJJFmnP3vPva90vqX99zco05Ql4HLLymtv/gnrT8cMfLTT5Wu+4HtOzvqqZ9xZAeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJNKMsw/6dNMtoGYXPPhGx+se//mFNXZyfuDIDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJzJlx9rPXjZTWr1/4TG8aQc9cuvj/Ol535ZNnauzk/ND2yG57o+1J27umLRuyvcX2S8XfJd1tE0BVszmN/5akW96x7G5JWyPiCklbi8cA+ljbsEfE05KOvGPxGkmbivubJN1Wb1sA6tbpe/bhiDhY3H9N0nCrJ9oekzQmSQu1qMPNAaiq8tX4iAhJUVIfj4jRiBgd1IKqmwPQoU7Dfsj2ckkq/k7W1xKAbug07JslrS/ur5f0WD3tAOiWtu/ZbT8s6UZJF9neL+mLku6T9F3bd0jaJ2ltN5ucjX0fe09pfdkA1wvONxdc+sHS+ieGNnf82u/5n1+V1ufiKHzbsEfEuhalm2ruBUAX8XFZIAnCDiRB2IEkCDuQBGEHkpgzX3G94EPHKq3/5ovvr6cR1ObVf1xcWr92wdnS+kNHL25d/PXRTlo6r3FkB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEk5sw4e1XLJsrHbDGzgYuWltYPfXxVy9rQ2v2l6/7HqofabH1hafWBb9zWsrbs0I/bvPbcw5EdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5JgnL1wfKj8/73yb1ZXc/b6q0rrMeDS+qsfaT3TzskPnCpdd9788h9NfuL6fyqtD5a3ptfOtO7tb1+5vXTdI2fLP/uwaF5578PbWv/GQcspjOYwjuxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kMScGWc/8eZgaf1sm5HVf7nn/tL65g0j59rSrN219MHS+jyVD2Yfj5Mta784Uz4W/c+Hbyytf+TJO0vr7//Z/NL68icOtax5X/n32Q/vKZ+Ge3ig/DMEsX1naT2btkd22xttT9reNW3ZvbYP2N5R3G7tbpsAqprNafy3JN0yw/L7I2KkuD1eb1sA6tY27BHxtKQjPegFQBdVuUC3wfbzxWn+klZPsj1me8L2xCmdqLA5AFV0GvYHJF0uaUTSQUlfbfXEiBiPiNGIGB1U6y9FAOiujsIeEYci4kxEnJX0TUmr620LQN06Crvt5dMe3i5pV6vnAugPbcfZbT8s6UZJF9neL+mLkm60PaKprwXvlfTZ7rU4Ox/61M9K67//9xtK6yuvPlBnO+fkqcnWv60uSYd/WDLPuKSlu1uPN8//0fY2Wy8fq16liTbrlysb5T9w14dL1716wU9K64+8vqKDjvJqG/aIWDfD4na/3g+gz/BxWSAJwg4kQdiBJAg7kARhB5KYM19xbeeyvyofxulny/W/TbfQFYtuOFxp/b956uOl9VX6aaXXn2s4sgNJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEmnG2TH3XPJYxomXO8eRHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Lg++zoWwMuPxb9atVgaf13f1hnN+e/tkd22yttP2X7Bdu7bX++WD5ke4vtl4q/S7rfLoBOzeY0/rSkL0TElZL+WNLnbF8p6W5JWyPiCklbi8cA+lTbsEfEwYh4rrh/TNIeSSskrZG0qXjaJkm3dalHADU4p/fsti+VdJWkbZKGI+JgUXpN0nCLdcYkjUnSQi3quFEA1cz6arzt90r6vqQ7I+Lo9FpEhKQZf/0vIsYjYjQiRge1oFKzADo3q7DbHtRU0L8TET8oFh+yvbyoL5c02Z0WAdRhNlfjLekhSXsi4mvTSpslrS/ur5f0WP3tIbMzcbb0pnkqv+FtZvOe/VpJn5a00/aOYtk9ku6T9F3bd0jaJ2ltVzoEUIu2YY+IZyS5RfmmetsB0C2c7ABJEHYgCcIOJEHYgSQIO5AEX3HFeeuNq99ouoXzCkd2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCcXb0rXY/JY1zw94EkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQYZ0djTjz5O6X1MyNne9RJDhzZgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJR0T5E+yVkr4taVhSSBqPiK/bvlfSX0o6XDz1noh4vOy1LvRQXGMmfgW6ZVts1dE4MuOsy7P5UM1pSV+IiOdsv0/Ss7a3FLX7I+If6moUQPfMZn72g5IOFveP2d4jaUW3GwNQr3N6z277UklXSdpWLNpg+3nbG20vabHOmO0J2xOndKJatwA6Nuuw236vpO9LujMijkp6QNLlkkY0deT/6kzrRcR4RIxGxOigFlTvGEBHZhV224OaCvp3IuIHkhQRhyLiTESclfRNSau71yaAqtqG3bYlPSRpT0R8bdry5dOedrukXfW3B6Aus7kaf62kT0vaaXtHseweSetsj2hqOG6vpM92oT8ANZnN1fhnJM00blc6pg6gv/AJOiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBJtf0q61o3ZhyXtm7boIkm/7FkD56Zfe+vXviR661SdvV0SETPOhd3TsL9r4/ZERIw21kCJfu2tX/uS6K1TveqN03ggCcIOJNF02Mcb3n6Zfu2tX/uS6K1TPemt0ffsAHqn6SM7gB4h7EASjYTd9i22/8v2y7bvbqKHVmzvtb3T9g7bEw33stH2pO1d05YN2d5i+6Xi74xz7DXU2722DxT7boftWxvqbaXtp2y/YHu37c8XyxvddyV99WS/9fw9u+0BSf8t6c8k7Ze0XdK6iHihp420YHuvpNGIaPwDGLZvkPS6pG9HxB8Uy74i6UhE3Ff8R7kkIu7qk97ulfR609N4F7MVLZ8+zbik2yT9hRrcdyV9rVUP9lsTR/bVkl6OiFci4qSkRyStaaCPvhcRT0s68o7FayRtKu5v0tQ/lp5r0VtfiIiDEfFccf+YpLemGW9035X01RNNhH2FpFenPd6v/prvPSQ9YftZ22NNNzOD4Yg4WNx/TdJwk83MoO003r30jmnG+2bfdTL9eVVcoHu36yLijyR9VNLnitPVvhRT78H6aex0VtN498oM04z/VpP7rtPpz6tqIuwHJK2c9vjiYllfiIgDxd9JSY+q/6aiPvTWDLrF38mG+/mtfprGe6ZpxtUH+67J6c+bCPt2SVfYvsz2fEmflLS5gT7exfbi4sKJbC+WdLP6byrqzZLWF/fXS3qswV7epl+m8W41zbga3neNT38eET2/SbpVU1fkfy7pr5vooUVfvyfpP4vb7qZ7k/Swpk7rTmnq2sYdkpZK2irpJUlPShrqo97+VdJOSc9rKljLG+rtOk2doj8vaUdxu7XpfVfSV0/2Gx+XBZLgAh2QBGEHkiDsQBKEHUiCsANJEHYgCcIOJPH/OLDzSn+ERVIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(images[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cool, so our images are 28x28x1 arrays. This should be a little easier than RGB.\n",
    "\n",
    "Let's just rebuild our CNN from scratch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "class ConvLayer:\n",
    "    def __init__(self, num_filters, filter_size, strides=1):\n",
    "        self.num_filters = num_filters\n",
    "        self.strides = strides\n",
    "        self.filter_size = filter_size\n",
    "        self.filters = np.random.randn(num_filters, filter_size, filter_size) / (filter_size*filter_size)\n",
    "        \n",
    "    def patch_generator(self, image):\n",
    "        height, width = image.shape\n",
    "        self.image = image\n",
    "        \n",
    "        out_height, out_width, num_filters = self.out_size\n",
    "        \n",
    "        # iterate over image, yielding patches as we go\n",
    "        for i in range(out_height):\n",
    "            for j in range(out_width):\n",
    "                image_patch = image[i:i+self.filter_size, j:j+self.filter_size]\n",
    "                yield image_patch, i, j\n",
    "                \n",
    "    \n",
    "    def forward_prop(self, image):\n",
    "        height, width = image.shape\n",
    "        self.out_size = (int((height - self.filter_size) / self.strides + 1), int((width - self.filter_size) / self.strides + 1), self.num_filters)\n",
    "        conv_out = np.zeros(self.out_size)\n",
    "        \n",
    "        for image_patch, i, j in self.patch_generator(image):\n",
    "            conv_out[i, j] = np.sum(image_patch * self.filters, axis=(1,2))\n",
    "        \n",
    "        return conv_out\n",
    "    \n",
    "    \n",
    "    def backward_prop(self, dL_dout, learning_rate):\n",
    "        dL_dW_params = np.zeros(self.filters.shape)\n",
    "        \n",
    "        for image_patch, i, j in self.patch_generator(self.image):\n",
    "            for f in range(self.num_filters):\n",
    "                dL_dW_params[f] += image_patch * dL_dout[i,j,f]\n",
    "            \n",
    "        self.filters -= learning_rate*dL_dW_params\n",
    "        \n",
    "        return dL_dW_params\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(27, 27, 4)\n"
     ]
    }
   ],
   "source": [
    "# Quick forward propagation test\n",
    "\n",
    "D = ConvLayer(num_filters=4, filter_size=2)\n",
    "out = D.forward_prop(images[0])\n",
    "print(out.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MaxPoolLayer:\n",
    "    def __init__(self, filter_size):\n",
    "        self.filter_size = filter_size\n",
    "        \n",
    "    def patch_generator(self, image):\n",
    "        self.image = image\n",
    "        out_height, out_width, out_depth = self.out_size\n",
    "        \n",
    "        for i in range(out_height):\n",
    "            for j in range(out_width):\n",
    "                image_patch = image[(i*self.filter_size):(self.filter_size*(i+1)), (j*self.filter_size):(self.filter_size*(j+1))]\n",
    "                yield image_patch, i, j\n",
    "        \n",
    "    def forward_prop(self, image):\n",
    "        inp_height, inp_width, inp_num_filters = image.shape\n",
    "        self.out_size = ((inp_height // self.filter_size), (inp_width // self.filter_size), inp_num_filters)\n",
    "        out = np.zeros(self.out_size)\n",
    "        \n",
    "        for patch, i, j in self.patch_generator(image):\n",
    "            out[i, j] = np.amax(patch, axis=(0,1))\n",
    "        \n",
    "        return out\n",
    "    \n",
    "    def backward_prop(self, dL_dout):\n",
    "        dL_dMP = np.zeros(self.image.shape)\n",
    "        \n",
    "        for patch, i, j in self.patch_generator(self.image):\n",
    "            ph, pw, pf = patch.shape\n",
    "            max_val = np.amax(patch, axis=(0,1))\n",
    "            \n",
    "            # Here we only want the gradient that corresponds to the\n",
    "            # maximum value from the input image to be updated\n",
    "        \n",
    "            for h in range(ph):\n",
    "                for w in range(pw):\n",
    "                    for f in range(pf):\n",
    "                        if patch[h,w,f] == max_val[f]:\n",
    "                            dL_dMP[i*self.filter_size + h, j*self.filter_size + w, f] = dL_dout[i,j,f]\n",
    "            \n",
    "            return dL_dMP\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(13, 13, 4)\n"
     ]
    }
   ],
   "source": [
    "# Test out forward prop thru maxpool\n",
    "MP = MaxPoolLayer(2)\n",
    "mp_out = MP.forward_prop(out)\n",
    "print(mp_out.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SoftmaxLayer:\n",
    "    def __init__(self, num_inputs, num_classes):\n",
    "        self.weights = np.random.randn(num_inputs, num_classes) / num_inputs\n",
    "        self.biases = np.zeros(num_classes)\n",
    "        \n",
    "        \n",
    "    def forward_prop(self, image):\n",
    "        self.orig_img_shape = image.shape\n",
    "        flattened = image.flatten()\n",
    "        self.flattened = flattened\n",
    "        out = np.dot(flattened, self.weights) + self.biases\n",
    "        self.out = out\n",
    "        exp_out = np.exp(out)\n",
    "        \n",
    "        return exp_out / np.sum(exp_out, axis=0)\n",
    "    \n",
    "    def backward_prop(self, dL_dout, learning_rate):\n",
    "        \n",
    "        # The gradient here will be the -log of our predicted y\n",
    "        # The rest of the values will be 0\n",
    "        for i, gradient in enumerate(dL_dout):\n",
    "            if gradient == 0:\n",
    "                continue\n",
    "                \n",
    "            exp = np.exp(self.out)\n",
    "            S_total = np.sum(exp)\n",
    "            \n",
    "            # case 2: i != l\n",
    "            dy_dz = -exp[i] * exp / (S_total**2)\n",
    "            \n",
    "            # case 1: i == l (this is equiv. to y*(1-y))\n",
    "            dy_dz[i] = exp[i]*(S_total - exp[i]) / (S_total**2)\n",
    "            \n",
    "            # dz_dw, dz_db, dz_dinput\n",
    "            dz_dw = self.flattened\n",
    "            dz_db = 1\n",
    "            dz_dinput = self.weights\n",
    "            \n",
    "            # grad is dL_dy\n",
    "            # dL/dz = dL/dy * dy/dz\n",
    "            dL_dz = gradient * dy_dz\n",
    "            \n",
    "            # loss wrt. weights, biases, input\n",
    "            dL_dW = dz_dw[np.newaxis].T @ dL_dz[np.newaxis]\n",
    "            dL_db = dL_dz * dz_db\n",
    "            dL_dinput = dz_dinput @ dL_dz\n",
    "            \n",
    "            # update weights and biases\n",
    "            self.weights = self.weights - (learning_rate * dL_dW)\n",
    "            self.biases = self.biases - (learning_rate * dL_db)\n",
    "            \n",
    "            return dL_dinput.reshape(self.orig_img_shape)\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.10002007 0.09945614 0.10014249 0.09981342 0.09977065 0.09931503\n",
      " 0.10020608 0.10049994 0.10106636 0.09970982]\n"
     ]
    }
   ],
   "source": [
    "from math import prod\n",
    "\n",
    "SM = SoftmaxLayer(num_inputs=prod(mp_out.shape), num_classes=10)\n",
    "sm_out = SM.forward_prop(mp_out)\n",
    "print(sm_out)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## This softmax output is promising\n",
    "This is actually way more promising than the CIFAR CNN at this stage\n",
    "\n",
    "The output of the softmax *should* look like this (approx. equal probabilities because we have done zero training), and on the CIFAR it looked far more random than this.\n",
    "\n",
    "So maybe I have something wrong with my forward propagation on the other attempt."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(13, 13, 4)\n",
      "(27, 27, 4)\n",
      "(4, 2, 2)\n"
     ]
    }
   ],
   "source": [
    "# Moment of truth: backpropagation\n",
    "\n",
    "image_label = labels[0]\n",
    "num_classes = 10\n",
    "learning_rate = 0.005\n",
    "\n",
    "# Cross-entropy-loss\n",
    "def initial_gradient (sm_out, image_label):\n",
    "    cross_entropy_loss = -np.log(sm_out[image_label])\n",
    "    acc = 1 if np.argmax(sm_out) == image_label else 0\n",
    "\n",
    "    gradient = np.zeros(num_classes)\n",
    "    gradient[image_label] = -1 / sm_out[image_label]\n",
    "    \n",
    "    return gradient\n",
    "\n",
    "gradient = initial_gradient(sm_out, image_label)\n",
    "\n",
    "gradient = SM.backward_prop(gradient, learning_rate)\n",
    "print(gradient.shape)\n",
    "gradient = MP.backward_prop(gradient)\n",
    "print(gradient.shape)\n",
    "gradient = D.backward_prop(gradient, learning_rate)\n",
    "print(gradient.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0.10212861 0.09944136 0.10015389 0.09974955 0.09962152 0.1002198\n",
      " 0.09977817 0.09977813 0.09956042 0.09956855]\n",
      "(4, 2, 2)\n"
     ]
    }
   ],
   "source": [
    "# Second moment of truth: feeding in a 2nd image\n",
    "\n",
    "img2 = images[2]\n",
    "label = labels[2]\n",
    "\n",
    "fwd = D.forward_prop(img2)\n",
    "fwd = MP.forward_prop(fwd)\n",
    "fwd = SM.forward_prop(fwd)\n",
    "print(fwd)\n",
    "\n",
    "gradient = initial_gradient(fwd, label)\n",
    "\n",
    "back = SM.backward_prop(gradient, learning_rate)\n",
    "back = MP.backward_prop(back)\n",
    "back = D.backward_prop(back, learning_rate)\n",
    "print(back.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Alright! No real change but no exploding/vanishing gradient immediately, so we'll train on many images and see what happens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This is partly reused code from cnn_cifar.ipynb\n",
    "\n",
    "class CNNModel:\n",
    "    def __init__(self, layers, num_classes, learning_rate=0.005):\n",
    "        assert len(layers) >= 1\n",
    "        self.layers = layers\n",
    "        self.num_classes = num_classes\n",
    "        self.learning_rate = learning_rate\n",
    "        \n",
    "    def forward_prop(self, image, label):\n",
    "        out_forward = self.layers[0].forward_prop(image)\n",
    "        for layer in self.layers[1:]:\n",
    "            out_forward = layer.forward_prop(out_forward)\n",
    "        \n",
    "        cross_entropy_loss = -np.log(out_forward[label])\n",
    "        accuracy = 0\n",
    "        if np.argmax(out_forward) == label:\n",
    "            accuracy = 1\n",
    "        \n",
    "        return out_forward, cross_entropy_loss, accuracy\n",
    "    \n",
    "    def backward_prop(self, initial_gradient):\n",
    "        back_gradient = self.layers[-1].backward_prop(initial_gradient, self.learning_rate)\n",
    "        for layer in reversed(self.layers[:-1]):\n",
    "            if isinstance(layer, MaxPoolLayer):\n",
    "                back_gradient = layer.backward_prop(back_gradient)\n",
    "            else:\n",
    "                back_gradient = layer.backward_prop(back_gradient, self.learning_rate)\n",
    "    \n",
    "    def train_image(self, image, label):\n",
    "        \n",
    "        # Forward propagation\n",
    "        out_fw, loss, acc = self.forward_prop(image, label)\n",
    "        # calc initial gradient\n",
    "        gradient = np.zeros(self.num_classes)\n",
    "        gradient[label] = -1 / out_fw[label]\n",
    "        \n",
    "        # Backward propagation\n",
    "        self.backward_prop(gradient)\n",
    "        \n",
    "        return loss, acc\n",
    "    \n",
    "    def train(self, images, labels):\n",
    "        loss = 0\n",
    "        num_correct = 0\n",
    "        for i, (img, label) in enumerate(zip(images, labels)):\n",
    "            if i % 500 == 0:\n",
    "                print('after {num} images, loss={loss} and correct%={correct}'.format(num=i, loss=loss, correct=(num_correct/500)))\n",
    "                loss = 0\n",
    "                num_correct = 0\n",
    "            \n",
    "            l, a = self.train_image(img, label)\n",
    "            loss += l\n",
    "            num_correct += a\n",
    "            \n",
    "    def classify(self, image, label):\n",
    "        fwd_out, cel, acc = self.forward_prop(image, label)\n",
    "        predicted = np.argmax(fwd_out)\n",
    "        \n",
    "        return predicted\n",
    "    \n",
    "    def assess_model(self, test_images, test_labels):\n",
    "        predictions = np.zeros(len(test_images))\n",
    "        num_correct = 0\n",
    "        \n",
    "        for i, (test_img, test_lbl) in enumerate(zip(test_images, test_labels)):\n",
    "            predictions[i] = self.classify(test_img, test_lbl)\n",
    "            \n",
    "            if predictions[i] == test_lbl:\n",
    "                num_correct += 1\n",
    "        \n",
    "        correct_percent = num_correct / len(test_images)\n",
    "        \n",
    "        return correct_percent, predictions\n",
    "        \n",
    "\n",
    "model_test = CNNModel(\n",
    "    layers=[\n",
    "        ConvLayer(num_filters=4, filter_size=4),\n",
    "        MaxPoolLayer(2),\n",
    "        SoftmaxLayer(num_inputs=12*12*4, num_classes=10)\n",
    "    ],\n",
    "    num_classes=10,\n",
    "    learning_rate=0.005\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.3003869191251365 0\n"
     ]
    }
   ],
   "source": [
    "loss, acc = model_test.train_image(images[0], labels[0])\n",
    "print(loss, acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Let's set up some training and testing data\n",
    "# images contains 59999 images\n",
    "\n",
    "train_set = images[:50000]\n",
    "train_labels = labels[:50000]\n",
    "\n",
    "test_set = images[50000:]\n",
    "test_labels = labels[50000:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 0 images, loss=0 and correct%=0.0\n",
      "after 500 images, loss=1094.5585054706096 and correct%=0.484\n",
      "after 1000 images, loss=1015.6305741939198 and correct%=0.632\n",
      "after 1500 images, loss=936.6841563193602 and correct%=0.68\n",
      "after 2000 images, loss=850.0954918969437 and correct%=0.754\n",
      "after 2500 images, loss=780.7521894994605 and correct%=0.774\n",
      "after 3000 images, loss=730.898165698512 and correct%=0.762\n",
      "after 3500 images, loss=723.4627834139749 and correct%=0.77\n",
      "after 4000 images, loss=666.0953950831735 and correct%=0.794\n",
      "after 4500 images, loss=636.055588921022 and correct%=0.814\n",
      "after 5000 images, loss=583.6674990881046 and correct%=0.796\n",
      "after 5500 images, loss=550.0295889000178 and correct%=0.784\n",
      "after 6000 images, loss=551.7925508129504 and correct%=0.824\n",
      "after 6500 images, loss=481.26223038733224 and correct%=0.838\n",
      "after 7000 images, loss=511.5187493825767 and correct%=0.842\n",
      "after 7500 images, loss=569.7972468054193 and correct%=0.812\n",
      "after 8000 images, loss=507.8801211702941 and correct%=0.802\n",
      "after 8500 images, loss=489.5466217987073 and correct%=0.828\n",
      "after 9000 images, loss=474.9007408344954 and correct%=0.79\n",
      "after 9500 images, loss=435.2132379360337 and correct%=0.854\n",
      "after 10000 images, loss=408.60030437192177 and correct%=0.862\n",
      "after 10500 images, loss=394.2210282303252 and correct%=0.848\n",
      "after 11000 images, loss=392.12774009568835 and correct%=0.87\n",
      "after 11500 images, loss=378.4833098589691 and correct%=0.858\n",
      "after 12000 images, loss=437.1938833838499 and correct%=0.818\n",
      "after 12500 images, loss=400.94803109558563 and correct%=0.842\n",
      "after 13000 images, loss=429.2086349280894 and correct%=0.798\n",
      "after 13500 images, loss=400.4862773882708 and correct%=0.82\n",
      "after 14000 images, loss=406.1183374278052 and correct%=0.816\n",
      "after 14500 images, loss=415.61199517349644 and correct%=0.812\n",
      "after 15000 images, loss=444.4869591430352 and correct%=0.776\n",
      "after 15500 images, loss=357.8682728224866 and correct%=0.874\n",
      "after 16000 images, loss=396.5626053014333 and correct%=0.82\n",
      "after 16500 images, loss=348.59138612484696 and correct%=0.85\n",
      "after 17000 images, loss=385.5916001054487 and correct%=0.828\n",
      "after 17500 images, loss=360.86473046275285 and correct%=0.84\n",
      "after 18000 images, loss=408.14457780371083 and correct%=0.828\n",
      "after 18500 images, loss=324.37173585975046 and correct%=0.882\n",
      "after 19000 images, loss=309.291143172446 and correct%=0.896\n",
      "after 19500 images, loss=335.05934723306063 and correct%=0.874\n",
      "after 20000 images, loss=291.19520443557013 and correct%=0.884\n",
      "after 20500 images, loss=332.3997024829603 and correct%=0.872\n",
      "after 21000 images, loss=342.98558782683534 and correct%=0.822\n",
      "after 21500 images, loss=301.17253192193056 and correct%=0.874\n",
      "after 22000 images, loss=275.4860406488648 and correct%=0.892\n",
      "after 22500 images, loss=331.7537938023969 and correct%=0.872\n",
      "after 23000 images, loss=319.5232566243781 and correct%=0.842\n",
      "after 23500 images, loss=289.7693644371029 and correct%=0.866\n",
      "after 24000 images, loss=326.40452769141046 and correct%=0.868\n",
      "after 24500 images, loss=301.2645917573907 and correct%=0.864\n",
      "after 25000 images, loss=342.50164479717444 and correct%=0.826\n",
      "after 25500 images, loss=262.669820286226 and correct%=0.884\n",
      "after 26000 images, loss=279.1727480193581 and correct%=0.884\n",
      "after 26500 images, loss=298.0919253460875 and correct%=0.882\n",
      "after 27000 images, loss=327.4165357821589 and correct%=0.862\n",
      "after 27500 images, loss=292.3386392084625 and correct%=0.878\n",
      "after 28000 images, loss=275.03235962263443 and correct%=0.89\n",
      "after 28500 images, loss=252.7800803051205 and correct%=0.898\n",
      "after 29000 images, loss=278.86518699593734 and correct%=0.88\n",
      "after 29500 images, loss=297.9392235589604 and correct%=0.854\n",
      "after 30000 images, loss=339.5664472667703 and correct%=0.832\n",
      "after 30500 images, loss=313.70476732946696 and correct%=0.85\n",
      "after 31000 images, loss=328.9495218784595 and correct%=0.844\n",
      "after 31500 images, loss=331.5410100158356 and correct%=0.85\n",
      "after 32000 images, loss=313.23147412632795 and correct%=0.856\n",
      "after 32500 images, loss=336.60164944850266 and correct%=0.806\n",
      "after 33000 images, loss=273.7681078626779 and correct%=0.884\n",
      "after 33500 images, loss=294.74366732354605 and correct%=0.856\n",
      "after 34000 images, loss=219.40714344251793 and correct%=0.922\n",
      "after 34500 images, loss=252.05080925596985 and correct%=0.9\n",
      "after 35000 images, loss=299.84813689122643 and correct%=0.854\n",
      "after 35500 images, loss=262.9800306494825 and correct%=0.872\n",
      "after 36000 images, loss=261.8549253921761 and correct%=0.874\n",
      "after 36500 images, loss=271.7407710338495 and correct%=0.87\n",
      "after 37000 images, loss=216.02750964158597 and correct%=0.9\n",
      "after 37500 images, loss=309.8681179083485 and correct%=0.838\n",
      "after 38000 images, loss=274.20004324119327 and correct%=0.868\n",
      "after 38500 images, loss=251.13977068661444 and correct%=0.888\n",
      "after 39000 images, loss=249.5192001024967 and correct%=0.896\n",
      "after 39500 images, loss=264.4189459923346 and correct%=0.872\n",
      "after 40000 images, loss=297.78460388214415 and correct%=0.86\n",
      "after 40500 images, loss=242.48888049664922 and correct%=0.888\n",
      "after 41000 images, loss=230.98848209059597 and correct%=0.9\n",
      "after 41500 images, loss=303.44831976177426 and correct%=0.85\n",
      "after 42000 images, loss=271.2779602077612 and correct%=0.868\n",
      "after 42500 images, loss=299.80294611275 and correct%=0.83\n",
      "after 43000 images, loss=264.8602345722367 and correct%=0.86\n",
      "after 43500 images, loss=228.02585907506617 and correct%=0.884\n",
      "after 44000 images, loss=242.9289262111077 and correct%=0.878\n",
      "after 44500 images, loss=287.1841049328622 and correct%=0.85\n",
      "after 45000 images, loss=227.74249938297655 and correct%=0.904\n",
      "after 45500 images, loss=265.7517713860781 and correct%=0.858\n",
      "after 46000 images, loss=270.47311651043844 and correct%=0.86\n",
      "after 46500 images, loss=286.8629827360709 and correct%=0.85\n",
      "after 47000 images, loss=210.6349938841717 and correct%=0.896\n",
      "after 47500 images, loss=258.42425933008377 and correct%=0.86\n",
      "after 48000 images, loss=249.41185366113214 and correct%=0.878\n",
      "after 48500 images, loss=195.84372931331086 and correct%=0.904\n",
      "after 49000 images, loss=229.3381032496961 and correct%=0.876\n",
      "after 49500 images, loss=269.52121743873784 and correct%=0.836\n"
     ]
    }
   ],
   "source": [
    "# Now let's train\n",
    "\n",
    "model_test.train(train_set, train_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "% correct:  0.890989098909891\n"
     ]
    }
   ],
   "source": [
    "correct_percent, predictions = model_test.assess_model(test_set, test_labels)\n",
    "print('% correct: ', correct_percent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 0 images, loss=0 and correct%=0.0\n",
      "after 500 images, loss=1126.3772857320073 and correct%=0.348\n",
      "after 1000 images, loss=1090.6417633076665 and correct%=0.556\n",
      "after 1500 images, loss=1050.7014769924986 and correct%=0.564\n",
      "after 2000 images, loss=994.9852188308643 and correct%=0.696\n",
      "after 2500 images, loss=953.9220341723973 and correct%=0.714\n",
      "after 3000 images, loss=916.5599752146067 and correct%=0.696\n",
      "after 3500 images, loss=903.4040956242077 and correct%=0.7\n",
      "after 4000 images, loss=854.5350152119445 and correct%=0.732\n",
      "after 4500 images, loss=833.5799448586803 and correct%=0.766\n",
      "after 5000 images, loss=783.2096248990467 and correct%=0.72\n",
      "after 5500 images, loss=768.5172467707195 and correct%=0.746\n",
      "after 6000 images, loss=760.1845637524829 and correct%=0.796\n",
      "after 6500 images, loss=706.2733494066116 and correct%=0.812\n",
      "after 7000 images, loss=702.3951003333918 and correct%=0.784\n",
      "after 7500 images, loss=737.81765571823 and correct%=0.762\n",
      "after 8000 images, loss=685.9998499985371 and correct%=0.768\n",
      "after 8500 images, loss=672.780447236734 and correct%=0.788\n",
      "after 9000 images, loss=648.3742976218683 and correct%=0.756\n",
      "after 9500 images, loss=596.8682652207847 and correct%=0.822\n",
      "after 10000 images, loss=573.3314321619341 and correct%=0.824\n",
      "after 10500 images, loss=576.6705291511796 and correct%=0.812\n",
      "after 11000 images, loss=562.2924419323771 and correct%=0.84\n",
      "after 11500 images, loss=537.9612655280325 and correct%=0.828\n",
      "after 12000 images, loss=577.4636798814153 and correct%=0.79\n",
      "after 12500 images, loss=552.2212493756103 and correct%=0.814\n",
      "after 13000 images, loss=559.5715012992366 and correct%=0.792\n",
      "after 13500 images, loss=553.0854986299299 and correct%=0.804\n",
      "after 14000 images, loss=536.7699776893086 and correct%=0.772\n",
      "after 14500 images, loss=540.2834085624402 and correct%=0.772\n",
      "after 15000 images, loss=546.1198486993637 and correct%=0.75\n",
      "after 15500 images, loss=481.6155588473197 and correct%=0.838\n",
      "after 16000 images, loss=516.0352269468823 and correct%=0.764\n",
      "after 16500 images, loss=473.12280029603124 and correct%=0.818\n",
      "after 17000 images, loss=485.583854430535 and correct%=0.816\n",
      "after 17500 images, loss=479.3662075409866 and correct%=0.818\n",
      "after 18000 images, loss=502.42698526690435 and correct%=0.784\n",
      "after 18500 images, loss=436.24586562750335 and correct%=0.826\n",
      "after 19000 images, loss=403.0977292462261 and correct%=0.89\n",
      "after 19500 images, loss=437.69259987193044 and correct%=0.828\n",
      "after 20000 images, loss=390.2610921548629 and correct%=0.848\n",
      "after 20500 images, loss=429.36662639673716 and correct%=0.83\n",
      "after 21000 images, loss=434.9040567449726 and correct%=0.802\n",
      "after 21500 images, loss=378.11020683642056 and correct%=0.856\n",
      "after 22000 images, loss=361.4701371402183 and correct%=0.88\n",
      "after 22500 images, loss=416.0771097367273 and correct%=0.84\n",
      "after 23000 images, loss=412.2705728416043 and correct%=0.83\n",
      "after 23500 images, loss=360.82748172375966 and correct%=0.854\n",
      "after 24000 images, loss=399.20461824260354 and correct%=0.838\n",
      "after 24500 images, loss=377.8717590078121 and correct%=0.834\n",
      "after 25000 images, loss=413.1420195899828 and correct%=0.798\n",
      "after 25500 images, loss=331.4642883743305 and correct%=0.864\n",
      "after 26000 images, loss=328.4151792951414 and correct%=0.868\n",
      "after 26500 images, loss=345.39529590337577 and correct%=0.864\n",
      "after 27000 images, loss=373.3715995739954 and correct%=0.838\n",
      "after 27500 images, loss=343.2584904962746 and correct%=0.854\n",
      "after 28000 images, loss=326.31396547217196 and correct%=0.87\n",
      "after 28500 images, loss=298.8095275366371 and correct%=0.898\n",
      "after 29000 images, loss=338.3012393631452 and correct%=0.84\n",
      "after 29500 images, loss=340.26922691181795 and correct%=0.848\n",
      "after 30000 images, loss=388.88553361589277 and correct%=0.82\n",
      "after 30500 images, loss=372.38576148432963 and correct%=0.816\n",
      "after 31000 images, loss=365.98452870158775 and correct%=0.83\n",
      "after 31500 images, loss=373.6312768738598 and correct%=0.83\n",
      "after 32000 images, loss=345.3309935234725 and correct%=0.83\n",
      "after 32500 images, loss=394.87095110122743 and correct%=0.794\n",
      "after 33000 images, loss=300.00723628250296 and correct%=0.894\n",
      "after 33500 images, loss=337.6454416036721 and correct%=0.85\n",
      "after 34000 images, loss=255.47365050725497 and correct%=0.89\n",
      "after 34500 images, loss=285.94509824895897 and correct%=0.894\n",
      "after 35000 images, loss=335.00672707849117 and correct%=0.84\n",
      "after 35500 images, loss=299.0753432302001 and correct%=0.89\n",
      "after 36000 images, loss=298.19500463422753 and correct%=0.868\n",
      "after 36500 images, loss=286.49972120243814 and correct%=0.866\n",
      "after 37000 images, loss=261.4294466297689 and correct%=0.896\n",
      "after 37500 images, loss=350.3014467828507 and correct%=0.814\n",
      "after 38000 images, loss=292.2968011736415 and correct%=0.874\n",
      "after 38500 images, loss=273.30008980958036 and correct%=0.882\n",
      "after 39000 images, loss=275.7438734539154 and correct%=0.868\n",
      "after 39500 images, loss=280.018820149718 and correct%=0.876\n",
      "after 40000 images, loss=317.8816829912322 and correct%=0.844\n",
      "after 40500 images, loss=268.03744934492414 and correct%=0.89\n",
      "after 41000 images, loss=252.0569285376433 and correct%=0.892\n",
      "after 41500 images, loss=332.3742717145127 and correct%=0.836\n",
      "after 42000 images, loss=289.56710529424436 and correct%=0.876\n",
      "after 42500 images, loss=330.3575146376315 and correct%=0.824\n",
      "after 43000 images, loss=292.1250258649089 and correct%=0.852\n",
      "after 43500 images, loss=254.91482208786107 and correct%=0.88\n",
      "after 44000 images, loss=263.3481055194556 and correct%=0.884\n",
      "after 44500 images, loss=314.5871214875993 and correct%=0.838\n",
      "after 45000 images, loss=240.62275096881817 and correct%=0.884\n",
      "after 45500 images, loss=273.9691090178922 and correct%=0.858\n",
      "after 46000 images, loss=279.3038086834651 and correct%=0.876\n",
      "after 46500 images, loss=298.31403674209145 and correct%=0.838\n",
      "after 47000 images, loss=214.2885373422505 and correct%=0.904\n",
      "after 47500 images, loss=264.8797921984418 and correct%=0.862\n",
      "after 48000 images, loss=255.52513151049078 and correct%=0.872\n",
      "after 48500 images, loss=207.27072624455437 and correct%=0.904\n",
      "after 49000 images, loss=238.35721714313985 and correct%=0.9\n",
      "after 49500 images, loss=272.79475483730084 and correct%=0.85\n"
     ]
    }
   ],
   "source": [
    "model_1 = CNNModel(\n",
    "    layers=[\n",
    "        ConvLayer(num_filters=6, filter_size=5),\n",
    "        MaxPoolLayer(3),\n",
    "        SoftmaxLayer(num_inputs=8*8*6, num_classes=10)\n",
    "    ],\n",
    "    num_classes=10,\n",
    "    learning_rate=0.005\n",
    ")\n",
    "\n",
    "model_1.train(train_set, train_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "% correct:  89.39893989398941 %\n"
     ]
    }
   ],
   "source": [
    "correct_percent_model1, predictions_model1 = model_1.assess_model(test_set, test_labels)\n",
    "print('% correct: ', correct_percent_model1*100, '%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 0 images, loss=0 and correct%=0.0\n",
      "after 500 images, loss=1125.6132955178307 and correct%=0.45\n",
      "after 1000 images, loss=1087.9466433740704 and correct%=0.566\n",
      "after 1500 images, loss=1047.214278410815 and correct%=0.578\n",
      "after 2000 images, loss=997.327832203414 and correct%=0.698\n",
      "after 2500 images, loss=952.6302266231811 and correct%=0.71\n",
      "after 3000 images, loss=914.2831812839671 and correct%=0.692\n",
      "after 3500 images, loss=893.7848598151875 and correct%=0.706\n",
      "after 4000 images, loss=850.2945348057062 and correct%=0.73\n",
      "after 4500 images, loss=814.7053563822491 and correct%=0.764\n",
      "after 5000 images, loss=760.9140411064002 and correct%=0.756\n",
      "after 5500 images, loss=713.3887974889847 and correct%=0.758\n",
      "after 6000 images, loss=694.8483517373992 and correct%=0.788\n",
      "after 6500 images, loss=618.4100718436928 and correct%=0.814\n",
      "after 7000 images, loss=610.609566103251 and correct%=0.812\n",
      "after 7500 images, loss=643.3862820827186 and correct%=0.794\n",
      "after 8000 images, loss=568.6245501527243 and correct%=0.798\n",
      "after 8500 images, loss=537.151270619279 and correct%=0.81\n",
      "after 9000 images, loss=508.5215763813897 and correct%=0.78\n",
      "after 9500 images, loss=440.0816388824546 and correct%=0.852\n",
      "after 10000 images, loss=391.65373812794536 and correct%=0.866\n",
      "after 10500 images, loss=371.8516406068772 and correct%=0.846\n",
      "after 11000 images, loss=345.9067560299951 and correct%=0.86\n",
      "after 11500 images, loss=321.928665108917 and correct%=0.87\n",
      "after 12000 images, loss=379.3841952246493 and correct%=0.826\n",
      "after 12500 images, loss=344.3047532500081 and correct%=0.842\n",
      "after 13000 images, loss=353.31824428133086 and correct%=0.834\n",
      "after 13500 images, loss=325.27288158549356 and correct%=0.832\n",
      "after 14000 images, loss=311.41086900609196 and correct%=0.832\n",
      "after 14500 images, loss=333.7216278590752 and correct%=0.816\n",
      "after 15000 images, loss=344.63022134138885 and correct%=0.8\n",
      "after 15500 images, loss=247.04267033272234 and correct%=0.886\n",
      "after 16000 images, loss=300.17986904769344 and correct%=0.836\n",
      "after 16500 images, loss=247.83754385829278 and correct%=0.882\n",
      "after 17000 images, loss=272.3736012086853 and correct%=0.838\n",
      "after 17500 images, loss=258.28206990940146 and correct%=0.848\n",
      "after 18000 images, loss=279.24124124252785 and correct%=0.852\n",
      "after 18500 images, loss=204.09113034292778 and correct%=0.898\n",
      "after 19000 images, loss=190.82127521978506 and correct%=0.916\n",
      "after 19500 images, loss=230.25927054351652 and correct%=0.858\n",
      "after 20000 images, loss=184.42835066784102 and correct%=0.904\n",
      "after 20500 images, loss=213.13320108918342 and correct%=0.884\n",
      "after 21000 images, loss=255.1867542205449 and correct%=0.854\n",
      "after 21500 images, loss=181.76923190507713 and correct%=0.908\n",
      "after 22000 images, loss=171.1474188144987 and correct%=0.908\n",
      "after 22500 images, loss=204.31130803125194 and correct%=0.882\n",
      "after 23000 images, loss=216.85408362703149 and correct%=0.858\n",
      "after 23500 images, loss=187.4844621691933 and correct%=0.878\n",
      "after 24000 images, loss=218.3112359628443 and correct%=0.87\n",
      "after 24500 images, loss=203.95275458911507 and correct%=0.868\n",
      "after 25000 images, loss=239.34272918334813 and correct%=0.848\n",
      "after 25500 images, loss=173.88275509720128 and correct%=0.876\n",
      "after 26000 images, loss=163.96691939423332 and correct%=0.9\n",
      "after 26500 images, loss=191.9208118392735 and correct%=0.9\n",
      "after 27000 images, loss=242.8411324433659 and correct%=0.874\n",
      "after 27500 images, loss=183.32013532845733 and correct%=0.884\n",
      "after 28000 images, loss=170.48443875052538 and correct%=0.89\n",
      "after 28500 images, loss=164.99163695386503 and correct%=0.91\n",
      "after 29000 images, loss=202.20574099522563 and correct%=0.882\n",
      "after 29500 images, loss=214.92798010054813 and correct%=0.864\n",
      "after 30000 images, loss=222.58206756451858 and correct%=0.864\n",
      "after 30500 images, loss=207.4829059985193 and correct%=0.872\n",
      "after 31000 images, loss=224.02543432426305 and correct%=0.876\n",
      "after 31500 images, loss=255.52985436018344 and correct%=0.868\n",
      "after 32000 images, loss=213.99396448714668 and correct%=0.874\n",
      "after 32500 images, loss=254.6084742372603 and correct%=0.832\n",
      "after 33000 images, loss=181.36940839820565 and correct%=0.912\n",
      "after 33500 images, loss=188.48042875177396 and correct%=0.896\n",
      "after 34000 images, loss=126.54180892444464 and correct%=0.93\n",
      "after 34500 images, loss=157.96809821373918 and correct%=0.92\n",
      "after 35000 images, loss=216.04714341476316 and correct%=0.854\n",
      "after 35500 images, loss=161.60810118985046 and correct%=0.896\n",
      "after 36000 images, loss=151.19521252081907 and correct%=0.906\n",
      "after 36500 images, loss=161.4984327063098 and correct%=0.908\n",
      "after 37000 images, loss=130.9246835930743 and correct%=0.926\n",
      "after 37500 images, loss=270.6916499087967 and correct%=0.844\n",
      "after 38000 images, loss=185.34904291850725 and correct%=0.886\n",
      "after 38500 images, loss=154.58016178797004 and correct%=0.908\n",
      "after 39000 images, loss=181.2252473196177 and correct%=0.89\n",
      "after 39500 images, loss=196.64949798085897 and correct%=0.896\n",
      "after 40000 images, loss=187.58678225326784 and correct%=0.894\n",
      "after 40500 images, loss=160.5412520790167 and correct%=0.9\n",
      "after 41000 images, loss=151.36261496279644 and correct%=0.914\n",
      "after 41500 images, loss=214.80394733092115 and correct%=0.882\n",
      "after 42000 images, loss=180.5265992544841 and correct%=0.894\n",
      "after 42500 images, loss=214.80833203394698 and correct%=0.87\n",
      "after 43000 images, loss=217.8534966743039 and correct%=0.876\n",
      "after 43500 images, loss=137.02813505932554 and correct%=0.906\n",
      "after 44000 images, loss=170.31738580474232 and correct%=0.892\n",
      "after 44500 images, loss=192.27381328919202 and correct%=0.882\n",
      "after 45000 images, loss=163.22235028728682 and correct%=0.9\n",
      "after 45500 images, loss=193.983917156508 and correct%=0.874\n",
      "after 46000 images, loss=206.85740703328477 and correct%=0.872\n",
      "after 46500 images, loss=221.98113631883342 and correct%=0.868\n",
      "after 47000 images, loss=126.87193846778801 and correct%=0.926\n",
      "after 47500 images, loss=181.4634487620815 and correct%=0.876\n",
      "after 48000 images, loss=176.78316615109173 and correct%=0.902\n",
      "after 48500 images, loss=144.47209520428714 and correct%=0.924\n",
      "after 49000 images, loss=153.44863044848654 and correct%=0.898\n",
      "after 49500 images, loss=205.89656321521574 and correct%=0.878\n"
     ]
    }
   ],
   "source": [
    "# A slightly larger model (more filters, larger filters)\n",
    "\n",
    "model_2 = CNNModel(\n",
    "    layers=[\n",
    "        ConvLayer(num_filters=10, filter_size=8),\n",
    "        MaxPoolLayer(2),\n",
    "        SoftmaxLayer(num_inputs=10*10*10, num_classes=10)\n",
    "    ],\n",
    "    num_classes=10,\n",
    "    learning_rate=0.005\n",
    ")\n",
    "\n",
    "model_2.train(train_set, train_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "% correct:  90.25902590259027 %\n"
     ]
    }
   ],
   "source": [
    "correct_percent_model2, predictions_model2 = model_2.assess_model(test_set, test_labels)\n",
    "print('% correct: ', correct_percent_model2*100, '%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "after 0 images, loss=0 and correct%=0.0\n",
      "after 500 images, loss=1091.7414963075903 and correct%=0.476\n",
      "after 1000 images, loss=1007.7420996646855 and correct%=0.61\n",
      "after 1500 images, loss=929.2858608007286 and correct%=0.658\n",
      "after 2000 images, loss=836.6445782527004 and correct%=0.744\n",
      "after 2500 images, loss=766.8580949660654 and correct%=0.762\n",
      "after 3000 images, loss=714.4706785474345 and correct%=0.762\n",
      "after 3500 images, loss=701.0667892851263 and correct%=0.764\n",
      "after 4000 images, loss=644.6854321386253 and correct%=0.782\n",
      "after 4500 images, loss=614.0392013573736 and correct%=0.802\n",
      "after 5000 images, loss=571.6754619495205 and correct%=0.79\n",
      "after 5500 images, loss=537.2907482616208 and correct%=0.79\n",
      "after 6000 images, loss=534.7686976558207 and correct%=0.838\n",
      "after 6500 images, loss=470.89812950262086 and correct%=0.85\n",
      "after 7000 images, loss=487.6380413214927 and correct%=0.84\n",
      "after 7500 images, loss=544.7149994688632 and correct%=0.808\n",
      "after 8000 images, loss=489.18638591483653 and correct%=0.798\n",
      "after 8500 images, loss=473.04658589711556 and correct%=0.834\n",
      "after 9000 images, loss=468.97155942078047 and correct%=0.79\n",
      "after 9500 images, loss=411.93071581634354 and correct%=0.86\n",
      "after 10000 images, loss=389.69559350874215 and correct%=0.858\n",
      "after 10500 images, loss=379.536953570474 and correct%=0.846\n",
      "after 11000 images, loss=371.56455224662875 and correct%=0.864\n",
      "after 11500 images, loss=356.7529551998838 and correct%=0.858\n",
      "after 12000 images, loss=417.43462555945945 and correct%=0.824\n",
      "after 12500 images, loss=385.7476308115478 and correct%=0.842\n",
      "after 13000 images, loss=405.76355277495736 and correct%=0.824\n",
      "after 13500 images, loss=385.38956069849195 and correct%=0.83\n",
      "after 14000 images, loss=383.9552934419628 and correct%=0.822\n",
      "after 14500 images, loss=405.785529866262 and correct%=0.804\n",
      "after 15000 images, loss=432.3393227212483 and correct%=0.774\n",
      "after 15500 images, loss=333.97027430755145 and correct%=0.882\n",
      "after 16000 images, loss=379.3188380779191 and correct%=0.824\n",
      "after 16500 images, loss=334.38262014704645 and correct%=0.864\n",
      "after 17000 images, loss=360.1383472278211 and correct%=0.832\n",
      "after 17500 images, loss=344.53113258401305 and correct%=0.84\n",
      "after 18000 images, loss=382.05319809052503 and correct%=0.824\n",
      "after 18500 images, loss=303.37005550689474 and correct%=0.876\n",
      "after 19000 images, loss=282.942749493916 and correct%=0.914\n",
      "after 19500 images, loss=314.0727736301797 and correct%=0.854\n",
      "after 20000 images, loss=274.9103562028823 and correct%=0.888\n",
      "after 20500 images, loss=312.2939742841422 and correct%=0.886\n",
      "after 21000 images, loss=323.42228902513745 and correct%=0.818\n",
      "after 21500 images, loss=281.11034471525664 and correct%=0.89\n",
      "after 22000 images, loss=254.03304286052847 and correct%=0.902\n",
      "after 22500 images, loss=309.7894361366085 and correct%=0.874\n",
      "after 23000 images, loss=307.7085071815109 and correct%=0.832\n",
      "after 23500 images, loss=273.472032371049 and correct%=0.876\n",
      "after 24000 images, loss=301.95177773424774 and correct%=0.868\n",
      "after 24500 images, loss=283.0270775941224 and correct%=0.876\n",
      "after 25000 images, loss=319.2989033274736 and correct%=0.84\n",
      "after 25500 images, loss=248.00602531316207 and correct%=0.878\n",
      "after 26000 images, loss=252.39967470232213 and correct%=0.9\n",
      "after 26500 images, loss=270.7194658358633 and correct%=0.894\n",
      "after 27000 images, loss=304.3608891707603 and correct%=0.86\n",
      "after 27500 images, loss=270.4369213144348 and correct%=0.876\n",
      "after 28000 images, loss=247.3857725536386 and correct%=0.888\n",
      "after 28500 images, loss=234.79671568843543 and correct%=0.904\n",
      "after 29000 images, loss=260.34150128984004 and correct%=0.87\n",
      "after 29500 images, loss=274.26482861536726 and correct%=0.864\n",
      "after 30000 images, loss=313.302383918547 and correct%=0.832\n",
      "after 30500 images, loss=292.2846042860894 and correct%=0.846\n",
      "after 31000 images, loss=299.90540663005305 and correct%=0.852\n",
      "after 31500 images, loss=308.60736099753615 and correct%=0.854\n",
      "after 32000 images, loss=288.5447316404726 and correct%=0.854\n",
      "after 32500 images, loss=321.40528420923437 and correct%=0.816\n",
      "after 33000 images, loss=251.15685861277152 and correct%=0.888\n",
      "after 33500 images, loss=266.5540340838622 and correct%=0.874\n",
      "after 34000 images, loss=196.97232082114206 and correct%=0.928\n",
      "after 34500 images, loss=227.98795720213354 and correct%=0.906\n",
      "after 35000 images, loss=277.76672382673354 and correct%=0.854\n",
      "after 35500 images, loss=234.20443420132833 and correct%=0.89\n",
      "after 36000 images, loss=234.06840288445775 and correct%=0.884\n",
      "after 36500 images, loss=239.64985102516587 and correct%=0.88\n",
      "after 37000 images, loss=197.88721802272352 and correct%=0.904\n",
      "after 37500 images, loss=291.6459647931954 and correct%=0.85\n",
      "after 38000 images, loss=248.66991668604578 and correct%=0.878\n",
      "after 38500 images, loss=224.37191869146446 and correct%=0.894\n",
      "after 39000 images, loss=224.23844752696553 and correct%=0.89\n",
      "after 39500 images, loss=242.2479946537236 and correct%=0.882\n",
      "after 40000 images, loss=269.97411351617376 and correct%=0.868\n",
      "after 40500 images, loss=218.59933049854908 and correct%=0.892\n",
      "after 41000 images, loss=204.7250275030151 and correct%=0.916\n",
      "after 41500 images, loss=276.38244794130236 and correct%=0.858\n",
      "after 42000 images, loss=242.50610498165622 and correct%=0.868\n",
      "after 42500 images, loss=275.4461631763067 and correct%=0.84\n",
      "after 43000 images, loss=248.33505709540734 and correct%=0.872\n",
      "after 43500 images, loss=203.041491217215 and correct%=0.902\n",
      "after 44000 images, loss=218.45536807425154 and correct%=0.886\n",
      "after 44500 images, loss=263.41201617379994 and correct%=0.864\n",
      "after 45000 images, loss=204.0695268999046 and correct%=0.904\n",
      "after 45500 images, loss=236.72378187716754 and correct%=0.872\n",
      "after 46000 images, loss=245.432081251318 and correct%=0.874\n",
      "after 46500 images, loss=263.00989615612605 and correct%=0.86\n",
      "after 47000 images, loss=186.80622689661143 and correct%=0.912\n",
      "after 47500 images, loss=232.6185093910975 and correct%=0.88\n",
      "after 48000 images, loss=224.2445398534598 and correct%=0.878\n",
      "after 48500 images, loss=175.7036582845933 and correct%=0.922\n",
      "after 49000 images, loss=203.8857972583578 and correct%=0.89\n",
      "after 49500 images, loss=242.64113062333564 and correct%=0.85\n"
     ]
    }
   ],
   "source": [
    "# Let's go even larger, \n",
    "\n",
    "model_3 = CNNModel(\n",
    "    layers=[\n",
    "        ConvLayer(num_filters=15, filter_size=6),\n",
    "        MaxPoolLayer(2),\n",
    "        SoftmaxLayer(num_inputs=11*11*15, num_classes=10)\n",
    "    ],\n",
    "    num_classes=10,\n",
    "    learning_rate=0.005\n",
    ")\n",
    "\n",
    "model_3.train(train_set, train_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "% correct:  89.96899689968997 %\n"
     ]
    }
   ],
   "source": [
    "correct_percent_model3, predictions_model3 = model_3.assess_model(test_set, test_labels)\n",
    "print('% correct: ', correct_percent_model3*100, '%')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Okay, it's looking like I can't get much better than 90%, at least by increasing the size of the models. "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "env",
   "language": "python",
   "name": "env"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}