benchmark_tflite_model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Quantized model benchmarking\n",
    "Computation of the accuracies of the quantized models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys \n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow.keras.models import load_model\n",
    "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n",
    "from tensorflow.keras.applications.mobilenet import preprocess_input\n",
    "from sklearn.metrics import classification_report, confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# configuration parameters \n",
    "TEST_DATA_DIR = '/Users/sanchit/Documents/Projects/Datasets/animals/test/'\n",
    "MODEL_PATH = \"./models/mobilenet.h5\"\n",
    "TFLITE_MODEL_DIR = \"./models/tflite/\"\n",
    "# choose the model type here\n",
    "QUANT_TYPE = \"both_int8\" # no_quant, w_int8, w_fp16, both_int8, both_fp16\n",
    "TEST_SAMPLES = 450\n",
    "NUM_CLASSES = 3\n",
    "IMG_WIDTH, IMG_HEIGHT = 224, 224\n",
    "BATCH_SIZE = 64\n",
    "LABELS = [\"cats\", \"dogs\", \"panda\"]\n",
    "QUANT_NAME_MAP = {\"no_quant\": \"no quantization\", \"w_int8\": \"weights 8-bit INT quantized\", \n",
    "                  \"w_fp16\": \"weights 16-bit FP quantized\", \"both_int8\": \"both weights and activations INT8 quantized\", \n",
    "                 \"both_fp16\": \"both weights and activations FP-16 quantized\"}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load quantized model according to the type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "interpreter for both weights and activations INT8 quantized loading ...\n"
     ]
    }
   ],
   "source": [
    "if QUANT_TYPE == \"no_quant\":\n",
    "    # model without any quantization\n",
    "    print(f\"interpreter for {QUANT_NAME_MAP[QUANT_TYPE]} loading ...\")\n",
    "    interpret = tf.lite.Interpreter(model_path = TFLITE_MODEL_DIR + \"mobilenet_no_quant.tflite\")\n",
    "    interpret.allocate_tensors() # allocate memory to the model\n",
    "    \n",
    "elif QUANT_TYPE == \"w_int8\":\n",
    "    # model with weights INT8 quantization\n",
    "    print(f\"interpreter for {QUANT_NAME_MAP[QUANT_TYPE]} loading ...\")\n",
    "    interpret = tf.lite.Interpreter(model_path = TFLITE_MODEL_DIR + \"mobilenet_weights_int8_quant.tflite\")\n",
    "    interpret.allocate_tensors() # allocate memory to the model\n",
    "    \n",
    "elif QUANT_TYPE == \"w_fp16\":\n",
    "    # model with weights FP16 quantization \n",
    "    print(f\"interpreter for {QUANT_NAME_MAP[QUANT_TYPE]} loading ...\")\n",
    "    interpret = tf.lite.Interpreter(model_path = TFLITE_MODEL_DIR + \"mobilenet_weights_float16_quant.tflite\")\n",
    "    interpret.allocate_tensors() # allocate memory to the model\n",
    "    \n",
    "elif QUANT_TYPE == \"both_int8\":\n",
    "    # model with both weights and activations INT8 quantization \n",
    "    print(f\"interpreter for {QUANT_NAME_MAP[QUANT_TYPE]} loading ...\")\n",
    "    interpret = tf.lite.Interpreter(model_path = TFLITE_MODEL_DIR + \"mobilenet_both_int8_quant.tflite\")\n",
    "    interpret.allocate_tensors() # allocate memory to the model\n",
    "    \n",
    "elif QUANT_TYPE == \"both_fp16\":\n",
    "    # model with both weights and activations INT8 quantization \n",
    "    print(f\"interpreter for {QUANT_NAME_MAP[QUANT_TYPE]} loading ...\")\n",
    "    interpret = tf.lite.Interpreter(model_path = TFLITE_MODEL_DIR + \"mobilenet_both_fp16_quant.tflite\")\n",
    "    interpret.allocate_tensors() # allocate memory to the model\n",
    "\n",
    "else:\n",
    "    print(f\"Wrong quantization type has been chosen for {QUANT_NAME_MAP[QUANT_TYPE]}\")\n",
    "    sys.exit(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get indices of input and output tensors for each model \n",
    "input_ind = interpret.get_input_details()[0][\"index\"]\n",
    "out_ind   = interpret.get_output_details()[0][\"index\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create test generator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 450 images belonging to 3 classes.\n"
     ]
    }
   ],
   "source": [
    "test_datagen = ImageDataGenerator(preprocessing_function=preprocess_input)\n",
    "test_generator = test_datagen.flow_from_directory(\n",
    "    TEST_DATA_DIR,\n",
    "    target_size=(IMG_WIDTH, IMG_HEIGHT),\n",
    "    batch_size=1,\n",
    "    shuffle=False,\n",
    "    class_mode='categorical')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "References for using interpreter: \n",
    "- (check evaluate the models): https://www.tensorflow.org/lite/performance/post_training_float16_quant \n",
    "- it uses batch generator and batch predictions (TODO later): https://thinkmobile.dev/testing-tensorflow-lite-image-classification-model/ \n",
    "- TinyML book chapter on Interpreter"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Note: TFLite inference on a desktop is slower simply due to it is optimized for mobile devices (ARM processors) and not for Intel processors. For more details, see: \n",
    "- https://stackoverflow.com/questions/54093424/why-is-tensorflow-lite-slower-than-tensorflow-on-desktop \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "computing results for both weights and activations INT8 quantized ... \n",
      "\n",
      "computing results for 0th image ...\n",
      "computing results for 1th image ...\n",
      "computing results for 2th image ...\n",
      "computing results for 3th image ...\n",
      "computing results for 4th image ...\n",
      "computing results for 5th image ...\n",
      "computing results for 6th image ...\n",
      "computing results for 7th image ...\n",
      "computing results for 8th image ...\n",
      "computing results for 9th image ...\n",
      "computing results for 10th image ...\n",
      "computing results for 11th image ...\n",
      "computing results for 12th image ...\n",
      "computing results for 13th image ...\n",
      "computing results for 14th image ...\n",
      "computing results for 15th image ...\n",
      "computing results for 16th image ...\n",
      "computing results for 17th image ...\n",
      "computing results for 18th image ...\n",
      "computing results for 19th image ...\n",
      "computing results for 20th image ...\n",
      "computing results for 21th image ...\n",
      "computing results for 22th image ...\n",
      "computing results for 23th image ...\n",
      "computing results for 24th image ...\n",
      "computing results for 25th image ...\n",
      "computing results for 26th image ...\n",
      "computing results for 27th image ...\n",
      "computing results for 28th image ...\n",
      "computing results for 29th image ...\n",
      "computing results for 30th image ...\n",
      "computing results for 31th image ...\n",
      "computing results for 32th image ...\n",
      "computing results for 33th image ...\n",
      "computing results for 34th image ...\n",
      "computing results for 35th image ...\n",
      "computing results for 36th image ...\n",
      "computing results for 37th image ...\n",
      "computing results for 38th image ...\n",
      "computing results for 39th image ...\n",
      "computing results for 40th image ...\n",
      "computing results for 41th image ...\n",
      "computing results for 42th image ...\n",
      "computing results for 43th image ...\n",
      "computing results for 44th image ...\n",
      "computing results for 45th image ...\n",
      "computing results for 46th image ...\n",
      "computing results for 47th image ...\n",
      "computing results for 48th image ...\n",
      "computing results for 49th image ...\n",
      "computing results for 50th image ...\n",
      "computing results for 51th image ...\n",
      "computing results for 52th image ...\n",
      "computing results for 53th image ...\n",
      "computing results for 54th image ...\n",
      "computing results for 55th image ...\n",
      "computing results for 56th image ...\n",
      "computing results for 57th image ...\n",
      "computing results for 58th image ...\n",
      "computing results for 59th image ...\n",
      "computing results for 60th image ...\n",
      "computing results for 61th image ...\n",
      "computing results for 62th image ...\n",
      "computing results for 63th image ...\n",
      "computing results for 64th image ...\n",
      "computing results for 65th image ...\n",
      "computing results for 66th image ...\n",
      "computing results for 67th image ...\n",
      "computing results for 68th image ...\n",
      "computing results for 69th image ...\n",
      "computing results for 70th image ...\n",
      "computing results for 71th image ...\n",
      "computing results for 72th image ...\n",
      "computing results for 73th image ...\n",
      "computing results for 74th image ...\n",
      "computing results for 75th image ...\n",
      "computing results for 76th image ...\n",
      "computing results for 77th image ...\n",
      "computing results for 78th image ...\n",
      "computing results for 79th image ...\n",
      "computing results for 80th image ...\n",
      "computing results for 81th image ...\n",
      "computing results for 82th image ...\n",
      "computing results for 83th image ...\n",
      "computing results for 84th image ...\n",
      "computing results for 85th image ...\n",
      "computing results for 86th image ...\n",
      "computing results for 87th image ...\n",
      "computing results for 88th image ...\n",
      "computing results for 89th image ...\n",
      "computing results for 90th image ...\n",
      "computing results for 91th image ...\n",
      "computing results for 92th image ...\n",
      "computing results for 93th image ...\n",
      "computing results for 94th image ...\n",
      "computing results for 95th image ...\n",
      "computing results for 96th image ...\n",
      "computing results for 97th image ...\n",
      "computing results for 98th image ...\n",
      "computing results for 99th image ...\n",
      "computing results for 100th image ...\n",
      "computing results for 101th image ...\n",
      "computing results for 102th image ...\n",
      "computing results for 103th image ...\n",
      "computing results for 104th image ...\n",
      "computing results for 105th image ...\n",
      "computing results for 106th image ...\n",
      "computing results for 107th image ...\n",
      "computing results for 108th image ...\n",
      "computing results for 109th image ...\n",
      "computing results for 110th image ...\n",
      "computing results for 111th image ...\n",
      "computing results for 112th image ...\n",
      "computing results for 113th image ...\n",
      "computing results for 114th image ...\n",
      "computing results for 115th image ...\n",
      "computing results for 116th image ...\n",
      "computing results for 117th image ...\n",
      "computing results for 118th image ...\n",
      "computing results for 119th image ...\n",
      "computing results for 120th image ...\n",
      "computing results for 121th image ...\n",
      "computing results for 122th image ...\n",
      "computing results for 123th image ...\n",
      "computing results for 124th image ...\n",
      "computing results for 125th image ...\n",
      "computing results for 126th image ...\n",
      "computing results for 127th image ...\n",
      "computing results for 128th image ...\n",
      "computing results for 129th image ...\n",
      "computing results for 130th image ...\n",
      "computing results for 131th image ...\n",
      "computing results for 132th image ...\n",
      "computing results for 133th image ...\n",
      "computing results for 134th image ...\n",
      "computing results for 135th image ...\n",
      "computing results for 136th image ...\n",
      "computing results for 137th image ...\n",
      "computing results for 138th image ...\n",
      "computing results for 139th image ...\n",
      "computing results for 140th image ...\n",
      "computing results for 141th image ...\n",
      "computing results for 142th image ...\n",
      "computing results for 143th image ...\n",
      "computing results for 144th image ...\n",
      "computing results for 145th image ...\n",
      "computing results for 146th image ...\n",
      "computing results for 147th image ...\n",
      "computing results for 148th image ...\n",
      "computing results for 149th image ...\n",
      "computing results for 150th image ...\n",
      "computing results for 151th image ...\n",
      "computing results for 152th image ...\n",
      "computing results for 153th image ...\n",
      "computing results for 154th image ...\n",
      "computing results for 155th image ...\n",
      "computing results for 156th image ...\n",
      "computing results for 157th image ...\n",
      "computing results for 158th image ...\n",
      "computing results for 159th image ...\n",
      "computing results for 160th image ...\n",
      "computing results for 161th image ...\n",
      "computing results for 162th image ...\n",
      "computing results for 163th image ...\n",
      "computing results for 164th image ...\n",
      "computing results for 165th image ...\n",
      "computing results for 166th image ...\n",
      "computing results for 167th image ...\n",
      "computing results for 168th image ...\n",
      "computing results for 169th image ...\n",
      "computing results for 170th image ...\n",
      "computing results for 171th image ...\n",
      "computing results for 172th image ...\n",
      "computing results for 173th image ...\n",
      "computing results for 174th image ...\n",
      "computing results for 175th image ...\n",
      "computing results for 176th image ...\n",
      "computing results for 177th image ...\n",
      "computing results for 178th image ...\n",
      "computing results for 179th image ...\n",
      "computing results for 180th image ...\n",
      "computing results for 181th image ...\n",
      "computing results for 182th image ...\n",
      "computing results for 183th image ...\n",
      "computing results for 184th image ...\n",
      "computing results for 185th image ...\n",
      "computing results for 186th image ...\n",
      "computing results for 187th image ...\n",
      "computing results for 188th image ...\n",
      "computing results for 189th image ...\n",
      "computing results for 190th image ...\n",
      "computing results for 191th image ...\n",
      "computing results for 192th image ...\n",
      "computing results for 193th image ...\n",
      "computing results for 194th image ...\n",
      "computing results for 195th image ...\n",
      "computing results for 196th image ...\n",
      "computing results for 197th image ...\n",
      "computing results for 198th image ...\n",
      "computing results for 199th image ...\n",
      "computing results for 200th image ...\n",
      "computing results for 201th image ...\n",
      "computing results for 202th image ...\n",
      "computing results for 203th image ...\n",
      "computing results for 204th image ...\n",
      "computing results for 205th image ...\n",
      "computing results for 206th image ...\n",
      "computing results for 207th image ...\n",
      "computing results for 208th image ...\n",
      "computing results for 209th image ...\n",
      "computing results for 210th image ...\n",
      "computing results for 211th image ...\n",
      "computing results for 212th image ...\n",
      "computing results for 213th image ...\n",
      "computing results for 214th image ...\n",
      "computing results for 215th image ...\n",
      "computing results for 216th image ...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "computing results for 217th image ...\n",
      "computing results for 218th image ...\n",
      "computing results for 219th image ...\n",
      "computing results for 220th image ...\n",
      "computing results for 221th image ...\n",
      "computing results for 222th image ...\n",
      "computing results for 223th image ...\n",
      "computing results for 224th image ...\n",
      "computing results for 225th image ...\n",
      "computing results for 226th image ...\n",
      "computing results for 227th image ...\n",
      "computing results for 228th image ...\n",
      "computing results for 229th image ...\n",
      "computing results for 230th image ...\n",
      "computing results for 231th image ...\n",
      "computing results for 232th image ...\n",
      "computing results for 233th image ...\n",
      "computing results for 234th image ...\n",
      "computing results for 235th image ...\n",
      "computing results for 236th image ...\n",
      "computing results for 237th image ...\n",
      "computing results for 238th image ...\n",
      "computing results for 239th image ...\n",
      "computing results for 240th image ...\n",
      "computing results for 241th image ...\n",
      "computing results for 242th image ...\n",
      "computing results for 243th image ...\n",
      "computing results for 244th image ...\n",
      "computing results for 245th image ...\n",
      "computing results for 246th image ...\n",
      "computing results for 247th image ...\n",
      "computing results for 248th image ...\n",
      "computing results for 249th image ...\n",
      "computing results for 250th image ...\n",
      "computing results for 251th image ...\n",
      "computing results for 252th image ...\n",
      "computing results for 253th image ...\n",
      "computing results for 254th image ...\n",
      "computing results for 255th image ...\n",
      "computing results for 256th image ...\n",
      "computing results for 257th image ...\n",
      "computing results for 258th image ...\n",
      "computing results for 259th image ...\n",
      "computing results for 260th image ...\n",
      "computing results for 261th image ...\n",
      "computing results for 262th image ...\n",
      "computing results for 263th image ...\n",
      "computing results for 264th image ...\n",
      "computing results for 265th image ...\n",
      "computing results for 266th image ...\n",
      "computing results for 267th image ...\n",
      "computing results for 268th image ...\n",
      "computing results for 269th image ...\n",
      "computing results for 270th image ...\n",
      "computing results for 271th image ...\n",
      "computing results for 272th image ...\n",
      "computing results for 273th image ...\n",
      "computing results for 274th image ...\n",
      "computing results for 275th image ...\n",
      "computing results for 276th image ...\n",
      "computing results for 277th image ...\n",
      "computing results for 278th image ...\n",
      "computing results for 279th image ...\n",
      "computing results for 280th image ...\n",
      "computing results for 281th image ...\n",
      "computing results for 282th image ...\n",
      "computing results for 283th image ...\n",
      "computing results for 284th image ...\n",
      "computing results for 285th image ...\n",
      "computing results for 286th image ...\n",
      "computing results for 287th image ...\n",
      "computing results for 288th image ...\n",
      "computing results for 289th image ...\n",
      "computing results for 290th image ...\n",
      "computing results for 291th image ...\n",
      "computing results for 292th image ...\n",
      "computing results for 293th image ...\n",
      "computing results for 294th image ...\n",
      "computing results for 295th image ...\n",
      "computing results for 296th image ...\n",
      "computing results for 297th image ...\n",
      "computing results for 298th image ...\n",
      "computing results for 299th image ...\n",
      "computing results for 300th image ...\n",
      "computing results for 301th image ...\n",
      "computing results for 302th image ...\n",
      "computing results for 303th image ...\n",
      "computing results for 304th image ...\n",
      "computing results for 305th image ...\n",
      "computing results for 306th image ...\n",
      "computing results for 307th image ...\n",
      "computing results for 308th image ...\n",
      "computing results for 309th image ...\n",
      "computing results for 310th image ...\n",
      "computing results for 311th image ...\n",
      "computing results for 312th image ...\n",
      "computing results for 313th image ...\n",
      "computing results for 314th image ...\n",
      "computing results for 315th image ...\n",
      "computing results for 316th image ...\n",
      "computing results for 317th image ...\n",
      "computing results for 318th image ...\n",
      "computing results for 319th image ...\n",
      "computing results for 320th image ...\n",
      "computing results for 321th image ...\n",
      "computing results for 322th image ...\n",
      "computing results for 323th image ...\n",
      "computing results for 324th image ...\n",
      "computing results for 325th image ...\n",
      "computing results for 326th image ...\n",
      "computing results for 327th image ...\n",
      "computing results for 328th image ...\n",
      "computing results for 329th image ...\n",
      "computing results for 330th image ...\n",
      "computing results for 331th image ...\n",
      "computing results for 332th image ...\n",
      "computing results for 333th image ...\n",
      "computing results for 334th image ...\n",
      "computing results for 335th image ...\n",
      "computing results for 336th image ...\n",
      "computing results for 337th image ...\n",
      "computing results for 338th image ...\n",
      "computing results for 339th image ...\n",
      "computing results for 340th image ...\n",
      "computing results for 341th image ...\n",
      "computing results for 342th image ...\n",
      "computing results for 343th image ...\n",
      "computing results for 344th image ...\n",
      "computing results for 345th image ...\n",
      "computing results for 346th image ...\n",
      "computing results for 347th image ...\n",
      "computing results for 348th image ...\n",
      "computing results for 349th image ...\n",
      "computing results for 350th image ...\n",
      "computing results for 351th image ...\n",
      "computing results for 352th image ...\n",
      "computing results for 353th image ...\n",
      "computing results for 354th image ...\n",
      "computing results for 355th image ...\n",
      "computing results for 356th image ...\n",
      "computing results for 357th image ...\n",
      "computing results for 358th image ...\n",
      "computing results for 359th image ...\n",
      "computing results for 360th image ...\n",
      "computing results for 361th image ...\n",
      "computing results for 362th image ...\n",
      "computing results for 363th image ...\n",
      "computing results for 364th image ...\n",
      "computing results for 365th image ...\n",
      "computing results for 366th image ...\n",
      "computing results for 367th image ...\n",
      "computing results for 368th image ...\n",
      "computing results for 369th image ...\n",
      "computing results for 370th image ...\n",
      "computing results for 371th image ...\n",
      "computing results for 372th image ...\n",
      "computing results for 373th image ...\n",
      "computing results for 374th image ...\n",
      "computing results for 375th image ...\n",
      "computing results for 376th image ...\n",
      "computing results for 377th image ...\n",
      "computing results for 378th image ...\n",
      "computing results for 379th image ...\n",
      "computing results for 380th image ...\n",
      "computing results for 381th image ...\n",
      "computing results for 382th image ...\n",
      "computing results for 383th image ...\n",
      "computing results for 384th image ...\n",
      "computing results for 385th image ...\n",
      "computing results for 386th image ...\n",
      "computing results for 387th image ...\n",
      "computing results for 388th image ...\n",
      "computing results for 389th image ...\n",
      "computing results for 390th image ...\n",
      "computing results for 391th image ...\n",
      "computing results for 392th image ...\n",
      "computing results for 393th image ...\n",
      "computing results for 394th image ...\n",
      "computing results for 395th image ...\n",
      "computing results for 396th image ...\n",
      "computing results for 397th image ...\n",
      "computing results for 398th image ...\n",
      "computing results for 399th image ...\n",
      "computing results for 400th image ...\n",
      "computing results for 401th image ...\n",
      "computing results for 402th image ...\n",
      "computing results for 403th image ...\n",
      "computing results for 404th image ...\n",
      "computing results for 405th image ...\n",
      "computing results for 406th image ...\n",
      "computing results for 407th image ...\n",
      "computing results for 408th image ...\n",
      "computing results for 409th image ...\n",
      "computing results for 410th image ...\n",
      "computing results for 411th image ...\n",
      "computing results for 412th image ...\n",
      "computing results for 413th image ...\n",
      "computing results for 414th image ...\n",
      "computing results for 415th image ...\n",
      "computing results for 416th image ...\n",
      "computing results for 417th image ...\n",
      "computing results for 418th image ...\n",
      "computing results for 419th image ...\n",
      "computing results for 420th image ...\n",
      "computing results for 421th image ...\n",
      "computing results for 422th image ...\n",
      "computing results for 423th image ...\n",
      "computing results for 424th image ...\n",
      "computing results for 425th image ...\n",
      "computing results for 426th image ...\n",
      "computing results for 427th image ...\n",
      "computing results for 428th image ...\n",
      "computing results for 429th image ...\n",
      "computing results for 430th image ...\n",
      "computing results for 431th image ...\n",
      "computing results for 432th image ...\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "computing results for 433th image ...\n",
      "computing results for 434th image ...\n",
      "computing results for 435th image ...\n",
      "computing results for 436th image ...\n",
      "computing results for 437th image ...\n",
      "computing results for 438th image ...\n",
      "computing results for 439th image ...\n",
      "computing results for 440th image ...\n",
      "computing results for 441th image ...\n",
      "computing results for 442th image ...\n",
      "computing results for 443th image ...\n",
      "computing results for 444th image ...\n",
      "computing results for 445th image ...\n",
      "computing results for 446th image ...\n",
      "computing results for 447th image ...\n",
      "computing results for 448th image ...\n",
      "computing results for 449th image ...\n",
      "CPU times: user 8min 19s, sys: 1.42 s, total: 8min 21s\n",
      "Wall time: 8min 23s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "# save the predicted class label (highest probability one) in a list\n",
    "print(f\"computing results for {QUANT_NAME_MAP[QUANT_TYPE]} ... \\n\")\n",
    "results = []\n",
    "accuracy_count = 0\n",
    "for i in range(TEST_SAMPLES): \n",
    "    \n",
    "    print(f\"computing results for {i}th image ...\")\n",
    "    \n",
    "    # generate a batch of images \n",
    "    test_image = test_generator.next() \n",
    "    \n",
    "    # set the input image to the input index \n",
    "    interpret.set_tensor(input_ind, test_image[0]) \n",
    "    \n",
    "    # run the inference \n",
    "    interpret.invoke() \n",
    "    \n",
    "    # read the predictions from the output tensor\n",
    "    predictions = interpret.tensor(out_ind) # or, get_tensor(out_ind)\n",
    "    \n",
    "    # get the highest predicted class\n",
    "    pred_class = np.argmax(predictions()[0])\n",
    "    \n",
    "    #print(\"predicted class: \", pred_class, \" and actual class: \", test_generator.classes[i])\n",
    "    \n",
    "    results.append(pred_class)\n",
    "    \n",
    "    if pred_class == test_generator.classes[i]:\n",
    "        accuracy_count += 1 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accuracy percentage for both weights and activations INT8 quantized: 97.778% \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# compute the accuracy percentage\n",
    "print(f\"accuracy percentage for {QUANT_NAME_MAP[QUANT_TYPE]}: {round((accuracy_count / TEST_SAMPLES) * 100, 3)}% \\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------------\n",
      "Confusion matrix for both weights and activations INT8 quantized: \n",
      "\n",
      "[[147   3   0]\n",
      " [  7 143   0]\n",
      " [  0   0 150]]\n",
      "--------------------------------------------------\n",
      "Classification report for both weights and activations INT8 quantized: \n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "        cats       0.95      0.98      0.97       150\n",
      "        dogs       0.98      0.95      0.97       150\n",
      "       panda       1.00      1.00      1.00       150\n",
      "\n",
      "    accuracy                           0.98       450\n",
      "   macro avg       0.98      0.98      0.98       450\n",
      "weighted avg       0.98      0.98      0.98       450\n",
      "\n",
      "--------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "# Plot confusion matrix, classification report\n",
    "print(\"-\"*50)\n",
    "print(f\"Confusion matrix for {QUANT_NAME_MAP[QUANT_TYPE]}: \\n\")\n",
    "print(confusion_matrix(y_true=test_generator.classes, y_pred=results))\n",
    "print(\"-\"*50)\n",
    "print(f\"Classification report for {QUANT_NAME_MAP[QUANT_TYPE]}: \\n\")\n",
    "print(classification_report(y_true=test_generator.classes, y_pred=results, target_names=LABELS))\n",
    "print(\"-\"*50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}