test_models.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "df102e9d-57da-40d7-b4c2-4a4646e9e2ac",
   "metadata": {},
   "source": [
    "# Dry & Wet Growing Media Model Evaluation\n",
    "This notebook takes the samples from box 40 and evalautes all models' performance on the 2022-08-15 and 2022-08-24 dates. Metrics include DICE, +IOU, AP, and inference time (in seconds)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "abf63154-f93e-4df3-a65d-2b88ee420644",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2024-06-18 07:55:59,430] [INFO] [real_accelerator.py:203:get_accelerator] Setting ds_accelerator to cuda (auto detect)\n",
      "Warning: The default cache directory for DeepSpeed Triton autotune, /home/chang.spencer/.triton/autotune, appears to be on an NFS system. While this is generally acceptable, if you experience slowdowns or hanging when DeepSpeed exits, it is recommended to set the TRITON_CACHE_DIR environment variable to a non-NFS path.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/lustre/blue/azare/chang.spencer/textures/compiler_compat/ld: cannot find -laio: No such file or directory\n",
      "collect2: error: ld returned 1 exit status\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[93m [WARNING] \u001b[0m async_io requires the dev libaio .so object and headers but these were not found.\n",
      "\u001b[93m [WARNING] \u001b[0m async_io: please install the libaio-devel package with yum\n",
      "\u001b[93m [WARNING] \u001b[0m If libaio is already installed (perhaps from source), try setting the CFLAGS and LDFLAGS environment variables to where it can be found.\n",
      "\u001b[93m [WARNING] \u001b[0m Please specify the CUTLASS repo directory as environment variable $CUTLASS_PATH\n",
      "\u001b[93m [WARNING] \u001b[0m sparse_attn requires a torch version >= 1.5 and < 2.0 but detected 2.0\n",
      "\u001b[93m [WARNING] \u001b[0m using untested triton version (2.0.0), only 1.0.0 is known to be compatible\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "import sys\n",
    "import os\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "os.environ[\"CUDA_HOME\"] = \"/apps/compilers/cuda/12.2.2\"\n",
    "\n",
    "import spectral\n",
    "\n",
    "import torch\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "from src.PLTrainer import validate_net, test_net\n",
    "from src.Experiments.params_HyperPRI import ExpRedGreenBluePRI, ExpHyperspectralPRI"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1d039172-8913-4e1b-9ce4-e931ffae1a86",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Model Parameter Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d02a6e9e-e718-4251-adfe-dc1d7924003f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "start_split = 0\n",
    "num_splits = 5   # Assuming multiple splits\n",
    "rel_call_path = os.path.dirname(os.path.abspath(\"./test_models.ipynb\"))\n",
    "\n",
    "update_params = {\n",
    "    'model_name': [\n",
    "         'UNET',\n",
    "         'SpectralUNET',\n",
    "         'CubeNET',\n",
    "    ],\n",
    "    'dataset': [\n",
    "        'RGB',\n",
    "        'HSI',\n",
    "        'HSI',\n",
    "    ],\n",
    "    # 'device': [\n",
    "    #     'cpu',\n",
    "    #     'cpu',\n",
    "    #     'cpu',\n",
    "    # ],\n",
    "    'spectral_bn_size': [\n",
    "        0,\n",
    "        1650,\n",
    "        0,\n",
    "    ],\n",
    "    'cube_featmaps': [\n",
    "        0,\n",
    "        0,\n",
    "        64,\n",
    "    ],\n",
    "    'criterion': [\n",
    "        torch.nn.BCEWithLogitsLoss(),\n",
    "        torch.nn.BCEWithLogitsLoss(),\n",
    "        torch.nn.BCEWithLogitsLoss(),\n",
    "    ]\n",
    "}\n",
    "thresholds = [\n",
    "    [0.36, 0.41, 0.42, 0.56, 0.38],   # UNET\n",
    "    [0.45, 0.39, 0.48, 0.36, 0.28],   # SpectralUNET\n",
    "    [0.33, 0.46, 0.39, 0.46, 0.27],   # CubeNET\n",
    "]\n",
    "\n",
    "segmaps = [\n",
    "    False,\n",
    "    False,\n",
    "    False,\n",
    "]\n",
    "\n",
    "models = update_params['model_name']\n",
    "datasets = update_params['dataset']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8b69679c-dcee-40ff-b14c-ba9c94d51af0",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## Get Initial Test Metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e9ce4ee5-ea66-4b0f-9ec9-8fc980c6b0ed",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " ********** Split 1 **********\n",
      "   Model: UNET\n",
      "   Test JSON: /blue/azare/chang.spencer/HyperPRI/Datasets/HyperPRI/data_splits/test.json\n",
      "   LOADING FROM CKPT FILE: /blue/azare/chang.spencer/HyperPRI/Saved_Models/RGB/UNET/Run_1/Checkpoints/epoch=69-val_loss=0.079-val_dice=0.816.ckpt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
      "SLURM auto-requeueing enabled. Setting signal handlers.\n",
      "/lustre/blue/azare/chang.spencer/textures/lib/python3.10/site-packages/lightning/pytorch/trainer/connectors/data_connector.py:438: PossibleUserWarning: The dataloader, predict_dataloader, does not have many workers which may be a bottleneck. Consider increasing the value of the `num_workers` argument` (try 128 which is the number of cpus on this machine) in the `DataLoader` init to improve performance.\n",
      "  rank_zero_warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting DataLoader 0: 100%|██████████| 2/2 [00:30<00:00, 15.24s/it]\n",
      "Threshold 0.360:\n",
      "      Pixel Acc: 0.762\n",
      "      DICE     : 0.163\n",
      "      +IOU     : 0.089\n",
      "      Avg Prec : 0.208\n",
      "\n",
      "      Conf Mat : [0.7587399482727051, 0.2412600815296173]\n",
      "                 [0.10382945090532303, 0.8961705565452576]\n",
      "   Model: SpectralUNET_1650\n",
      "   Test JSON: /blue/azare/chang.spencer/HyperPRI/Datasets/HyperPRI/data_splits/test.json\n",
      "   LOADING FROM CKPT FILE: /blue/azare/chang.spencer/HyperPRI/Saved_Models/HSI/SpectralUNET_1650/Run_1/Checkpoints/epoch=341-val_loss=0.166-val_dice=0.669.ckpt\n",
      "      (Converting DeepSpeed ZeRO-2 Checkpoint...)\n",
      "Processing zero checkpoint './checkpoint'\n",
      "Detected checkpoint of type zero stage 2, world_size: 2\n",
      "Parsing checkpoint created by deepspeed==0.14.2\n",
      "Reconstructed fp32 state dict with 38 params 30388051 elements\n",
      "Saving fp32 state dict to deepspeed.ckpt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
      "SLURM auto-requeueing enabled. Setting signal handlers.\n",
      "/lustre/blue/azare/chang.spencer/textures/lib/python3.10/site-packages/spectral/io/envi.py:175: UserWarning: Parameters with non-lowercase names encountered and converted to lowercase. To retain source file parameter name capitalization, set spectral.settings.envi_support_nonlowercase_params to True.\n",
      "  warnings.warn(msg)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting: 0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/lustre/blue/azare/chang.spencer/textures/lib/python3.10/site-packages/spectral/io/envi.py:175: UserWarning: Parameters with non-lowercase names encountered and converted to lowercase. To retain source file parameter name capitalization, set spectral.settings.envi_support_nonlowercase_params to True.\n",
      "  warnings.warn(msg)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting DataLoader 0: 100%|██████████| 1/1 [00:02<00:00,  2.76s/it]\n",
      "Threshold 0.450:\n",
      "      Pixel Acc: 0.898\n",
      "      DICE     : 0.266\n",
      "      +IOU     : 0.154\n",
      "      Avg Prec : 0.335\n",
      "\n",
      "      Conf Mat : [0.90324866771698, 0.09675134718418121]\n",
      "                 [0.28523489832878113, 0.7147650718688965]\n",
      "   Model: CubeNET_64\n",
      "   Test JSON: /blue/azare/chang.spencer/HyperPRI/Datasets/HyperPRI/data_splits/test.json\n",
      "   LOADING FROM CKPT FILE: /blue/azare/chang.spencer/HyperPRI/Saved_Models/HSI/CubeNET_64/Run_1/Checkpoints/epoch=84-val_loss=0.074-val_dice=0.824.ckpt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
      "SLURM auto-requeueing enabled. Setting signal handlers.\n",
      "/lustre/blue/azare/chang.spencer/textures/lib/python3.10/site-packages/spectral/io/envi.py:175: UserWarning: Parameters with non-lowercase names encountered and converted to lowercase. To retain source file parameter name capitalization, set spectral.settings.envi_support_nonlowercase_params to True.\n",
      "  warnings.warn(msg)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting: 0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/lustre/blue/azare/chang.spencer/textures/lib/python3.10/site-packages/spectral/io/envi.py:175: UserWarning: Parameters with non-lowercase names encountered and converted to lowercase. To retain source file parameter name capitalization, set spectral.settings.envi_support_nonlowercase_params to True.\n",
      "  warnings.warn(msg)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting DataLoader 0: 100%|██████████| 1/1 [00:10<00:00, 10.55s/it]\n",
      "Threshold 0.330:\n",
      "      Pixel Acc: 0.922\n",
      "      DICE     : 0.365\n",
      "      +IOU     : 0.223\n",
      "      Avg Prec : 0.618\n",
      "\n",
      "      Conf Mat : [0.9227455854415894, 0.07725439965724945]\n",
      "                 [0.12534543871879578, 0.8746545314788818]\n",
      " ********** Split 2 **********\n",
      "   Model: UNET\n",
      "   Test JSON: /blue/azare/chang.spencer/HyperPRI/Datasets/HyperPRI/data_splits/test.json\n",
      "   LOADING FROM CKPT FILE: /blue/azare/chang.spencer/HyperPRI/Saved_Models/RGB/UNET/Run_2/Checkpoints/epoch=95-val_loss=0.109-val_dice=0.822.ckpt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
      "SLURM auto-requeueing enabled. Setting signal handlers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting DataLoader 0: 100%|██████████| 2/2 [00:00<00:00, 29.38it/s]\n",
      "Threshold 0.410:\n",
      "      Pixel Acc: 0.865\n",
      "      DICE     : 0.239\n",
      "      +IOU     : 0.135\n",
      "      Avg Prec : 0.304\n",
      "\n",
      "      Conf Mat : [0.8661785125732422, 0.13382145762443542]\n",
      "                 [0.18087905645370483, 0.8191209435462952]\n",
      "   Model: SpectralUNET_1650\n",
      "   Test JSON: /blue/azare/chang.spencer/HyperPRI/Datasets/HyperPRI/data_splits/test.json\n",
      "   LOADING FROM CKPT FILE: /blue/azare/chang.spencer/HyperPRI/Saved_Models/HSI/SpectralUNET_1650/Run_2/Checkpoints/epoch=267-val_loss=0.178-val_dice=0.717.ckpt\n",
      "      (Converting DeepSpeed ZeRO-2 Checkpoint...)\n",
      "Processing zero checkpoint './checkpoint'\n",
      "Detected checkpoint of type zero stage 2, world_size: 2\n",
      "Parsing checkpoint created by deepspeed==0.14.2\n",
      "Reconstructed fp32 state dict with 38 params 30388051 elements\n",
      "Saving fp32 state dict to deepspeed.ckpt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
      "SLURM auto-requeueing enabled. Setting signal handlers.\n",
      "/lustre/blue/azare/chang.spencer/textures/lib/python3.10/site-packages/spectral/io/envi.py:175: UserWarning: Parameters with non-lowercase names encountered and converted to lowercase. To retain source file parameter name capitalization, set spectral.settings.envi_support_nonlowercase_params to True.\n",
      "  warnings.warn(msg)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting: 0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/lustre/blue/azare/chang.spencer/textures/lib/python3.10/site-packages/spectral/io/envi.py:175: UserWarning: Parameters with non-lowercase names encountered and converted to lowercase. To retain source file parameter name capitalization, set spectral.settings.envi_support_nonlowercase_params to True.\n",
      "  warnings.warn(msg)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting DataLoader 0: 100%|██████████| 1/1 [00:01<00:00,  1.69s/it]\n",
      "Threshold 0.390:\n",
      "      Pixel Acc: 0.626\n",
      "      DICE     : 0.105\n",
      "      +IOU     : 0.055\n",
      "      Avg Prec : 0.150\n",
      "\n",
      "      Conf Mat : [0.6204839944839478, 0.37951603531837463]\n",
      "                 [0.1546914130449295, 0.8453086018562317]\n",
      "   Model: CubeNET_64\n",
      "   Test JSON: /blue/azare/chang.spencer/HyperPRI/Datasets/HyperPRI/data_splits/test.json\n",
      "   LOADING FROM CKPT FILE: /blue/azare/chang.spencer/HyperPRI/Saved_Models/HSI/CubeNET_64/Run_2/Checkpoints/epoch=50-val_loss=0.104-val_dice=0.831.ckpt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception ignored in: <function _ConnectionBase.__del__ at 0x1503d78013f0>\n",
      "Traceback (most recent call last):\n",
      "  File \"/lustre/blue/azare/chang.spencer/textures/lib/python3.10/multiprocessing/connection.py\", line 132, in __del__\n",
      "    self._close()\n",
      "  File \"/lustre/blue/azare/chang.spencer/textures/lib/python3.10/multiprocessing/connection.py\", line 361, in _close\n",
      "    _close(self._handle)\n",
      "OSError: [Errno 9] Bad file descriptor\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
      "SLURM auto-requeueing enabled. Setting signal handlers.\n",
      "/lustre/blue/azare/chang.spencer/textures/lib/python3.10/site-packages/spectral/io/envi.py:175: UserWarning: Parameters with non-lowercase names encountered and converted to lowercase. To retain source file parameter name capitalization, set spectral.settings.envi_support_nonlowercase_params to True.\n",
      "  warnings.warn(msg)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting: 0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/lustre/blue/azare/chang.spencer/textures/lib/python3.10/site-packages/spectral/io/envi.py:175: UserWarning: Parameters with non-lowercase names encountered and converted to lowercase. To retain source file parameter name capitalization, set spectral.settings.envi_support_nonlowercase_params to True.\n",
      "  warnings.warn(msg)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting DataLoader 0: 100%|██████████| 1/1 [00:00<00:00,  8.44it/s]\n",
      "Threshold 0.460:\n",
      "      Pixel Acc: 0.633\n",
      "      DICE     : 0.113\n",
      "      +IOU     : 0.060\n",
      "      Avg Prec : 0.403\n",
      "\n",
      "      Conf Mat : [0.6260730624198914, 0.37392690777778625]\n",
      "                 [0.09616396576166153, 0.9038360118865967]\n",
      " ********** Split 3 **********\n",
      "   Model: UNET\n",
      "   Test JSON: /blue/azare/chang.spencer/HyperPRI/Datasets/HyperPRI/data_splits/test.json\n",
      "   LOADING FROM CKPT FILE: /blue/azare/chang.spencer/HyperPRI/Saved_Models/RGB/UNET/Run_3/Checkpoints/epoch=61-val_loss=0.069-val_dice=0.860.ckpt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
      "SLURM auto-requeueing enabled. Setting signal handlers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting DataLoader 0: 100%|██████████| 2/2 [00:00<00:00, 24.50it/s]\n",
      "Threshold 0.420:\n",
      "      Pixel Acc: 0.525\n",
      "      DICE     : 0.091\n",
      "      +IOU     : 0.048\n",
      "      Avg Prec : 0.099\n",
      "\n",
      "      Conf Mat : [0.5142174363136292, 0.48578259348869324]\n",
      "                 [0.080175019800663, 0.9198249578475952]\n",
      "   Model: SpectralUNET_1650\n",
      "   Test JSON: /blue/azare/chang.spencer/HyperPRI/Datasets/HyperPRI/data_splits/test.json\n",
      "   LOADING FROM CKPT FILE: /blue/azare/chang.spencer/HyperPRI/Saved_Models/HSI/SpectralUNET_1650/Run_3/Checkpoints/epoch=269-val_loss=0.132-val_dice=0.764.ckpt\n",
      "      (Converting DeepSpeed ZeRO-2 Checkpoint...)\n",
      "Processing zero checkpoint './checkpoint'\n",
      "Detected checkpoint of type zero stage 2, world_size: 2\n",
      "Parsing checkpoint created by deepspeed==0.14.2\n",
      "Reconstructed fp32 state dict with 38 params 30388051 elements\n",
      "Saving fp32 state dict to deepspeed.ckpt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
      "SLURM auto-requeueing enabled. Setting signal handlers.\n",
      "/lustre/blue/azare/chang.spencer/textures/lib/python3.10/site-packages/spectral/io/envi.py:175: UserWarning: Parameters with non-lowercase names encountered and converted to lowercase. To retain source file parameter name capitalization, set spectral.settings.envi_support_nonlowercase_params to True.\n",
      "  warnings.warn(msg)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting: 0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/lustre/blue/azare/chang.spencer/textures/lib/python3.10/site-packages/spectral/io/envi.py:175: UserWarning: Parameters with non-lowercase names encountered and converted to lowercase. To retain source file parameter name capitalization, set spectral.settings.envi_support_nonlowercase_params to True.\n",
      "  warnings.warn(msg)\n",
      "Exception ignored in: <function _MultiProcessingDataLoaderIter.__del__ at 0x1503d72e7490>\n",
      "Traceback (most recent call last):\n",
      "  File \"/lustre/blue/azare/chang.spencer/textures/lib/python3.10/site-packages/torch/utils/data/dataloader.py\", line 1478, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/lustre/blue/azare/chang.spencer/textures/lib/python3.10/site-packages/torch/utils/data/dataloader.py\", line 1461, in _shutdown_workers\n",
      "    if w.is_alive():\n",
      "  File \"/lustre/blue/azare/chang.spencer/textures/lib/python3.10/multiprocessing/process.py\", line 160, in is_alive\n",
      "    assert self._parent_pid == os.getpid(), 'can only test a child process'\n",
      "AssertionError: can only test a child process\n",
      "Exception ignored in:   File \"/lustre/blue/azare/chang.spencer/textures/lib/python3.10/site-packages/torch/utils/data/dataloader.py\", line 1478, in __del__\n",
      "<function _MultiProcessingDataLoaderIter.__del__ at 0x1503d72e7490>\n",
      "Traceback (most recent call last):\n",
      "    self._shutdown_workers()\n",
      "  File \"/lustre/blue/azare/chang.spencer/textures/lib/python3.10/site-packages/torch/utils/data/dataloader.py\", line 1461, in _shutdown_workers\n",
      "    if w.is_alive():\n",
      "  File \"/lustre/blue/azare/chang.spencer/textures/lib/python3.10/multiprocessing/process.py\", line 160, in is_alive\n",
      "    assert self._parent_pid == os.getpid(), 'can only test a child process'\n",
      "AssertionError: can only test a child process\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting DataLoader 0: 100%|██████████| 1/1 [00:01<00:00,  1.70s/it]\n",
      "Threshold 0.480:\n",
      "      Pixel Acc: 0.770\n",
      "      DICE     : 0.143\n",
      "      +IOU     : 0.077\n",
      "      Avg Prec : 0.211\n",
      "\n",
      "      Conf Mat : [0.7704082727432251, 0.2295917272567749]\n",
      "                 [0.25480327010154724, 0.7451967597007751]\n",
      "   Model: CubeNET_64\n",
      "   Test JSON: /blue/azare/chang.spencer/HyperPRI/Datasets/HyperPRI/data_splits/test.json\n",
      "   LOADING FROM CKPT FILE: /blue/azare/chang.spencer/HyperPRI/Saved_Models/HSI/CubeNET_64/Run_3/Checkpoints/epoch=112-val_loss=0.067-val_dice=0.861.ckpt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
      "SLURM auto-requeueing enabled. Setting signal handlers.\n",
      "/lustre/blue/azare/chang.spencer/textures/lib/python3.10/site-packages/spectral/io/envi.py:175: UserWarning: Parameters with non-lowercase names encountered and converted to lowercase. To retain source file parameter name capitalization, set spectral.settings.envi_support_nonlowercase_params to True.\n",
      "  warnings.warn(msg)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting: 0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/lustre/blue/azare/chang.spencer/textures/lib/python3.10/site-packages/spectral/io/envi.py:175: UserWarning: Parameters with non-lowercase names encountered and converted to lowercase. To retain source file parameter name capitalization, set spectral.settings.envi_support_nonlowercase_params to True.\n",
      "  warnings.warn(msg)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting DataLoader 0: 100%|██████████| 1/1 [00:00<00:00,  3.65it/s]\n",
      "Threshold 0.390:\n",
      "      Pixel Acc: 0.979\n",
      "      DICE     : 0.620\n",
      "      +IOU     : 0.449\n",
      "      Avg Prec : 0.630\n",
      "\n",
      "      Conf Mat : [0.9875799417495728, 0.012420074082911015]\n",
      "                 [0.3406369388103485, 0.6593630909919739]\n",
      " ********** Split 4 **********\n",
      "   Model: UNET\n",
      "   Test JSON: /blue/azare/chang.spencer/HyperPRI/Datasets/HyperPRI/data_splits/test.json\n",
      "   LOADING FROM CKPT FILE: /blue/azare/chang.spencer/HyperPRI/Saved_Models/RGB/UNET/Run_4/Checkpoints/epoch=84-val_loss=0.074-val_dice=0.843.ckpt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
      "SLURM auto-requeueing enabled. Setting signal handlers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting DataLoader 0: 100%|██████████| 2/2 [00:00<00:00, 22.34it/s]\n",
      "Threshold 0.560:\n",
      "      Pixel Acc: 0.676\n",
      "      DICE     : 0.118\n",
      "      +IOU     : 0.063\n",
      "      Avg Prec : 0.200\n",
      "\n",
      "      Conf Mat : [0.6715090274810791, 0.3284910023212433]\n",
      "                 [0.16011975705623627, 0.8398802280426025]\n",
      "   Model: SpectralUNET_1650\n",
      "   Test JSON: /blue/azare/chang.spencer/HyperPRI/Datasets/HyperPRI/data_splits/test.json\n",
      "   LOADING FROM CKPT FILE: /blue/azare/chang.spencer/HyperPRI/Saved_Models/HSI/SpectralUNET_1650/Run_4/Checkpoints/epoch=559-val_loss=0.119-val_dice=0.735.ckpt\n",
      "      (Converting DeepSpeed ZeRO-2 Checkpoint...)\n",
      "Processing zero checkpoint './checkpoint'\n",
      "Detected checkpoint of type zero stage 2, world_size: 2\n",
      "Parsing checkpoint created by deepspeed==0.14.2\n",
      "Reconstructed fp32 state dict with 38 params 30388051 elements\n",
      "Saving fp32 state dict to deepspeed.ckpt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
      "SLURM auto-requeueing enabled. Setting signal handlers.\n",
      "/lustre/blue/azare/chang.spencer/textures/lib/python3.10/site-packages/spectral/io/envi.py:175: UserWarning: Parameters with non-lowercase names encountered and converted to lowercase. To retain source file parameter name capitalization, set spectral.settings.envi_support_nonlowercase_params to True.\n",
      "  warnings.warn(msg)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting: 0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/lustre/blue/azare/chang.spencer/textures/lib/python3.10/site-packages/spectral/io/envi.py:175: UserWarning: Parameters with non-lowercase names encountered and converted to lowercase. To retain source file parameter name capitalization, set spectral.settings.envi_support_nonlowercase_params to True.\n",
      "  warnings.warn(msg)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting DataLoader 0: 100%|██████████| 1/1 [00:01<00:00,  1.64s/it]\n",
      "Threshold 0.360:\n",
      "      Pixel Acc: 0.614\n",
      "      DICE     : 0.094\n",
      "      +IOU     : 0.049\n",
      "      Avg Prec : 0.114\n",
      "\n",
      "      Conf Mat : [0.6096441149711609, 0.3903559148311615]\n",
      "                 [0.2285498082637787, 0.7714501619338989]\n",
      "   Model: CubeNET_64\n",
      "   Test JSON: /blue/azare/chang.spencer/HyperPRI/Datasets/HyperPRI/data_splits/test.json\n",
      "   LOADING FROM CKPT FILE: /blue/azare/chang.spencer/HyperPRI/Saved_Models/HSI/CubeNET_64/Run_4/Checkpoints/epoch=77-val_loss=0.072-val_dice=0.841.ckpt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception ignored in: <function _ConnectionBase.__del__ at 0x1503d78013f0>\n",
      "Traceback (most recent call last):\n",
      "  File \"/lustre/blue/azare/chang.spencer/textures/lib/python3.10/multiprocessing/connection.py\", line 132, in __del__\n",
      "    self._close()\n",
      "  File \"/lustre/blue/azare/chang.spencer/textures/lib/python3.10/multiprocessing/connection.py\", line 361, in _close\n",
      "    _close(self._handle)\n",
      "OSError: [Errno 9] Bad file descriptor\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
      "SLURM auto-requeueing enabled. Setting signal handlers.\n",
      "/lustre/blue/azare/chang.spencer/textures/lib/python3.10/site-packages/spectral/io/envi.py:175: UserWarning: Parameters with non-lowercase names encountered and converted to lowercase. To retain source file parameter name capitalization, set spectral.settings.envi_support_nonlowercase_params to True.\n",
      "  warnings.warn(msg)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting: 0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/lustre/blue/azare/chang.spencer/textures/lib/python3.10/site-packages/spectral/io/envi.py:175: UserWarning: Parameters with non-lowercase names encountered and converted to lowercase. To retain source file parameter name capitalization, set spectral.settings.envi_support_nonlowercase_params to True.\n",
      "  warnings.warn(msg)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting DataLoader 0: 100%|██████████| 1/1 [00:00<00:00,  4.26it/s]\n",
      "Threshold 0.460:\n",
      "      Pixel Acc: 0.983\n",
      "      DICE     : 0.658\n",
      "      +IOU     : 0.490\n",
      "      Avg Prec : 0.712\n",
      "\n",
      "      Conf Mat : [0.9923143982887268, 0.007685585878789425]\n",
      "                 [0.36800894141197205, 0.6319910287857056]\n",
      " ********** Split 5 **********\n",
      "   Model: UNET\n",
      "   Test JSON: /blue/azare/chang.spencer/HyperPRI/Datasets/HyperPRI/data_splits/test.json\n",
      "   LOADING FROM CKPT FILE: /blue/azare/chang.spencer/HyperPRI/Saved_Models/RGB/UNET/Run_5/Checkpoints/epoch=81-val_loss=0.071-val_dice=0.816.ckpt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
      "SLURM auto-requeueing enabled. Setting signal handlers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting DataLoader 0: 100%|██████████| 2/2 [00:00<00:00, 30.22it/s]\n",
      "Threshold 0.380:\n",
      "      Pixel Acc: 0.835\n",
      "      DICE     : 0.199\n",
      "      +IOU     : 0.111\n",
      "      Avg Prec : 0.317\n",
      "\n",
      "      Conf Mat : [0.8355234265327454, 0.16447660326957703]\n",
      "                 [0.2022305577993393, 0.7977694272994995]\n",
      "   Model: SpectralUNET_1650\n",
      "   Test JSON: /blue/azare/chang.spencer/HyperPRI/Datasets/HyperPRI/data_splits/test.json\n",
      "   LOADING FROM CKPT FILE: /blue/azare/chang.spencer/HyperPRI/Saved_Models/HSI/SpectralUNET_1650/Run_5/Checkpoints/epoch=414-val_loss=0.136-val_dice=0.616.ckpt\n",
      "      (Converting DeepSpeed ZeRO-2 Checkpoint...)\n",
      "Processing zero checkpoint './checkpoint'\n",
      "Detected checkpoint of type zero stage 2, world_size: 2\n",
      "Parsing checkpoint created by deepspeed==0.14.2\n",
      "Reconstructed fp32 state dict with 38 params 30388051 elements\n",
      "Saving fp32 state dict to deepspeed.ckpt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
      "SLURM auto-requeueing enabled. Setting signal handlers.\n",
      "/lustre/blue/azare/chang.spencer/textures/lib/python3.10/site-packages/spectral/io/envi.py:175: UserWarning: Parameters with non-lowercase names encountered and converted to lowercase. To retain source file parameter name capitalization, set spectral.settings.envi_support_nonlowercase_params to True.\n",
      "  warnings.warn(msg)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting: 0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/lustre/blue/azare/chang.spencer/textures/lib/python3.10/site-packages/spectral/io/envi.py:175: UserWarning: Parameters with non-lowercase names encountered and converted to lowercase. To retain source file parameter name capitalization, set spectral.settings.envi_support_nonlowercase_params to True.\n",
      "  warnings.warn(msg)\n",
      "Exception ignored in: <function _MultiProcessingDataLoaderIter.__del__ at 0x1503d72e7490>\n",
      "Traceback (most recent call last):\n",
      "  File \"/lustre/blue/azare/chang.spencer/textures/lib/python3.10/site-packages/torch/utils/data/dataloader.py\", line 1478, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/lustre/blue/azare/chang.spencer/textures/lib/python3.10/site-packages/torch/utils/data/dataloader.py\", line 1461, in _shutdown_workers\n",
      "    if w.is_alive():\n",
      "  File \"/lustre/blue/azare/chang.spencer/textures/lib/python3.10/multiprocessing/process.py\", line 160, in is_alive\n",
      "    \n",
      "assert self._parent_pid == os.getpid(), 'can only test a child process'AssertionError: can only test a child process\n",
      "Exception ignored in: <function _MultiProcessingDataLoaderIter.__del__ at 0x1503d72e7490>\n",
      "Traceback (most recent call last):\n",
      "  File \"/lustre/blue/azare/chang.spencer/textures/lib/python3.10/site-packages/torch/utils/data/dataloader.py\", line 1478, in __del__\n",
      "    self._shutdown_workers()\n",
      "  File \"/lustre/blue/azare/chang.spencer/textures/lib/python3.10/site-packages/torch/utils/data/dataloader.py\", line 1461, in _shutdown_workers\n",
      "    if w.is_alive():\n",
      "  File \"/lustre/blue/azare/chang.spencer/textures/lib/python3.10/multiprocessing/process.py\", line 160, in is_alive\n",
      "    assert self._parent_pid == os.getpid(), 'can only test a child process'\n",
      "AssertionError: can only test a child process\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting DataLoader 0: 100%|██████████| 1/1 [00:01<00:00,  1.65s/it]\n",
      "Threshold 0.280:\n",
      "      Pixel Acc: 0.845\n",
      "      DICE     : 0.196\n",
      "      +IOU     : 0.109\n",
      "      Avg Prec : 0.292\n",
      "\n",
      "      Conf Mat : [0.8485277891159058, 0.15147224068641663]\n",
      "                 [0.26990392804145813, 0.7300960421562195]\n",
      "   Model: CubeNET_64\n",
      "   Test JSON: /blue/azare/chang.spencer/HyperPRI/Datasets/HyperPRI/data_splits/test.json\n",
      "   LOADING FROM CKPT FILE: /blue/azare/chang.spencer/HyperPRI/Saved_Models/HSI/CubeNET_64/Run_5/Checkpoints/epoch=98-val_loss=0.069-val_dice=0.807.ckpt\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Exception ignored in: <function _ConnectionBase.__del__ at 0x1503d78013f0>\n",
      "Traceback (most recent call last):\n",
      "  File \"/lustre/blue/azare/chang.spencer/textures/lib/python3.10/multiprocessing/connection.py\", line 132, in __del__\n",
      "    self._close()\n",
      "  File \"/lustre/blue/azare/chang.spencer/textures/lib/python3.10/multiprocessing/connection.py\", line 361, in _close\n",
      "    _close(self._handle)\n",
      "OSError: [Errno 9] Bad file descriptor\n",
      "GPU available: True (cuda), used: True\n",
      "TPU available: False, using: 0 TPU cores\n",
      "IPU available: False, using: 0 IPUs\n",
      "HPU available: False, using: 0 HPUs\n",
      "LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0]\n",
      "SLURM auto-requeueing enabled. Setting signal handlers.\n",
      "/lustre/blue/azare/chang.spencer/textures/lib/python3.10/site-packages/spectral/io/envi.py:175: UserWarning: Parameters with non-lowercase names encountered and converted to lowercase. To retain source file parameter name capitalization, set spectral.settings.envi_support_nonlowercase_params to True.\n",
      "  warnings.warn(msg)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting: 0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/lustre/blue/azare/chang.spencer/textures/lib/python3.10/site-packages/spectral/io/envi.py:175: UserWarning: Parameters with non-lowercase names encountered and converted to lowercase. To retain source file parameter name capitalization, set spectral.settings.envi_support_nonlowercase_params to True.\n",
      "  warnings.warn(msg)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicting DataLoader 0: 100%|██████████| 1/1 [00:00<00:00,  8.98it/s]\n",
      "Threshold 0.270:\n",
      "      Pixel Acc: 0.974\n",
      "      DICE     : 0.597\n",
      "      +IOU     : 0.425\n",
      "      Avg Prec : 0.687\n",
      "\n",
      "      Conf Mat : [0.9795664548873901, 0.020433560013771057]\n",
      "                 [0.24736806750297546, 0.7526319026947021]\n"
     ]
    }
   ],
   "source": [
    "for run in range(start_split, num_splits):\n",
    "    print(f\" ********** Split {run+1} **********\")\n",
    "\n",
    "    for m_idx, (m, dset) in enumerate(zip(models, datasets)):\n",
    "        change_params = {}\n",
    "        for k_idx, k in enumerate(update_params):\n",
    "            change_params[k] = update_params[k][m_idx]\n",
    "\n",
    "        split_no = run + 1\n",
    "        if dset.lower() == 'rgb':\n",
    "            exp_params = ExpRedGreenBluePRI(rel_call_path, split_no=run+1, augment=False)\n",
    "            # Switch to a different model (ie. change internal parameter strings)\n",
    "            exp_params.change_network_param(m, rel_call_path, run+1, model_params=change_params)  # Num bins\n",
    "        else:\n",
    "            exp_params = ExpHyperspectralPRI(rel_call_path, split_no=run+1)\n",
    "            # Switch to a different model (ie. change internal parameter strings)\n",
    "            exp_params.change_network_param(m, rel_call_path, run+1, model_params=None)  # Num bins\n",
    "        exp_params.json_dir['test'] = os.path.join(f\"{exp_params.data_dir}\", \"data_splits\", \"test.json\")\n",
    "\n",
    "        print(f\"   Model: {exp_params.model_param_str}\")\n",
    "        print(f\"   Test JSON: {exp_params.json_dir['test']}\")\n",
    "        # pr_curve_info = validate_net(exp_params.get_val_data(),\n",
    "        test_net(exp_params.get_test_data(), exp_params,\n",
    "                 best_threshold=thresholds[m_idx][run],\n",
    "                 save_segmaps=segmaps[m_idx])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e69f5dba-b1cc-437b-9d38-1b28e138ecbc",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# temp_dataset = exp_params.get_test_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "a3c55dd1-54cf-481b-abe4-8f391594a5b4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# import json\n",
    "# with open(exp_params.json_dir['test'], 'r') as f:\n",
    "#     data_dict = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "1fa5e0a4-6aaf-4949-a473-a9b57f65cf55",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# box_names = list(data_dict.keys())\n",
    "# box = box_names[-1]\n",
    "\n",
    "# box_class = data_dict[box]['plant_folder']\n",
    "# box_sz = data_dict[box]['resolution']\n",
    "# img_dir = f\"{temp_dataset.root}/{box_class}_{box_sz}/{data_dict['img_dir']}/\"\n",
    "# label_dir = f\"{temp_dataset.root}/{box_class}_{box_sz}/{data_dict['mask_dir']}/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "3e863971-81c1-45e0-8ad1-df08f3d2bcf3",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['20220815', '20220824']"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# data_dict[box]['dates']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d34b207c-8b08-4c69-bfc4-fadcfc882f58",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Manual Model Evaluation\n",
    "For each model, evaluate the CPU inference time when using each model in evaluation mode. This is averaged across 5 runs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a510a599-9cee-4ff9-8900-b7fe238c97a1",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: UNET\n",
      "   0.018 s\n",
      "   0.003 s\n",
      "   0.003 s\n",
      "   0.008 s\n",
      "   0.003 s\n",
      "   Average: 0.0072 s\n",
      "\n",
      "Model: SpectralUNET\n",
      "   0.017 s\n",
      "   0.062 s\n",
      "   0.002 s\n",
      "   0.005 s\n",
      "   0.002 s\n",
      "   Average: 0.0174 s\n",
      "\n",
      "Model: CubeNET\n",
      "   0.011 s\n",
      "   0.010 s\n",
      "   0.007 s\n",
      "   0.008 s\n",
      "   0.045 s\n",
      "   Average: 0.0159 s\n",
      "\n"
     ]
    }
   ],
   "source": [
    "spectral.settings.envi_support_nonlowercase_params = True\n",
    "with torch.no_grad():\n",
    "\n",
    "    # Number of times to average the inference time on GPU\n",
    "    num_splits = 5\n",
    "    \n",
    "    for m_idx, (m, dset) in enumerate(zip(models, datasets)):\n",
    "        print(f\"Model: {m}\")\n",
    "        final_time = 0\n",
    "        for run in range(start_split, num_splits):\n",
    "            \n",
    "            change_params = {}\n",
    "            for k_idx, k in enumerate(update_params):\n",
    "                change_params[k] = update_params[k][m_idx]\n",
    "\n",
    "            # change_params['device'] = 'cpu'\n",
    "\n",
    "            split_no = run + 1\n",
    "            if dset.lower() == 'rgb':\n",
    "                exp_params = ExpRedGreenBluePRI(rel_call_path, split_no=run+1, augment=False)\n",
    "                # Switch to a different model (ie. change internal parameter strings)\n",
    "                exp_params.change_network_param(m, rel_call_path, run+1, model_params=change_params)  # Num bins\n",
    "            else:\n",
    "                exp_params = ExpHyperspectralPRI(rel_call_path, split_no=run+1)\n",
    "                # Switch to a different model (ie. change internal parameter strings)\n",
    "                exp_params.change_network_param(m, rel_call_path, run+1, model_params=None)  # Num bins\n",
    "            exp_params.json_dir['test'] = os.path.join(f\"{exp_params.data_dir}\", \"data_splits\", \"test.json\")\n",
    "\n",
    "            net = exp_params.get_network()\n",
    "            test_loader = DataLoader(exp_params.get_test_data(), batch_size=1)\n",
    "            for batch in test_loader:\n",
    "                X_img = batch['image']\n",
    "                break\n",
    "            \n",
    "            if exp_params.device.lower() == 'gpu' and torch.cuda.is_available() and torch.cuda.device_count() > 0:\n",
    "                net = net.to('cuda:0')\n",
    "                X_img = X_img.to('cuda:0')\n",
    "\n",
    "            t_start = time.time()\n",
    "            pred = net(X_img)\n",
    "            t_stop = time.time()\n",
    "\n",
    "            del net\n",
    "            del X_img\n",
    "\n",
    "            time_diff = t_stop - t_start\n",
    "\n",
    "            print(f\"   {time_diff:.3f} s\")\n",
    "            final_time += time_diff\n",
    "        print(f\"   Average: {final_time / num_splits:.4f} s\", end=\"\\n\\n\")\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "textures-kernel",
   "language": "python",
   "name": "textures"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}