AutonomousDriving_Object_Detection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Z7jy76W958Zm"
   },
   "source": [
    "# Autonomous Driving Object Detection"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "XRgqfTgs574N"
   },
   "outputs": [],
   "source": [
    "# IMPORTS\n",
    "import torch\n",
    "import torchvision\n",
    "import numpy as np\n",
    "import os\n",
    "import yaml\n",
    "\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.image as mpimg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Used to solve the error \"Error #15: Initializing libiomp5md.dll, but found libiomp5md.dll already initialized.\"\n",
    "# This is not a recommended solution, but it is easy to implement.\n",
    "# This problem appear to be present only when using torch and PIL inside anaconda.\n",
    "# In colab the problem is not present. Also, maybe in future try to use the builtin python envs instead of anaconda.\n",
    "os.environ[\"KMP_DUPLICATE_LIB_OK\"]=\"TRUE\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "hF9_ccYhfgHn",
    "outputId": "10152b23-9ffc-4b69-c574-d349d3c3feeb"
   },
   "outputs": [],
   "source": [
    "# Check if GPU is available and set device\n",
    "device = \"cpu\"\n",
    "if torch.cuda.is_available:\n",
    "  print(\"GPU AVAILABLE\")\n",
    "  device = torch.device(\"cuda:0\")\n",
    "else:\n",
    "  print(\"GPU NOT AVAILABLE\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "Hrgft9vS9Zl8"
   },
   "outputs": [],
   "source": [
    "# Set random seed\n",
    "seed = 7\n",
    "torch.manual_seed(seed)\n",
    "torch.backends.cudnn.benchmark = True\n",
    "torch.backends.cudnn.deterministic = True\n",
    "torch.use_deterministic_algorithms(True)\n",
    "os.environ[\"CUBLAS_WORKSPACE_CONFIG\"]=\":4096:8\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "nxfmmuzt0TY6",
    "outputId": "43b3deb4-d66d-4e3e-a06e-12421c1621ad"
   },
   "outputs": [],
   "source": [
    "# Mio url unico\n",
    "!wget -O dataset.zip https://public.roboflow.com/ds/OSbfqB4WlB?key=4L6u3xfkPg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "MQ3U59Fz0bSl"
   },
   "outputs": [],
   "source": [
    "# Unzip the dataset (capture is used to suppress the output of the colab cell)\n",
    "\n",
    "%%capture\n",
    "!unzip dataset.zip -d dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "FZVykiBxyqeb",
    "outputId": "7b0a3ee6-9fc6-4dc2-eb6c-2fd7e32f466b"
   },
   "outputs": [],
   "source": [
    "from data_loading.dir_utils import print_directory_tree, print_file_from_dir\n",
    "\n",
    "# Print directory tree\n",
    "print_directory_tree(\"dataset\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 452
    },
    "id": "EUjhACebcrM1",
    "outputId": "050370f6-5ddd-4f64-e077-e2d057c7d55b"
   },
   "outputs": [],
   "source": [
    "print_file_from_dir(os.path.join(\"dataset\",\"export\",\"images\"), file_n=43)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "U6_AeeEhcrVs",
    "outputId": "b3f9a17a-c624-4f7f-cc46-05dab35d80f9"
   },
   "outputs": [],
   "source": [
    "# Note: annotations for bounding boxes are in the form:\n",
    "# class_id center_x center_y width height\n",
    "# All values are normalized\n",
    "\n",
    "print_file_from_dir(os.path.join(\"dataset\",\"export\",\"labels\"), file_n=100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Rearrange dataset in folders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "ZH_TrKEa1yWH"
   },
   "outputs": [],
   "source": [
    "image_ext = \".jpg\"\n",
    "label_ext = \".txt\"\n",
    "image_width = 512\n",
    "image_height = 512"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "s-GrMPqwhCn8",
    "outputId": "99d0f2a5-b005-464c-f408-12578b7496aa"
   },
   "outputs": [],
   "source": [
    "from data_loading.split_dataset import split_dataset\n",
    "\n",
    "# Separate the samples in 3 sets creating 3 directories: train, validation, test\n",
    "# In each directory copy a random subset of the total data available, without repetitions and with fixed proportions.\n",
    "# At the end we will have the 3 directories each one containing two directories images and labels\n",
    "split_dataset(os.path.join(\"dataset\",\"export\",\"images\"),\n",
    "              image_ext,\n",
    "              os.path.join(\"dataset\",\"export\",\"labels\"),\n",
    "              label_ext,\n",
    "              os.path.join(\"dataset\",\"splitted\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "UA7VZUYotmat",
    "outputId": "9581ba09-f75d-4b16-8418-db11bf19cec9"
   },
   "outputs": [],
   "source": [
    "print_directory_tree(os.path.join(\"dataset\",\"splitted\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "adN7on60xgJU"
   },
   "source": [
    "## Create Dataloaders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Read YAML file to obtain the list of objects classes\n",
    "with open(os.path.join(\"dataset\",\"data.yaml\"), 'r') as yaml_file:\n",
    "    yaml_obj = yaml.safe_load(yaml_file)\n",
    "    classes = yaml_obj[\"names\"]\n",
    "    print(classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "VFxNHcVgwg-j"
   },
   "outputs": [],
   "source": [
    "from data_loading.autonomous_driving_dataset import AutonomousDrivingDataset\n",
    "\n",
    "# Create datasets\n",
    "\n",
    "adod_transforms = {\"train\": torchvision.transforms.Compose([torchvision.transforms.ToTensor()]),\n",
    "                  \"val\": torchvision.transforms.ToTensor()}\n",
    "\n",
    "\n",
    "train_dataset = AutonomousDrivingDataset(os.path.join(\"dataset\", \"splitted\", \"train\", \"images\"),\n",
    "                                         image_ext,\n",
    "                                         image_width,\n",
    "                                         image_height,\n",
    "                                         os.path.join(\"dataset\",\"splitted\",\"train\",\"labels\"),\n",
    "                                         label_ext,\n",
    "                                         classes,\n",
    "                                         adod_transforms[\"train\"])\n",
    "\n",
    "validation_dataset = AutonomousDrivingDataset(os.path.join(\"dataset\", \"splitted\", \"validation\", \"images\"),\n",
    "                                              image_ext,\n",
    "                                              image_width,\n",
    "                                              image_height,\n",
    "                                              os.path.join(\"dataset\",\"splitted\",\"validation\",\"labels\"),\n",
    "                                              label_ext,\n",
    "                                              classes,\n",
    "                                              adod_transforms[\"val\"])\n",
    "\n",
    "test_dataset = AutonomousDrivingDataset(os.path.join(\"dataset\", \"splitted\", \"test\", \"images\"),\n",
    "                                        image_ext,\n",
    "                                        image_width,\n",
    "                                        image_height,\n",
    "                                        os.path.join(\"dataset\",\"splitted\",\"test\",\"labels\"),\n",
    "                                        label_ext,\n",
    "                                        classes,\n",
    "                                        adod_transforms[\"val\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(train_dataset[3])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "eJUYnUvnwhEd"
   },
   "outputs": [],
   "source": [
    "from data_loading.dl_utils import collate_fn\n",
    "\n",
    "# Create dataloaders with datasets\n",
    "\n",
    "num_workers = 0 #4\n",
    "size_batch = 8 #8\n",
    "\n",
    "train_dataloader = torch.utils.data.DataLoader(train_dataset, \n",
    "                                              batch_size=size_batch, \n",
    "                                              shuffle=True, \n",
    "                                              pin_memory=True, \n",
    "                                              num_workers=num_workers,\n",
    "                                              collate_fn=collate_fn)\n",
    "\n",
    "validation_dataloader = torch.utils.data.DataLoader(validation_dataset,\n",
    "                                                    batch_size=size_batch,\n",
    "                                                    shuffle=False,\n",
    "                                                    num_workers=num_workers,\n",
    "                                                    collate_fn=collate_fn)\n",
    "\n",
    "test_dataloader = torch.utils.data.DataLoader(test_dataset,\n",
    "                                              batch_size=size_batch,\n",
    "                                              shuffle=False,\n",
    "                                              num_workers=num_workers,\n",
    "                                              collate_fn=collate_fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Try the dataloader\n",
    "it = iter(train_dataloader)\n",
    "first = next(it)\n",
    "print(first)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "8gq3i58EwhLi"
   },
   "source": [
    "## Define the neural model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torchvision.models.detection.backbone_utils as ut\n",
    "from torchvision.ops.feature_pyramid_network import LastLevelP6P7\n",
    "from torchvision.models.detection import RetinaNet\n",
    "from torchvision.models.detection.anchor_utils import AnchorGenerator\n",
    "\n",
    "\n",
    "use_big_model = False\n",
    "\n",
    "# Instantiate the model\n",
    "\n",
    "if use_big_model:\n",
    "    # The model builders retinanet_resnet50_fpn and retinanet_resnet50_fpn_v2 can be used to instantiate a\n",
    "    # RetinaNet model, with or without pre-trained weights.\n",
    "    # All the model builders internally rely on the torchvision.models.detection.retinanet.RetinaNet base class. \n",
    "    #model = torchvision.models.detection.retinanet_resnet50_fpn(weights=RetinaNet_ResNet50_FPN_Weights.DEFAULT)\n",
    "    model = torchvision.models.detection.retinanet_resnet50_fpn()\n",
    "else:\n",
    "    # https://github.com/pytorch/vision/blob/master/torchvision/models/detection/backbone_utils.py#L49\n",
    "    backbone = ut.resnet_fpn_backbone('resnet18', \n",
    "                                      pretrained=False)\n",
    "\n",
    "    anchor_generator = AnchorGenerator(sizes=(32, 64, 128, 256, 512), aspect_ratios=((0.5, 1.0, 2.0)))\n",
    "\n",
    "    model = RetinaNet(backbone,\n",
    "                      len(train_dataset.classes),\n",
    "                      anchor_generator=anchor_generator)\n",
    "\n",
    "model.to(device)\n",
    "    \n",
    "print(f\"Number of parameters: {sum(p.numel() for p in model.parameters())}\")\n",
    "print(f\"Number of trainable parameters: {sum(p.numel() for p in model.parameters() if p.requires_grad)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Try the model before training it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from object_detection.object_detection_utils import detect_objects, draw_bounding_boxes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = validation_dataset[3][0].to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.imshow(x.cpu().permute(1, 2, 0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(detect_objects(x, model, 0.1, classes))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from training.train import train_one_epoch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "starting_lr = 1e-4\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=starting_lr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = 1\n",
    "for i in range(1, n_epochs+1):\n",
    "    train_one_epoch(model, train_dataloader, optimizer, device)\n",
    "    #TODO: implement also validation with the validation dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#TODO: Test the model with the test dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test the model on a single image\n",
    "x = validation_dataset[3][0].to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "boxes_filtered, scores_filtered, labels_filtered, categories_filtered = detect_objects(x, model, 0.1, classes)\n",
    "print(categories_filtered)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "image = torchvision.transforms.ToPILImage()(x)\n",
    "draw_bounding_boxes(image, boxes_filtered, classes, labels_filtered,\n",
    "                    scores_filtered, colors=[(255,0,0)]*20, normalized_coordinates=False, add_text=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}