LogME_practice.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Download features\n",
        "PR layer1-layer5"
      ],
      "metadata": {
        "id": "MXtTnvmXWMmT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "! gdown --id 1GvwThxv4FZoSpLxXVuagTJiIWDEk1FQm\n",
        "! unzip LogME-CTC.zip"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CR8jVQB8WKeS",
        "outputId": "eb2f9765-4334-4a0e-8b51-5f60b3418901"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/usr/local/lib/python3.8/dist-packages/gdown/cli.py:127: FutureWarning: Option `--id` was deprecated in version 4.3.1 and will be removed in 5.0. You don't need to pass it anymore to use a file ID.\n",
            "  warnings.warn(\n",
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=1GvwThxv4FZoSpLxXVuagTJiIWDEk1FQm\n",
            "To: /content/LogME-CTC.zip\n",
            "100% 711k/711k [00:00<00:00, 147MB/s]\n",
            "Archive:  LogME-CTC.zip\n",
            "   creating: LogME-CTC/\n",
            "  inflating: LogME-CTC/layer_1_groundtruth.txt  \n",
            "  inflating: LogME-CTC/layer_1_output.txt  \n",
            "  inflating: LogME-CTC/layer_2_output.txt  \n",
            "  inflating: LogME-CTC/layer_3_output.txt  \n",
            "  inflating: LogME-CTC/layer_4_output.txt  \n",
            " extracting: LogME-CTC/pr_feature.npy  \n",
            " extracting: LogME-CTC/ranking_feature_layer_1.npy  \n",
            " extracting: LogME-CTC/ranking_feature_layer_2.npy  \n",
            " extracting: LogME-CTC/ranking_feature_layer_3.npy  \n",
            " extracting: LogME-CTC/ranking_feature_layer_4.npy  \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import warnings\n",
        "\n",
        "import numpy as np\n",
        "from numba import njit\n",
        "\n",
        "\n",
        "@njit\n",
        "def each_evidence(y_, f, fh, v, s, vh, N, D):\n",
        "    \"\"\"\n",
        "    compute the maximum evidence for each class\n",
        "    \"\"\"\n",
        "    epsilon = 1e-5\n",
        "    alpha = 1.0\n",
        "    beta = 1.0\n",
        "    lam = alpha / beta\n",
        "    tmp = (vh @ (f @ np.ascontiguousarray(y_)))\n",
        "    for _ in range(11):\n",
        "        # should converge after at most 10 steps\n",
        "        # typically converge after two or three steps\n",
        "        gamma = (s / (s + lam)).sum()\n",
        "        # A = v @ np.diag(alpha + beta * s) @ v.transpose() # no need to compute A\n",
        "        # A_inv = v @ np.diag(1.0 / (alpha + beta * s)) @ v.transpose() # no need to compute A_inv\n",
        "        m = v @ (tmp * beta / (alpha + beta * s))\n",
        "        alpha_de = (m * m).sum()\n",
        "        alpha = gamma / (alpha_de + epsilon)\n",
        "        beta_de = ((y_ - fh @ m) ** 2).sum()\n",
        "        beta = (N - gamma) / (beta_de + epsilon)\n",
        "        new_lam = alpha / beta\n",
        "        if np.abs(new_lam - lam) / lam < 0.01:\n",
        "            break\n",
        "        lam = new_lam\n",
        "    evidence = D / 2.0 * np.log(alpha) \\\n",
        "               + N / 2.0 * np.log(beta) \\\n",
        "               - 0.5 * np.sum(np.log(alpha + beta * s)) \\\n",
        "               - beta / 2.0 * (beta_de + epsilon) \\\n",
        "               - alpha / 2.0 * (alpha_de + epsilon) \\\n",
        "               - N / 2.0 * np.log(2 * np.pi)\n",
        "    return evidence / N, alpha, beta, m\n",
        "\n",
        "\n",
        "# use pseudo data to compile the function\n",
        "# D = 20, N = 50\n",
        "f_tmp = np.random.randn(20, 50).astype(np.float64)\n",
        "each_evidence(np.random.randint(0, 2, 50).astype(np.float64), f_tmp, f_tmp.transpose(), np.eye(20, dtype=np.float64), np.ones(20, dtype=np.float64), np.eye(20, dtype=np.float64), 50, 20)\n",
        "\n",
        "\n",
        "@njit\n",
        "def truncated_svd(x):\n",
        "    u, s, vh = np.linalg.svd(x.transpose() @ x)\n",
        "    s = np.sqrt(s)\n",
        "    u_times_sigma = x @ vh.transpose()\n",
        "    k = np.sum((s > 1e-10) * 1)  # rank of f\n",
        "    s = s.reshape(-1, 1)\n",
        "    s = s[:k]\n",
        "    vh = vh[:k]\n",
        "    u = u_times_sigma[:, :k] / s.reshape(1, -1)\n",
        "    return u, s, vh\n",
        "truncated_svd(np.random.randn(20, 10).astype(np.float64))\n",
        "\n",
        "\n",
        "class LogME(object):\n",
        "    def __init__(self, regression=False):\n",
        "        \"\"\"\n",
        "            :param regression: whether regression\n",
        "        \"\"\"\n",
        "        self.regression = regression\n",
        "        self.fitted = False\n",
        "        self.reset()\n",
        "\n",
        "    def reset(self):\n",
        "        self.num_dim = 0\n",
        "        self.alphas = []  # alpha for each class / dimension\n",
        "        self.betas = []  # beta for each class / dimension\n",
        "        # self.ms.shape --> [C, D]\n",
        "        self.ms = []  # m for each class / dimension\n",
        "\n",
        "    def _fit_icml(self, f: np.ndarray, y: np.ndarray):\n",
        "        \"\"\"\n",
        "        LogME calculation proposed in the ICML 2021 paper\n",
        "        \"LogME: Practical Assessment of Pre-trained Models for Transfer Learning\"\n",
        "        at http://proceedings.mlr.press/v139/you21b.html\n",
        "        \"\"\"\n",
        "        fh = f\n",
        "        f = f.transpose()\n",
        "        D, N = f.shape\n",
        "        v, s, vh = np.linalg.svd(f @ fh, full_matrices=True)\n",
        "\n",
        "        evidences = []\n",
        "        self.num_dim = y.shape[1] if self.regression else int(y.max() + 1)\n",
        "        for i in range(self.num_dim):\n",
        "            y_ = y[:, i] if self.regression else (y == i).astype(np.float64)\n",
        "            evidence, alpha, beta, m = each_evidence(y_, f, fh, v, s, vh, N, D)\n",
        "            evidences.append(evidence)\n",
        "            self.alphas.append(alpha)\n",
        "            self.betas.append(beta)\n",
        "            self.ms.append(m)\n",
        "        self.ms = np.stack(self.ms)\n",
        "        return np.mean(evidences)\n",
        "\n",
        "    def _fit_fixed_point(self, f: np.ndarray, y: np.ndarray):\n",
        "        \"\"\"\n",
        "        LogME calculation proposed in the arxiv 2021 paper\n",
        "        \"Ranking and Tuning Pre-trained Models: A New Paradigm of Exploiting Model Hubs\"\n",
        "        at https://arxiv.org/abs/2110.10545\n",
        "        \"\"\"\n",
        "        N, D = f.shape  # k = min(N, D)\n",
        "        if N > D: # direct SVD may be expensive\n",
        "            u, s, vh = truncated_svd(f)\n",
        "        else:\n",
        "            u, s, vh = np.linalg.svd(f, full_matrices=False)\n",
        "        # u.shape = N x k\n",
        "        # s.shape = k\n",
        "        # vh.shape = k x D\n",
        "        s = s.reshape(-1, 1)\n",
        "        sigma = (s ** 2)\n",
        "\n",
        "        evidences = []\n",
        "        self.num_dim = y.shape[1] if self.regression else int(y.max() + 1)\n",
        "        for i in range(self.num_dim):\n",
        "            y_ = y[:, i] if self.regression else (y == i).astype(np.float64)\n",
        "            y_ = y_.reshape(-1, 1)\n",
        "            x = u.T @ y_  # x has shape [k, 1], but actually x should have shape [N, 1]\n",
        "            x2 = x ** 2\n",
        "            res_x2 = (y_ ** 2).sum() - x2.sum()  # if k < N, we compute sum of xi for 0 singular values directly\n",
        "\n",
        "            alpha, beta = 1.0, 1.0\n",
        "            for _ in range(11):\n",
        "                t = alpha / beta\n",
        "                gamma = (sigma / (sigma + t)).sum()\n",
        "                m2 = (sigma * x2 / ((t + sigma) ** 2)).sum()\n",
        "                res2 = (x2 / ((1 + sigma / t) ** 2)).sum() + res_x2\n",
        "                alpha = gamma / (m2 + 1e-5)\n",
        "                beta = (N - gamma) / (res2 + 1e-5)\n",
        "                t_ = alpha / beta\n",
        "                evidence = D / 2.0 * np.log(alpha) \\\n",
        "                           + N / 2.0 * np.log(beta) \\\n",
        "                           - 0.5 * np.sum(np.log(alpha + beta * sigma)) \\\n",
        "                           - beta / 2.0 * res2 \\\n",
        "                           - alpha / 2.0 * m2 \\\n",
        "                           - N / 2.0 * np.log(2 * np.pi)\n",
        "                evidence /= N\n",
        "                if abs(t_ - t) / t <= 1e-3:  # abs(t_ - t) <= 1e-5 or abs(1 / t_ - 1 / t) <= 1e-5:\n",
        "                    break\n",
        "            evidence = D / 2.0 * np.log(alpha) \\\n",
        "                       + N / 2.0 * np.log(beta) \\\n",
        "                       - 0.5 * np.sum(np.log(alpha + beta * sigma)) \\\n",
        "                       - beta / 2.0 * res2 \\\n",
        "                       - alpha / 2.0 * m2 \\\n",
        "                       - N / 2.0 * np.log(2 * np.pi)\n",
        "            evidence /= N\n",
        "            m = 1.0 / (t + sigma) * s * x\n",
        "            m = (vh.T @ m).reshape(-1)\n",
        "            evidences.append(evidence)\n",
        "            self.alphas.append(alpha)\n",
        "            self.betas.append(beta)\n",
        "            self.ms.append(m)\n",
        "        self.ms = np.stack(self.ms)\n",
        "        return np.mean(evidences)\n",
        "\n",
        "    _fit = _fit_fixed_point\n",
        "\n",
        "    def fit(self, f: np.ndarray, y: np.ndarray):\n",
        "        \"\"\"\n",
        "        :param f: [N, F], feature matrix from pre-trained model\n",
        "        :param y: target labels.\n",
        "            For classification, y has shape [N] with element in [0, C_t).\n",
        "            For regression, y has shape [N, C] with C regression-labels\n",
        "        :return: LogME score (how well f can fit y directly)\n",
        "        \"\"\"\n",
        "        if self.fitted:\n",
        "            warnings.warn('re-fitting for new data. old parameters cleared.')\n",
        "            self.reset()\n",
        "        else:\n",
        "            self.fitted = True\n",
        "        f = f.astype(np.float64)\n",
        "        if self.regression:\n",
        "            y = y.astype(np.float64)\n",
        "            if len(y.shape) == 1:\n",
        "                y = y.reshape(-1, 1)\n",
        "        return self._fit(f, y)\n",
        "\n",
        "    def predict(self, f: np.ndarray):\n",
        "        \"\"\"\n",
        "        :param f: [N, F], feature matrix\n",
        "        :return: prediction, return shape [N, X]\n",
        "        \"\"\"\n",
        "        if not self.fitted:\n",
        "            raise RuntimeError(\"not fitted, please call fit first\")\n",
        "        f = f.astype(np.float64)\n",
        "        logits = f @ self.ms.T\n",
        "        if self.regression:\n",
        "            return logits\n",
        "        return np.argmax(logits, axis=-1)"
      ],
      "metadata": {
        "id": "VPo_Rr9UBlbB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Calculate the logME class directly"
      ],
      "metadata": {
        "id": "uYx_k4t2M1Vt"
      }
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "GWR7gQ0qNJnY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print('Conducting transferability calculation...')\n",
        "logme = LogME(regression=False)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RDe6CeejBR_h",
        "outputId": "5a3ef8cd-0220-4448-fdc4-c5a122333d68"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Conducting transferability calculation...\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import argparse, os\n",
        "import torch\n",
        "import torchvision.models as models\n",
        "import torchvision.transforms as transforms\n",
        "\n",
        "from torch.utils.data import DataLoader\n",
        "from torchvision import datasets\n",
        "from LogME import LogME\n",
        "import pprint\n",
        "\n",
        "models_hub = ['mobilenet_v2', 'mnasnet1_0', 'densenet121', 'densenet169', 'densenet201',\n",
        "               'resnet34', 'resnet50', 'resnet101', 'resnet152', 'googlenet', 'inception_v3']\n",
        "\n",
        "\n",
        "def get_configs():\n",
        "    parser = argparse.ArgumentParser(\n",
        "        description='Ranking pre-trained models')\n",
        "    parser.add_argument('--gpu', default=0, type=int,\n",
        "                        help='GPU num for training')\n",
        "    parser.add_argument('--batch_size', default=48, type=int)\n",
        "\n",
        "    # dataset\n",
        "    parser.add_argument('--dataset', default=\"aircraft\",\n",
        "                        type=str, help='Name of dataset')\n",
        "    parser.add_argument('--data_path', default=\"/data/FGVCAircraft/train\",\n",
        "                        type=str, help='Path of dataset')\n",
        "    parser.add_argument('--num_workers', default=2, type=int,\n",
        "                        help='Num of workers used in dataloading')\n",
        "    # model\n",
        "    configs = parser.parse_args()\n",
        "\n",
        "    return configs\n",
        "\n",
        "\n",
        "def forward_pass(score_loader, model, fc_layer):\n",
        "    \"\"\"\n",
        "    a forward pass on target dataset\n",
        "    :params score_loader: the dataloader for scoring transferability\n",
        "    :params model: the model for scoring transferability\n",
        "    :params fc_layer: the fc layer of the model, for registering hooks\n",
        "    returns\n",
        "        features: extracted features of model\n",
        "        outputs: outputs of model\n",
        "        targets: ground-truth labels of dataset\n",
        "    \"\"\"\n",
        "    features = []\n",
        "    outputs = []\n",
        "    targets = []\n",
        "    \n",
        "    def hook_fn_forward(module, input, output):\n",
        "        features.append(input[0].detach().cpu())\n",
        "        outputs.append(output.detach().cpu())\n",
        "    \n",
        "    forward_hook = fc_layer.register_forward_hook(hook_fn_forward)\n",
        "    print(\"before target data\")\n",
        "    model.eval()\n",
        "    with torch.no_grad():\n",
        "        for _, (data, target) in enumerate(score_loader):\n",
        "            targets.append(target)\n",
        "            data = data.cuda()\n",
        "            _ = model(data)\n",
        "    \n",
        "    forward_hook.remove()\n",
        "    features = torch.cat([x for x in features])\n",
        "    outputs = torch.cat([x for x in outputs])\n",
        "    targets = torch.cat([x for x in targets])\n",
        "    print(\"==================shape of the features\", features.shape)\n",
        "    return features, outputs, targets\n",
        "\n",
        "\n",
        "\n",
        "def score_model(score_loader, model_name, dataset_name):\n",
        "    print(f'Calc Transferabilities of {model_name} on {dataset_name}')\n",
        "\n",
        "    if model_name == 'inception_v3':\n",
        "            model = models.__dict__[model_name](pretrained=True, aux_logits=False).cuda()\n",
        "    else:\n",
        "        model = models.__dict__[model_name](pretrained=True).cuda()\n",
        "\n",
        "    # different models has different linear projection names\n",
        "    if model_name in ['mobilenet_v2', 'mnasnet1_0']:\n",
        "        fc_layer = model.classifier[-1]\n",
        "    elif model_name in ['densenet121', 'densenet169', 'densenet201']:\n",
        "        fc_layer = model.classifier\n",
        "    elif model_name in ['resnet34', 'resnet50', 'resnet101', 'resnet152', 'googlenet', 'inception_v3']:\n",
        "        fc_layer = model.fc\n",
        "    else:\n",
        "        # try your customized model\n",
        "        raise NotImplementedError\n",
        "\n",
        "    print('Conducting features extraction...')\n",
        "    features, outputs, targets = forward_pass(score_loader, model, fc_layer)\n",
        "    # predictions = F.softmax(outputs)\n",
        "\n",
        "    print('Conducting transferability calculation...')\n",
        "    logme = LogME(regression=False)\n",
        "    score = logme.fit(features.numpy(), targets.numpy())\n",
        "\n",
        "    \n",
        "\n",
        "    print(f'LogME of {model_name}: {score}\\n')\n",
        "    return score\n"
      ],
      "metadata": {
        "id": "ec0cD0duEH-j"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#config\n",
        "dataset = \"aircraft\"\n",
        "data_path =\"/data/FGVCAircraft/train\" \n",
        "num_workers = 2\n",
        "batch_size = 48"
      ],
      "metadata": {
        "id": "rsow66ivEgNG"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# training 時做 data augmentation\n",
        "train_transform = transforms.Compose([\n",
        "    transforms.RandomHorizontalFlip(), # 隨機將圖片水平翻轉\n",
        "    transforms.RandomRotation(15), # 隨機旋轉圖片\n",
        "    transforms.ToTensor(), # 將圖片轉成 Tensor，並把數值 normalize 到 [0,1] (data normalization)\n",
        "])\n",
        "# testing 時不需做 data augmentation\n",
        "test_transform = transforms.Compose([                          \n",
        "    transforms.ToTensor(),\n",
        "])\n",
        "batch_size = 32\n",
        "dataset = datasets.FGVCAircraft('../data',  download=True,\n",
        "                       transform=train_transform)\n",
        "test_set = datasets.FGVCAircraft('../data', \n",
        "                       transform=test_transform)"
      ],
      "metadata": {
        "id": "TmknSwnWFKXe"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "torch.cuda.set_device(0)\n",
        "\n",
        "normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],\n",
        "                                 std=[0.229, 0.224, 0.225])\n",
        "score_dict = {}\n",
        "for model in models_hub:\n",
        "        model = model\n",
        "        if model == 'inception_v3': # inception_v3 is pretrained on 299x299 images\n",
        "            transform=transforms.Compose([  \n",
        "                transforms.Resize((299, 299)),\n",
        "                transforms.ToTensor(),\n",
        "                normalize\n",
        "            ])\n",
        "        else:\n",
        "            transform=transforms.Compose([  # other models are pretrained on 224x224 images\n",
        "                transforms.Resize((224, 224)),\n",
        "                transforms.ToTensor(),\n",
        "                normalize\n",
        "            ])\n",
        "        score_dataset = dataset\n",
        "        # or try your customized dataset\n",
        "        score_loader = DataLoader(score_dataset, batch_size=batch_size, shuffle=False,\n",
        "            num_workers=num_workers, pin_memory=True)\n",
        "        score_dict[model] = score_model(score_loader, model, score_dataset)\n",
        "results = sorted(score_dict.items(), key=lambda i: i[1], reverse=True)\n",
        "print(f'Models ranking on {dataset}: ')\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "hY0hWJJVENZJ",
        "outputId": "dc397d2c-f487-462e-bd0f-98f139adf7cc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Calc Transferabilities of mobilenet_v2 on Dataset FGVCAircraft\n",
            "    Number of datapoints: 6667\n",
            "    Root location: ../data\n",
            "    StandardTransform\n",
            "Transform: Compose(\n",
            "               RandomHorizontalFlip(p=0.5)\n",
            "               RandomRotation(degrees=[-15.0, 15.0], interpolation=nearest, expand=False, fill=0)\n",
            "               ToTensor()\n",
            "           )\n",
            "Conducting features extraction...\n",
            "before target data\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.8/dist-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n",
            "  warnings.warn(\n",
            "/usr/local/lib/python3.8/dist-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=MobileNet_V2_Weights.IMAGENET1K_V1`. You can also use `weights=MobileNet_V2_Weights.DEFAULT` to get the most up-to-date weights.\n",
            "  warnings.warn(msg)\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "RuntimeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-53-9d82590cde3f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     22\u001b[0m         score_loader = DataLoader(score_dataset, batch_size=batch_size, shuffle=False,\n\u001b[1;32m     23\u001b[0m             num_workers=num_workers, pin_memory=True)\n\u001b[0;32m---> 24\u001b[0;31m         \u001b[0mscore_dict\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mscore_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mscore_loader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscore_dataset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     25\u001b[0m \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msorted\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mscore_dict\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreverse\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     26\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'Models ranking on {dataset}: '\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-50-2d0c4c7a735d>\u001b[0m in \u001b[0;36mscore_model\u001b[0;34m(score_loader, model_name, dataset_name)\u001b[0m\n\u001b[1;32m     90\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     91\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Conducting features extraction...'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 92\u001b[0;31m     \u001b[0mfeatures\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtargets\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mforward_pass\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mscore_loader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfc_layer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     93\u001b[0m     \u001b[0;31m# predictions = F.softmax(outputs)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-50-2d0c4c7a735d>\u001b[0m in \u001b[0;36mforward_pass\u001b[0;34m(score_loader, model, fc_layer)\u001b[0m\n\u001b[1;32m     56\u001b[0m     \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     57\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mno_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 58\u001b[0;31m         \u001b[0;32mfor\u001b[0m \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mscore_loader\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     59\u001b[0m             \u001b[0mtargets\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtarget\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     60\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    626\u001b[0m                 \u001b[0;31m# TODO(https://github.com/pytorch/pytorch/issues/76750)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    627\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# type: ignore[call-arg]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 628\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    629\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_yielded\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    630\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_kind\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0m_DatasetKind\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIterable\u001b[0m \u001b[0;32mand\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m_next_data\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1331\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1332\u001b[0m                 \u001b[0;32mdel\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_task_info\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1333\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_process_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1334\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1335\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_try_put_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m_process_data\u001b[0;34m(self, data)\u001b[0m\n\u001b[1;32m   1357\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_try_put_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1358\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mExceptionWrapper\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1359\u001b[0;31m             \u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreraise\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1360\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1361\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.8/dist-packages/torch/_utils.py\u001b[0m in \u001b[0;36mreraise\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    541\u001b[0m             \u001b[0;31m# instantiate since we don't know how to\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    542\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 543\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mexception\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    544\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    545\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mRuntimeError\u001b[0m: Caught RuntimeError in DataLoader worker process 0.\nOriginal Traceback (most recent call last):\n  File \"/usr/local/lib/python3.8/dist-packages/torch/utils/data/_utils/worker.py\", line 302, in _worker_loop\n    data = fetcher.fetch(index)\n  File \"/usr/local/lib/python3.8/dist-packages/torch/utils/data/_utils/fetch.py\", line 61, in fetch\n    return self.collate_fn(data)\n  File \"/usr/local/lib/python3.8/dist-packages/torch/utils/data/_utils/collate.py\", line 265, in default_collate\n    return collate(batch, collate_fn_map=default_collate_fn_map)\n  File \"/usr/local/lib/python3.8/dist-packages/torch/utils/data/_utils/collate.py\", line 143, in collate\n    return [collate(samples, collate_fn_map=collate_fn_map) for samples in transposed]  # Backwards compatibility.\n  File \"/usr/local/lib/python3.8/dist-packages/torch/utils/data/_utils/collate.py\", line 143, in <listcomp>\n    return [collate(samples, collate_fn_map=collate_fn_map) for samples in transposed]  # Backwards compatibility.\n  File \"/usr/local/lib/python3.8/dist-packages/torch/utils/data/_utils/collate.py\", line 120, in collate\n    return collate_fn_map[elem_type](batch, collate_fn_map=collate_fn_map)\n  File \"/usr/local/lib/python3.8/dist-packages/torch/utils/data/_utils/collate.py\", line 163, in collate_tensor_fn\n    return torch.stack(batch, 0, out=out)\nRuntimeError: stack expects each tensor to be equal size, but got [3, 695, 1024] at entry 0 and [3, 699, 1024] at entry 1\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "np.loadtxt('data_table.txt', skiprows=0 )"
      ],
      "metadata": {
        "id": "8NEm4hkoEcy7"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}