fine_tune_gpt2_to_HxH_dataset.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "fine-tune gpt2 to HxH dataset.ipynb",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JVX4ckf-F87e",
        "outputId": "1c46acec-997c-4694-e089-8062fa4915e4"
      },
      "source": [
        "!pip install git+https://github.com/huggingface/transformers\n",
        "!pip install datasets"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting git+https://github.com/huggingface/transformers\n",
            "  Cloning https://github.com/huggingface/transformers to /tmp/pip-req-build-rsjz0uk2\n",
            "  Running command git clone -q https://github.com/huggingface/transformers /tmp/pip-req-build-rsjz0uk2\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "    Preparing wheel metadata ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers==4.10.0.dev0) (4.6.4)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers==4.10.0.dev0) (21.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers==4.10.0.dev0) (2.23.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers==4.10.0.dev0) (3.0.12)\n",
            "Collecting pyyaml>=5.1\n",
            "  Downloading PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl (636 kB)\n",
            "\u001b[K     |████████████████████████████████| 636 kB 5.3 MB/s \n",
            "\u001b[?25hCollecting sacremoses\n",
            "  Downloading sacremoses-0.0.45-py3-none-any.whl (895 kB)\n",
            "\u001b[K     |████████████████████████████████| 895 kB 40.9 MB/s \n",
            "\u001b[?25hCollecting tokenizers<0.11,>=0.10.1\n",
            "  Downloading tokenizers-0.10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (3.3 MB)\n",
            "\u001b[K     |████████████████████████████████| 3.3 MB 36.9 MB/s \n",
            "\u001b[?25hCollecting huggingface-hub>=0.0.12\n",
            "  Downloading huggingface_hub-0.0.15-py3-none-any.whl (43 kB)\n",
            "\u001b[K     |████████████████████████████████| 43 kB 2.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers==4.10.0.dev0) (1.19.5)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers==4.10.0.dev0) (2019.12.20)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers==4.10.0.dev0) (4.62.0)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from huggingface-hub>=0.0.12->transformers==4.10.0.dev0) (3.7.4.3)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers==4.10.0.dev0) (2.4.7)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers==4.10.0.dev0) (3.5.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.10.0.dev0) (2021.5.30)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.10.0.dev0) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.10.0.dev0) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==4.10.0.dev0) (1.24.3)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==4.10.0.dev0) (1.0.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==4.10.0.dev0) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==4.10.0.dev0) (7.1.2)\n",
            "Building wheels for collected packages: transformers\n",
            "  Building wheel for transformers (PEP 517) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for transformers: filename=transformers-4.10.0.dev0-py3-none-any.whl size=2676935 sha256=fc7a03f528457cbb5ebcae37a93bf7078a79d5de8d37315d7c2b185a2accf53c\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-92sn8w57/wheels/35/2e/a7/d819e3310040329f0f47e57c9e3e7a7338aa5e74c49acfe522\n",
            "Successfully built transformers\n",
            "Installing collected packages: tokenizers, sacremoses, pyyaml, huggingface-hub, transformers\n",
            "  Attempting uninstall: pyyaml\n",
            "    Found existing installation: PyYAML 3.13\n",
            "    Uninstalling PyYAML-3.13:\n",
            "      Successfully uninstalled PyYAML-3.13\n",
            "Successfully installed huggingface-hub-0.0.15 pyyaml-5.4.1 sacremoses-0.0.45 tokenizers-0.10.3 transformers-4.10.0.dev0\n",
            "Collecting datasets\n",
            "  Downloading datasets-1.11.0-py3-none-any.whl (264 kB)\n",
            "\u001b[K     |████████████████████████████████| 264 kB 5.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from datasets) (1.19.5)\n",
            "Requirement already satisfied: tqdm>=4.42 in /usr/local/lib/python3.7/dist-packages (from datasets) (4.62.0)\n",
            "Requirement already satisfied: dill in /usr/local/lib/python3.7/dist-packages (from datasets) (0.3.4)\n",
            "Requirement already satisfied: pyarrow!=4.0.0,>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (3.0.0)\n",
            "Collecting xxhash\n",
            "  Downloading xxhash-2.0.2-cp37-cp37m-manylinux2010_x86_64.whl (243 kB)\n",
            "\u001b[K     |████████████████████████████████| 243 kB 46.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: multiprocess in /usr/local/lib/python3.7/dist-packages (from datasets) (0.70.12.2)\n",
            "Requirement already satisfied: huggingface-hub<0.1.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (0.0.15)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from datasets) (21.0)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from datasets) (4.6.4)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from datasets) (1.1.5)\n",
            "Collecting fsspec>=2021.05.0\n",
            "  Downloading fsspec-2021.7.0-py3-none-any.whl (118 kB)\n",
            "\u001b[K     |████████████████████████████████| 118 kB 54.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (2.23.0)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<0.1.0->datasets) (3.7.4.3)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<0.1.0->datasets) (3.0.12)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->datasets) (2.4.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (2021.5.30)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (3.0.4)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->datasets) (3.5.0)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2018.9)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas->datasets) (1.15.0)\n",
            "Installing collected packages: xxhash, fsspec, datasets\n",
            "Successfully installed datasets-1.11.0 fsspec-2021.7.0 xxhash-2.0.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MV6xHbwWFkO9",
        "outputId": "4416aaae-c6f1-40f7-b565-f6ffa76c202b"
      },
      "source": [
        "#test the un-tuned gpt2 with some keywords from HxH\n",
        "from transformers import pipeline, set_seed\n",
        "generator = pipeline('text-generation', model='gpt2')\n",
        "set_seed(0)\n",
        "\n",
        "generator(\"Fearsome monsters... Exotic creatures...\", max_length=30, num_return_sequences=5)\n"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using pad_token, but it is not set yet.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'generated_text': 'Fearsome monsters... Exotic creatures... The evil is the worst. You do what you can to protect your allies from any danger. A creature'},\n",
              " {'generated_text': 'Fearsome monsters... Exotic creatures...'},\n",
              " {'generated_text': 'Fearsome monsters... Exotic creatures... etc is the theme for so many RPGs. Some of the most well-known examples are:\\n'},\n",
              " {'generated_text': 'Fearsome monsters... Exotic creatures...\\n\\nToss in a new class who will only appear as a minor character, along with the new'},\n",
              " {'generated_text': \"Fearsome monsters... Exotic creatures...\\nPosted on 6/30/2006\\nPosted on 6/30/2006 What's the catch?\"}]"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6Cl4Db3ZbJ5A",
        "outputId": "49bc5964-8f25-48bb-9f74-a1a1d5644fb2"
      },
      "source": [
        "set_seed(0)\n",
        "generator(\"Killua\", max_length=30, num_return_sequences=5)"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using pad_token, but it is not set yet.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'generated_text': 'Killua, who is expected to be in Orlando Friday and Saturday, says she had three people in her home from her previous relationship with the man.'},\n",
              " {'generated_text': 'Killua-Sanjuro-Suri-Gumi/8384839.jpg/75″/79″\\n\\nGothic'},\n",
              " {'generated_text': 'Killua: Let the heat die so it can get out of the way and be as good as it gets.\\n\\nKiribayashi'},\n",
              " {'generated_text': 'Killua\\n\\nTurtle Riffage(2)\\n\\nMongoose Zombie(3)\\n\\nDuck Wolf(4)\\n'},\n",
              " {'generated_text': \"Killua's life was destroyed. The second act was a long one that involved the execution and the capture of a few more innocent people. The third\"}]"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-qDtY07ybL4H",
        "outputId": "5da583c3-89c4-4536-c39e-b6fede58f169"
      },
      "source": [
        "set_seed(0)\n",
        "generator(\"Gon\", max_length=30, num_return_sequences=5)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using pad_token, but it is not set yet.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'generated_text': \"Gonnie had no words to say.\\n\\nIt's a story that had been unfolding for almost an entire week on the beach in Punta\"},\n",
              " {'generated_text': \"Gonzo's second, third and fourth years, and there was a huge outpouring of support for his development. I'm curious to hear from\"},\n",
              " {'generated_text': 'Gonn is the fifth current American professional football star to play for the New York Jets. The former Florida State star, signed with the Jets on'},\n",
              " {'generated_text': 'Gonzo \"Titanium\" in Leland P. Putnam\\'s series of films. Among the films that made the list were The Jungle'},\n",
              " {'generated_text': 'Gonobra, one of the most well known and notorious mafia of New York (he\\'s the owner of \"Little Big Town\"). The \"'}]"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "w5GobWiGbNPo",
        "outputId": "e397f666-3c1e-4b7c-989d-1f3b56c5e9a0"
      },
      "source": [
        "set_seed(0)\n",
        "generator(\"Hunter\", max_length=30, num_return_sequences=5)"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using pad_token, but it is not set yet.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'generated_text': 'Hunter a second time.\\n\\n\\nAt the end of the first story arc, Tifa reveals her memories from her previous experience in the city of F'},\n",
              " {'generated_text': 'Hunter-E, Haze, Joona, Laiha, Yum, and Shokukan (2016)\\n\\n4.'},\n",
              " {'generated_text': \"Hunter:\\n\\nSo what I've written so far is a lot of work for the developers because I was able to come up with a lot of\"},\n",
              " {'generated_text': 'Hunter. \"If you want a good job doing that, you have to quit now.\"\\n\\nThe former president, who met with some of the'},\n",
              " {'generated_text': 'Hunter of the Storms\\n\\n\\nLevel 10 Boots\\n\\n\\nThe Heavyweight Assault Badge\\n\\n\\nLevel 10 Badge\\n\\n\\nThe Classic Atomic Badge\\n\\n\\n'}]"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QfDVpOFrbOy3",
        "outputId": "8eeda207-c26d-4ce7-bb76-820955be39bc"
      },
      "source": [
        "set_seed(0)\n",
        "generator(\"Ging\", max_length=30, num_return_sequences=5)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using pad_token, but it is not set yet.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'generated_text': \"Ging Wei and Jie Yang of the Shanghai Metropolitan University, who had studied Xiao Wu's theory of the afterlife, said in their paper.\\n\"},\n",
              " {'generated_text': 'Gingfang, Hui Jian. 2006a. The Nature and Health Characteristics of the Female Infant Breast Contraceptives Contraception'},\n",
              " {'generated_text': 'Ging: Let the baby talk so it can get its own oxygen supply and keep doing the normal work. Just make the baby sit on one knee'},\n",
              " {'generated_text': 'Ging Tian, who went by a nickname Lanyong, and Zhao Qingwei, who called him Jie Chen, were both at their most'},\n",
              " {'generated_text': 'Gingong Feng, Jian Lin, Yi Cheng, Yihsai Yan, Jing Zhong and others. \"With such an effective mechanism of'}]"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cHuNTrhWbP0o",
        "outputId": "3e927508-a408-4162-b6e7-ac285e10cd70"
      },
      "source": [
        "set_seed(0)\n",
        "generator(\"Kite\", max_length=30, num_return_sequences=5)"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using pad_token, but it is not set yet.\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'generated_text': \"Kite a bit longer, now that it's all over.\\n\\nGarrett: OK, so, you've been in the game.\\n\"},\n",
              " {'generated_text': 'Kite-like, for example, which he made at the start of his freshman year. Even before he took part in both the regular and postseason'},\n",
              " {'generated_text': \"Kite:\\n\\nSo what I've written so far is a lot of work for the developers because I was able to reach a few million dollars\"},\n",
              " {'generated_text': 'Kite. \"If you want a good job doing anything with respect to food you come to know all the other cooks in the restaurant. They give'},\n",
              " {'generated_text': 'Kite 4A 6X4X8 10\\n\\nNvidia 14xx 0.2 1 2 2 3 4 5 6 7 8 9\\n'}]"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bIyj3Bo87bfR"
      },
      "source": [
        "fine tuning GPT-2 model:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YuYd-b7FZEX5",
        "outputId": "d05b5539-c1c3-47f6-dea2-7ceec7dc8749"
      },
      "source": [
        "!python run_clm.py \\\n",
        "--help"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "usage: run_clm.py [-h] [--model_name_or_path MODEL_NAME_OR_PATH]\n",
            "                  [--model_type MODEL_TYPE]\n",
            "                  [--config_overrides CONFIG_OVERRIDES]\n",
            "                  [--config_name CONFIG_NAME]\n",
            "                  [--tokenizer_name TOKENIZER_NAME] [--cache_dir CACHE_DIR]\n",
            "                  [--no_use_fast_tokenizer]\n",
            "                  [--use_fast_tokenizer [USE_FAST_TOKENIZER]]\n",
            "                  [--model_revision MODEL_REVISION]\n",
            "                  [--use_auth_token [USE_AUTH_TOKEN]]\n",
            "                  [--dataset_name DATASET_NAME]\n",
            "                  [--dataset_config_name DATASET_CONFIG_NAME]\n",
            "                  [--train_file TRAIN_FILE]\n",
            "                  [--validation_file VALIDATION_FILE]\n",
            "                  [--max_train_samples MAX_TRAIN_SAMPLES]\n",
            "                  [--max_eval_samples MAX_EVAL_SAMPLES]\n",
            "                  [--block_size BLOCK_SIZE]\n",
            "                  [--overwrite_cache [OVERWRITE_CACHE]]\n",
            "                  [--validation_split_percentage VALIDATION_SPLIT_PERCENTAGE]\n",
            "                  [--preprocessing_num_workers PREPROCESSING_NUM_WORKERS]\n",
            "                  --output_dir OUTPUT_DIR\n",
            "                  [--overwrite_output_dir [OVERWRITE_OUTPUT_DIR]]\n",
            "                  [--do_train [DO_TRAIN]] [--do_eval [DO_EVAL]]\n",
            "                  [--do_predict [DO_PREDICT]]\n",
            "                  [--evaluation_strategy {no,steps,epoch}]\n",
            "                  [--prediction_loss_only [PREDICTION_LOSS_ONLY]]\n",
            "                  [--per_device_train_batch_size PER_DEVICE_TRAIN_BATCH_SIZE]\n",
            "                  [--per_device_eval_batch_size PER_DEVICE_EVAL_BATCH_SIZE]\n",
            "                  [--per_gpu_train_batch_size PER_GPU_TRAIN_BATCH_SIZE]\n",
            "                  [--per_gpu_eval_batch_size PER_GPU_EVAL_BATCH_SIZE]\n",
            "                  [--gradient_accumulation_steps GRADIENT_ACCUMULATION_STEPS]\n",
            "                  [--eval_accumulation_steps EVAL_ACCUMULATION_STEPS]\n",
            "                  [--learning_rate LEARNING_RATE]\n",
            "                  [--weight_decay WEIGHT_DECAY] [--adam_beta1 ADAM_BETA1]\n",
            "                  [--adam_beta2 ADAM_BETA2] [--adam_epsilon ADAM_EPSILON]\n",
            "                  [--max_grad_norm MAX_GRAD_NORM]\n",
            "                  [--num_train_epochs NUM_TRAIN_EPOCHS]\n",
            "                  [--max_steps MAX_STEPS]\n",
            "                  [--lr_scheduler_type {linear,cosine,cosine_with_restarts,polynomial,constant,constant_with_warmup}]\n",
            "                  [--warmup_ratio WARMUP_RATIO] [--warmup_steps WARMUP_STEPS]\n",
            "                  [--log_level {debug,info,warning,error,critical,passive}]\n",
            "                  [--log_level_replica {debug,info,warning,error,critical,passive}]\n",
            "                  [--no_log_on_each_node]\n",
            "                  [--log_on_each_node [LOG_ON_EACH_NODE]]\n",
            "                  [--logging_dir LOGGING_DIR]\n",
            "                  [--logging_strategy {no,steps,epoch}]\n",
            "                  [--logging_first_step [LOGGING_FIRST_STEP]]\n",
            "                  [--logging_steps LOGGING_STEPS]\n",
            "                  [--save_strategy {no,steps,epoch}] [--save_steps SAVE_STEPS]\n",
            "                  [--save_total_limit SAVE_TOTAL_LIMIT]\n",
            "                  [--save_on_each_node [SAVE_ON_EACH_NODE]]\n",
            "                  [--no_cuda [NO_CUDA]] [--seed SEED] [--fp16 [FP16]]\n",
            "                  [--fp16_opt_level FP16_OPT_LEVEL]\n",
            "                  [--fp16_backend {auto,amp,apex}]\n",
            "                  [--fp16_full_eval [FP16_FULL_EVAL]]\n",
            "                  [--local_rank LOCAL_RANK] [--tpu_num_cores TPU_NUM_CORES]\n",
            "                  [--tpu_metrics_debug [TPU_METRICS_DEBUG]] [--debug DEBUG]\n",
            "                  [--dataloader_drop_last [DATALOADER_DROP_LAST]]\n",
            "                  [--eval_steps EVAL_STEPS]\n",
            "                  [--dataloader_num_workers DATALOADER_NUM_WORKERS]\n",
            "                  [--past_index PAST_INDEX] [--run_name RUN_NAME]\n",
            "                  [--disable_tqdm DISABLE_TQDM] [--no_remove_unused_columns]\n",
            "                  [--remove_unused_columns [REMOVE_UNUSED_COLUMNS]]\n",
            "                  [--label_names LABEL_NAMES [LABEL_NAMES ...]]\n",
            "                  [--load_best_model_at_end [LOAD_BEST_MODEL_AT_END]]\n",
            "                  [--metric_for_best_model METRIC_FOR_BEST_MODEL]\n",
            "                  [--greater_is_better GREATER_IS_BETTER]\n",
            "                  [--ignore_data_skip [IGNORE_DATA_SKIP]]\n",
            "                  [--sharded_ddp SHARDED_DDP] [--deepspeed DEEPSPEED]\n",
            "                  [--label_smoothing_factor LABEL_SMOOTHING_FACTOR]\n",
            "                  [--adafactor [ADAFACTOR]]\n",
            "                  [--group_by_length [GROUP_BY_LENGTH]]\n",
            "                  [--length_column_name LENGTH_COLUMN_NAME]\n",
            "                  [--report_to REPORT_TO [REPORT_TO ...]]\n",
            "                  [--ddp_find_unused_parameters DDP_FIND_UNUSED_PARAMETERS]\n",
            "                  [--no_dataloader_pin_memory]\n",
            "                  [--dataloader_pin_memory [DATALOADER_PIN_MEMORY]]\n",
            "                  [--no_skip_memory_metrics]\n",
            "                  [--skip_memory_metrics [SKIP_MEMORY_METRICS]]\n",
            "                  [--use_legacy_prediction_loop [USE_LEGACY_PREDICTION_LOOP]]\n",
            "                  [--push_to_hub [PUSH_TO_HUB]]\n",
            "                  [--resume_from_checkpoint RESUME_FROM_CHECKPOINT]\n",
            "                  [--push_to_hub_model_id PUSH_TO_HUB_MODEL_ID]\n",
            "                  [--push_to_hub_organization PUSH_TO_HUB_ORGANIZATION]\n",
            "                  [--push_to_hub_token PUSH_TO_HUB_TOKEN]\n",
            "                  [--mp_parameters MP_PARAMETERS]\n",
            "\n",
            "optional arguments:\n",
            "  -h, --help            show this help message and exit\n",
            "  --model_name_or_path MODEL_NAME_OR_PATH\n",
            "                        The model checkpoint for weights initialization.Don't\n",
            "                        set if you want to train a model from scratch.\n",
            "                        (default: None)\n",
            "  --model_type MODEL_TYPE\n",
            "                        If training from scratch, pass a model type from the\n",
            "                        list: rembert, roformer, bigbird_pegasus, gpt_neo,\n",
            "                        big_bird, blenderbot-small, bert-generation,\n",
            "                        camembert, xlm-roberta, pegasus, marian, mbart,\n",
            "                        megatron-bert, bart, blenderbot, reformer, roberta,\n",
            "                        bert, openai-gpt, gpt2, transfo-xl, xlnet, xlm-\n",
            "                        prophetnet, prophetnet, xlm, ctrl (default: None)\n",
            "  --config_overrides CONFIG_OVERRIDES\n",
            "                        Override some existing default config settings when a\n",
            "                        model is trained from scratch. Example: n_embd=10,resi\n",
            "                        d_pdrop=0.2,scale_attn_weights=false,summary_type=cls_\n",
            "                        index (default: None)\n",
            "  --config_name CONFIG_NAME\n",
            "                        Pretrained config name or path if not the same as\n",
            "                        model_name (default: None)\n",
            "  --tokenizer_name TOKENIZER_NAME\n",
            "                        Pretrained tokenizer name or path if not the same as\n",
            "                        model_name (default: None)\n",
            "  --cache_dir CACHE_DIR\n",
            "                        Where do you want to store the pretrained models\n",
            "                        downloaded from huggingface.co (default: None)\n",
            "  --no_use_fast_tokenizer\n",
            "                        Whether to use one of the fast tokenizer (backed by\n",
            "                        the tokenizers library) or not. (default: True)\n",
            "  --use_fast_tokenizer [USE_FAST_TOKENIZER]\n",
            "                        Whether to use one of the fast tokenizer (backed by\n",
            "                        the tokenizers library) or not. (default: True)\n",
            "  --model_revision MODEL_REVISION\n",
            "                        The specific model version to use (can be a branch\n",
            "                        name, tag name or commit id). (default: main)\n",
            "  --use_auth_token [USE_AUTH_TOKEN]\n",
            "                        Will use the token generated when running\n",
            "                        `transformers-cli login` (necessary to use this script\n",
            "                        with private models). (default: False)\n",
            "  --dataset_name DATASET_NAME\n",
            "                        The name of the dataset to use (via the datasets\n",
            "                        library). (default: None)\n",
            "  --dataset_config_name DATASET_CONFIG_NAME\n",
            "                        The configuration name of the dataset to use (via the\n",
            "                        datasets library). (default: None)\n",
            "  --train_file TRAIN_FILE\n",
            "                        The input training data file (a text file). (default:\n",
            "                        None)\n",
            "  --validation_file VALIDATION_FILE\n",
            "                        An optional input evaluation data file to evaluate the\n",
            "                        perplexity on (a text file). (default: None)\n",
            "  --max_train_samples MAX_TRAIN_SAMPLES\n",
            "                        For debugging purposes or quicker training, truncate\n",
            "                        the number of training examples to this value if set.\n",
            "                        (default: None)\n",
            "  --max_eval_samples MAX_EVAL_SAMPLES\n",
            "                        For debugging purposes or quicker training, truncate\n",
            "                        the number of evaluation examples to this value if\n",
            "                        set. (default: None)\n",
            "  --block_size BLOCK_SIZE\n",
            "                        Optional input sequence length after tokenization. The\n",
            "                        training dataset will be truncated in block of this\n",
            "                        size for training. Default to the model max input\n",
            "                        length for single sentence inputs (take into account\n",
            "                        special tokens). (default: None)\n",
            "  --overwrite_cache [OVERWRITE_CACHE]\n",
            "                        Overwrite the cached training and evaluation sets\n",
            "                        (default: False)\n",
            "  --validation_split_percentage VALIDATION_SPLIT_PERCENTAGE\n",
            "                        The percentage of the train set used as validation set\n",
            "                        in case there's no validation split (default: 5)\n",
            "  --preprocessing_num_workers PREPROCESSING_NUM_WORKERS\n",
            "                        The number of processes to use for the preprocessing.\n",
            "                        (default: None)\n",
            "  --output_dir OUTPUT_DIR\n",
            "                        The output directory where the model predictions and\n",
            "                        checkpoints will be written. (default: None)\n",
            "  --overwrite_output_dir [OVERWRITE_OUTPUT_DIR]\n",
            "                        Overwrite the content of the output directory.Use this\n",
            "                        to continue training if output_dir points to a\n",
            "                        checkpoint directory. (default: False)\n",
            "  --do_train [DO_TRAIN]\n",
            "                        Whether to run training. (default: False)\n",
            "  --do_eval [DO_EVAL]   Whether to run eval on the dev set. (default: False)\n",
            "  --do_predict [DO_PREDICT]\n",
            "                        Whether to run predictions on the test set. (default:\n",
            "                        False)\n",
            "  --evaluation_strategy {no,steps,epoch}\n",
            "                        The evaluation strategy to use. (default: no)\n",
            "  --prediction_loss_only [PREDICTION_LOSS_ONLY]\n",
            "                        When performing evaluation and predictions, only\n",
            "                        returns the loss. (default: False)\n",
            "  --per_device_train_batch_size PER_DEVICE_TRAIN_BATCH_SIZE\n",
            "                        Batch size per GPU/TPU core/CPU for training.\n",
            "                        (default: 8)\n",
            "  --per_device_eval_batch_size PER_DEVICE_EVAL_BATCH_SIZE\n",
            "                        Batch size per GPU/TPU core/CPU for evaluation.\n",
            "                        (default: 8)\n",
            "  --per_gpu_train_batch_size PER_GPU_TRAIN_BATCH_SIZE\n",
            "                        Deprecated, the use of `--per_device_train_batch_size`\n",
            "                        is preferred. Batch size per GPU/TPU core/CPU for\n",
            "                        training. (default: None)\n",
            "  --per_gpu_eval_batch_size PER_GPU_EVAL_BATCH_SIZE\n",
            "                        Deprecated, the use of `--per_device_eval_batch_size`\n",
            "                        is preferred.Batch size per GPU/TPU core/CPU for\n",
            "                        evaluation. (default: None)\n",
            "  --gradient_accumulation_steps GRADIENT_ACCUMULATION_STEPS\n",
            "                        Number of updates steps to accumulate before\n",
            "                        performing a backward/update pass. (default: 1)\n",
            "  --eval_accumulation_steps EVAL_ACCUMULATION_STEPS\n",
            "                        Number of predictions steps to accumulate before\n",
            "                        moving the tensors to the CPU. (default: None)\n",
            "  --learning_rate LEARNING_RATE\n",
            "                        The initial learning rate for AdamW. (default: 5e-05)\n",
            "  --weight_decay WEIGHT_DECAY\n",
            "                        Weight decay for AdamW if we apply some. (default:\n",
            "                        0.0)\n",
            "  --adam_beta1 ADAM_BETA1\n",
            "                        Beta1 for AdamW optimizer (default: 0.9)\n",
            "  --adam_beta2 ADAM_BETA2\n",
            "                        Beta2 for AdamW optimizer (default: 0.999)\n",
            "  --adam_epsilon ADAM_EPSILON\n",
            "                        Epsilon for AdamW optimizer. (default: 1e-08)\n",
            "  --max_grad_norm MAX_GRAD_NORM\n",
            "                        Max gradient norm. (default: 1.0)\n",
            "  --num_train_epochs NUM_TRAIN_EPOCHS\n",
            "                        Total number of training epochs to perform. (default:\n",
            "                        3.0)\n",
            "  --max_steps MAX_STEPS\n",
            "                        If > 0: set total number of training steps to perform.\n",
            "                        Override num_train_epochs. (default: -1)\n",
            "  --lr_scheduler_type {linear,cosine,cosine_with_restarts,polynomial,constant,constant_with_warmup}\n",
            "                        The scheduler type to use. (default: linear)\n",
            "  --warmup_ratio WARMUP_RATIO\n",
            "                        Linear warmup over warmup_ratio fraction of total\n",
            "                        steps. (default: 0.0)\n",
            "  --warmup_steps WARMUP_STEPS\n",
            "                        Linear warmup over warmup_steps. (default: 0)\n",
            "  --log_level {debug,info,warning,error,critical,passive}\n",
            "                        Logger log level to use on the main node. Possible\n",
            "                        choices are the log levels as strings: 'debug',\n",
            "                        'info', 'warning', 'error' and 'critical', plus a\n",
            "                        'passive' level which doesn't set anything and lets\n",
            "                        the application set the level. Defaults to 'passive'.\n",
            "                        (default: passive)\n",
            "  --log_level_replica {debug,info,warning,error,critical,passive}\n",
            "                        Logger log level to use on replica nodes. Same choices\n",
            "                        and defaults as ``log_level`` (default: passive)\n",
            "  --no_log_on_each_node\n",
            "                        When doing a multinode distributed training, whether\n",
            "                        to log once per node or just once on the main node.\n",
            "                        (default: True)\n",
            "  --log_on_each_node [LOG_ON_EACH_NODE]\n",
            "                        When doing a multinode distributed training, whether\n",
            "                        to log once per node or just once on the main node.\n",
            "                        (default: True)\n",
            "  --logging_dir LOGGING_DIR\n",
            "                        Tensorboard log dir. (default: None)\n",
            "  --logging_strategy {no,steps,epoch}\n",
            "                        The logging strategy to use. (default: steps)\n",
            "  --logging_first_step [LOGGING_FIRST_STEP]\n",
            "                        Log the first global_step (default: False)\n",
            "  --logging_steps LOGGING_STEPS\n",
            "                        Log every X updates steps. (default: 500)\n",
            "  --save_strategy {no,steps,epoch}\n",
            "                        The checkpoint save strategy to use. (default: steps)\n",
            "  --save_steps SAVE_STEPS\n",
            "                        Save checkpoint every X updates steps. (default: 500)\n",
            "  --save_total_limit SAVE_TOTAL_LIMIT\n",
            "                        Limit the total amount of checkpoints.Deletes the\n",
            "                        older checkpoints in the output_dir. Default is\n",
            "                        unlimited checkpoints (default: None)\n",
            "  --save_on_each_node [SAVE_ON_EACH_NODE]\n",
            "                        When doing multi-node distributed training, whether to\n",
            "                        save models and checkpoints on each node, or only on\n",
            "                        the main one (default: False)\n",
            "  --no_cuda [NO_CUDA]   Do not use CUDA even when it is available (default:\n",
            "                        False)\n",
            "  --seed SEED           Random seed that will be set at the beginning of\n",
            "                        training. (default: 42)\n",
            "  --fp16 [FP16]         Whether to use 16-bit (mixed) precision instead of\n",
            "                        32-bit (default: False)\n",
            "  --fp16_opt_level FP16_OPT_LEVEL\n",
            "                        For fp16: Apex AMP optimization level selected in\n",
            "                        ['O0', 'O1', 'O2', and 'O3'].See details at\n",
            "                        https://nvidia.github.io/apex/amp.html (default: O1)\n",
            "  --fp16_backend {auto,amp,apex}\n",
            "                        The backend to be used for mixed precision. (default:\n",
            "                        auto)\n",
            "  --fp16_full_eval [FP16_FULL_EVAL]\n",
            "                        Whether to use full 16-bit precision evaluation\n",
            "                        instead of 32-bit (default: False)\n",
            "  --local_rank LOCAL_RANK\n",
            "                        For distributed training: local_rank (default: -1)\n",
            "  --tpu_num_cores TPU_NUM_CORES\n",
            "                        TPU: Number of TPU cores (automatically passed by\n",
            "                        launcher script) (default: None)\n",
            "  --tpu_metrics_debug [TPU_METRICS_DEBUG]\n",
            "                        Deprecated, the use of `--debug tpu_metrics_debug` is\n",
            "                        preferred. TPU: Whether to print debug metrics\n",
            "                        (default: False)\n",
            "  --debug DEBUG         Whether or not to enable debug mode. Current options:\n",
            "                        `underflow_overflow` (Detect underflow and overflow in\n",
            "                        activations and weights), `tpu_metrics_debug` (print\n",
            "                        debug metrics on TPU). (default: )\n",
            "  --dataloader_drop_last [DATALOADER_DROP_LAST]\n",
            "                        Drop the last incomplete batch if it is not divisible\n",
            "                        by the batch size. (default: False)\n",
            "  --eval_steps EVAL_STEPS\n",
            "                        Run an evaluation every X steps. (default: None)\n",
            "  --dataloader_num_workers DATALOADER_NUM_WORKERS\n",
            "                        Number of subprocesses to use for data loading\n",
            "                        (PyTorch only). 0 means that the data will be loaded\n",
            "                        in the main process. (default: 0)\n",
            "  --past_index PAST_INDEX\n",
            "                        If >=0, uses the corresponding part of the output as\n",
            "                        the past state for next step. (default: -1)\n",
            "  --run_name RUN_NAME   An optional descriptor for the run. Notably used for\n",
            "                        wandb logging. (default: None)\n",
            "  --disable_tqdm DISABLE_TQDM\n",
            "                        Whether or not to disable the tqdm progress bars.\n",
            "                        (default: None)\n",
            "  --no_remove_unused_columns\n",
            "                        Remove columns not required by the model when using an\n",
            "                        nlp.Dataset. (default: True)\n",
            "  --remove_unused_columns [REMOVE_UNUSED_COLUMNS]\n",
            "                        Remove columns not required by the model when using an\n",
            "                        nlp.Dataset. (default: True)\n",
            "  --label_names LABEL_NAMES [LABEL_NAMES ...]\n",
            "                        The list of keys in your dictionary of inputs that\n",
            "                        correspond to the labels. (default: None)\n",
            "  --load_best_model_at_end [LOAD_BEST_MODEL_AT_END]\n",
            "                        Whether or not to load the best model found during\n",
            "                        training at the end of training. (default: False)\n",
            "  --metric_for_best_model METRIC_FOR_BEST_MODEL\n",
            "                        The metric to use to compare two different models.\n",
            "                        (default: None)\n",
            "  --greater_is_better GREATER_IS_BETTER\n",
            "                        Whether the `metric_for_best_model` should be\n",
            "                        maximized or not. (default: None)\n",
            "  --ignore_data_skip [IGNORE_DATA_SKIP]\n",
            "                        When resuming training, whether or not to skip the\n",
            "                        first epochs and batches to get to the same training\n",
            "                        data. (default: False)\n",
            "  --sharded_ddp SHARDED_DDP\n",
            "                        Whether or not to use sharded DDP training (in\n",
            "                        distributed training only). The base option should be\n",
            "                        `simple`, `zero_dp_2` or `zero_dp_3` and you can add\n",
            "                        CPU-offload to `zero_dp_2` or `zero_dp_3` like this:\n",
            "                        zero_dp_2 offload` or `zero_dp_3 offload`. You can add\n",
            "                        auto-wrap to `zero_dp_2` or with the same syntax:\n",
            "                        zero_dp_2 auto_wrap` or `zero_dp_3 auto_wrap`.\n",
            "                        (default: )\n",
            "  --deepspeed DEEPSPEED\n",
            "                        Enable deepspeed and pass the path to deepspeed json\n",
            "                        config file (e.g. ds_config.json) or an already loaded\n",
            "                        json file as a dict (default: None)\n",
            "  --label_smoothing_factor LABEL_SMOOTHING_FACTOR\n",
            "                        The label smoothing epsilon to apply (zero means no\n",
            "                        label smoothing). (default: 0.0)\n",
            "  --adafactor [ADAFACTOR]\n",
            "                        Whether or not to replace AdamW by Adafactor.\n",
            "                        (default: False)\n",
            "  --group_by_length [GROUP_BY_LENGTH]\n",
            "                        Whether or not to group samples of roughly the same\n",
            "                        length together when batching. (default: False)\n",
            "  --length_column_name LENGTH_COLUMN_NAME\n",
            "                        Column name with precomputed lengths to use when\n",
            "                        grouping by length. (default: length)\n",
            "  --report_to REPORT_TO [REPORT_TO ...]\n",
            "                        The list of integrations to report the results and\n",
            "                        logs to. (default: None)\n",
            "  --ddp_find_unused_parameters DDP_FIND_UNUSED_PARAMETERS\n",
            "                        When using distributed training, the value of the flag\n",
            "                        `find_unused_parameters` passed to\n",
            "                        `DistributedDataParallel`. (default: None)\n",
            "  --no_dataloader_pin_memory\n",
            "                        Whether or not to pin memory for DataLoader. (default:\n",
            "                        True)\n",
            "  --dataloader_pin_memory [DATALOADER_PIN_MEMORY]\n",
            "                        Whether or not to pin memory for DataLoader. (default:\n",
            "                        True)\n",
            "  --no_skip_memory_metrics\n",
            "                        Whether or not to skip adding of memory profiler\n",
            "                        reports to metrics. (default: True)\n",
            "  --skip_memory_metrics [SKIP_MEMORY_METRICS]\n",
            "                        Whether or not to skip adding of memory profiler\n",
            "                        reports to metrics. (default: True)\n",
            "  --use_legacy_prediction_loop [USE_LEGACY_PREDICTION_LOOP]\n",
            "                        Whether or not to use the legacy prediction_loop in\n",
            "                        the Trainer. (default: False)\n",
            "  --push_to_hub [PUSH_TO_HUB]\n",
            "                        Whether or not to upload the trained model to the\n",
            "                        model hub after training. (default: False)\n",
            "  --resume_from_checkpoint RESUME_FROM_CHECKPOINT\n",
            "                        The path to a folder with a valid checkpoint for your\n",
            "                        model. (default: None)\n",
            "  --push_to_hub_model_id PUSH_TO_HUB_MODEL_ID\n",
            "                        The name of the repository to which push the\n",
            "                        `Trainer`. (default: None)\n",
            "  --push_to_hub_organization PUSH_TO_HUB_ORGANIZATION\n",
            "                        The name of the organization in with to which push the\n",
            "                        `Trainer`. (default: None)\n",
            "  --push_to_hub_token PUSH_TO_HUB_TOKEN\n",
            "                        The token to use to push to the Model Hub. (default:\n",
            "                        None)\n",
            "  --mp_parameters MP_PARAMETERS\n",
            "                        Used by the SageMaker launcher to send mp-specific\n",
            "                        args. Ignored in Trainer (default: )\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QVL9sc49kmWQ",
        "outputId": "f882adb0-7f41-44a3-e1f4-80487646dcd5"
      },
      "source": [
        "!python run_clm.py \\\n",
        "--model_name_or_path gpt2 \\\n",
        "--train_file /content/dataset.txt \\\n",
        "--do_train \\\n",
        "--per_device_train_batch_size=50 \\\n",
        "--output_dir test-clm \\\n",
        "--overwrite_output_dir \\\n",
        "--save_total_limit 2 \\\n",
        "--num_train_epochs=5 \\\n",
        "--block_size=20 \\\n",
        "--lr_scheduler_type=cosine"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "08/25/2021 09:10:49 - WARNING - __main__ - Process rank: -1, device: cuda:0, n_gpu: 1distributed training: False, 16-bits training: False\n",
            "08/25/2021 09:10:49 - INFO - __main__ - Training/evaluation parameters TrainingArguments(\n",
            "_n_gpu=1,\n",
            "adafactor=False,\n",
            "adam_beta1=0.9,\n",
            "adam_beta2=0.999,\n",
            "adam_epsilon=1e-08,\n",
            "dataloader_drop_last=False,\n",
            "dataloader_num_workers=0,\n",
            "dataloader_pin_memory=True,\n",
            "ddp_find_unused_parameters=None,\n",
            "debug=[],\n",
            "deepspeed=None,\n",
            "disable_tqdm=False,\n",
            "do_eval=False,\n",
            "do_predict=False,\n",
            "do_train=True,\n",
            "eval_accumulation_steps=None,\n",
            "eval_steps=None,\n",
            "evaluation_strategy=IntervalStrategy.NO,\n",
            "fp16=False,\n",
            "fp16_backend=auto,\n",
            "fp16_full_eval=False,\n",
            "fp16_opt_level=O1,\n",
            "gradient_accumulation_steps=1,\n",
            "greater_is_better=None,\n",
            "group_by_length=False,\n",
            "ignore_data_skip=False,\n",
            "label_names=None,\n",
            "label_smoothing_factor=0.0,\n",
            "learning_rate=5e-05,\n",
            "length_column_name=length,\n",
            "load_best_model_at_end=False,\n",
            "local_rank=-1,\n",
            "log_level=-1,\n",
            "log_level_replica=-1,\n",
            "log_on_each_node=True,\n",
            "logging_dir=test-clm/runs/Aug25_09-10-49_75fd4fc88355,\n",
            "logging_first_step=False,\n",
            "logging_steps=500,\n",
            "logging_strategy=IntervalStrategy.STEPS,\n",
            "lr_scheduler_type=SchedulerType.COSINE,\n",
            "max_grad_norm=1.0,\n",
            "max_steps=-1,\n",
            "metric_for_best_model=None,\n",
            "mp_parameters=,\n",
            "no_cuda=False,\n",
            "num_train_epochs=5.0,\n",
            "output_dir=test-clm,\n",
            "overwrite_output_dir=True,\n",
            "past_index=-1,\n",
            "per_device_eval_batch_size=8,\n",
            "per_device_train_batch_size=50,\n",
            "prediction_loss_only=False,\n",
            "push_to_hub=False,\n",
            "push_to_hub_model_id=test-clm,\n",
            "push_to_hub_organization=None,\n",
            "push_to_hub_token=None,\n",
            "remove_unused_columns=True,\n",
            "report_to=['tensorboard'],\n",
            "resume_from_checkpoint=None,\n",
            "run_name=test-clm,\n",
            "save_on_each_node=False,\n",
            "save_steps=500,\n",
            "save_strategy=IntervalStrategy.STEPS,\n",
            "save_total_limit=2,\n",
            "seed=42,\n",
            "sharded_ddp=[],\n",
            "skip_memory_metrics=True,\n",
            "tpu_metrics_debug=False,\n",
            "tpu_num_cores=None,\n",
            "use_legacy_prediction_loop=False,\n",
            "warmup_ratio=0.0,\n",
            "warmup_steps=0,\n",
            "weight_decay=0.0,\n",
            ")\n",
            "08/25/2021 09:10:49 - WARNING - datasets.builder - Using custom data configuration default-3165dcbb69aeec5c\n",
            "08/25/2021 09:10:49 - INFO - datasets.builder - Overwrite dataset info from restored data version.\n",
            "08/25/2021 09:10:49 - INFO - datasets.info - Loading Dataset info from /root/.cache/huggingface/datasets/text/default-3165dcbb69aeec5c/0.0.0/e16f44aa1b321ece1f87b07977cc5d70be93d69b20486d6dacd62e12cf25c9a5\n",
            "08/25/2021 09:10:49 - WARNING - datasets.builder - Reusing dataset text (/root/.cache/huggingface/datasets/text/default-3165dcbb69aeec5c/0.0.0/e16f44aa1b321ece1f87b07977cc5d70be93d69b20486d6dacd62e12cf25c9a5)\n",
            "08/25/2021 09:10:49 - INFO - datasets.info - Loading Dataset info from /root/.cache/huggingface/datasets/text/default-3165dcbb69aeec5c/0.0.0/e16f44aa1b321ece1f87b07977cc5d70be93d69b20486d6dacd62e12cf25c9a5\n",
            "100% 1/1 [00:00<00:00, 576.70it/s]\n",
            "08/25/2021 09:10:49 - WARNING - datasets.builder - Using custom data configuration default-3165dcbb69aeec5c\n",
            "08/25/2021 09:10:49 - INFO - datasets.builder - Overwrite dataset info from restored data version.\n",
            "08/25/2021 09:10:49 - INFO - datasets.info - Loading Dataset info from /root/.cache/huggingface/datasets/text/default-3165dcbb69aeec5c/0.0.0/e16f44aa1b321ece1f87b07977cc5d70be93d69b20486d6dacd62e12cf25c9a5\n",
            "08/25/2021 09:10:49 - WARNING - datasets.builder - Reusing dataset text (/root/.cache/huggingface/datasets/text/default-3165dcbb69aeec5c/0.0.0/e16f44aa1b321ece1f87b07977cc5d70be93d69b20486d6dacd62e12cf25c9a5)\n",
            "08/25/2021 09:10:49 - INFO - datasets.info - Loading Dataset info from /root/.cache/huggingface/datasets/text/default-3165dcbb69aeec5c/0.0.0/e16f44aa1b321ece1f87b07977cc5d70be93d69b20486d6dacd62e12cf25c9a5\n",
            "08/25/2021 09:10:49 - WARNING - datasets.builder - Using custom data configuration default-3165dcbb69aeec5c\n",
            "08/25/2021 09:10:49 - INFO - datasets.builder - Overwrite dataset info from restored data version.\n",
            "08/25/2021 09:10:49 - INFO - datasets.info - Loading Dataset info from /root/.cache/huggingface/datasets/text/default-3165dcbb69aeec5c/0.0.0/e16f44aa1b321ece1f87b07977cc5d70be93d69b20486d6dacd62e12cf25c9a5\n",
            "08/25/2021 09:10:49 - WARNING - datasets.builder - Reusing dataset text (/root/.cache/huggingface/datasets/text/default-3165dcbb69aeec5c/0.0.0/e16f44aa1b321ece1f87b07977cc5d70be93d69b20486d6dacd62e12cf25c9a5)\n",
            "08/25/2021 09:10:49 - INFO - datasets.info - Loading Dataset info from /root/.cache/huggingface/datasets/text/default-3165dcbb69aeec5c/0.0.0/e16f44aa1b321ece1f87b07977cc5d70be93d69b20486d6dacd62e12cf25c9a5\n",
            "[INFO|configuration_utils.py:557] 2021-08-25 09:10:49,864 >> loading configuration file https://huggingface.co/gpt2/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/fc674cd6907b4c9e933cb42d67662436b89fa9540a1f40d7c919d0109289ad01.7d2e0efa5ca20cef4fb199382111e9d3ad96fd77b849e1d4bed13a66e1336f51\n",
            "[INFO|configuration_utils.py:594] 2021-08-25 09:10:49,865 >> Model config GPT2Config {\n",
            "  \"activation_function\": \"gelu_new\",\n",
            "  \"architectures\": [\n",
            "    \"GPT2LMHeadModel\"\n",
            "  ],\n",
            "  \"attn_pdrop\": 0.1,\n",
            "  \"bos_token_id\": 50256,\n",
            "  \"embd_pdrop\": 0.1,\n",
            "  \"eos_token_id\": 50256,\n",
            "  \"gradient_checkpointing\": false,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"layer_norm_epsilon\": 1e-05,\n",
            "  \"model_type\": \"gpt2\",\n",
            "  \"n_ctx\": 1024,\n",
            "  \"n_embd\": 768,\n",
            "  \"n_head\": 12,\n",
            "  \"n_inner\": null,\n",
            "  \"n_layer\": 12,\n",
            "  \"n_positions\": 1024,\n",
            "  \"resid_pdrop\": 0.1,\n",
            "  \"scale_attn_weights\": true,\n",
            "  \"summary_activation\": null,\n",
            "  \"summary_first_dropout\": 0.1,\n",
            "  \"summary_proj_to_labels\": true,\n",
            "  \"summary_type\": \"cls_index\",\n",
            "  \"summary_use_proj\": true,\n",
            "  \"task_specific_params\": {\n",
            "    \"text-generation\": {\n",
            "      \"do_sample\": true,\n",
            "      \"max_length\": 50\n",
            "    }\n",
            "  },\n",
            "  \"transformers_version\": \"4.10.0.dev0\",\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50257\n",
            "}\n",
            "\n",
            "[INFO|tokenization_auto.py:321] 2021-08-25 09:10:49,999 >> Could not locate the tokenizer configuration file, will try to use the model config instead.\n",
            "[INFO|configuration_utils.py:557] 2021-08-25 09:10:50,122 >> loading configuration file https://huggingface.co/gpt2/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/fc674cd6907b4c9e933cb42d67662436b89fa9540a1f40d7c919d0109289ad01.7d2e0efa5ca20cef4fb199382111e9d3ad96fd77b849e1d4bed13a66e1336f51\n",
            "[INFO|configuration_utils.py:594] 2021-08-25 09:10:50,123 >> Model config GPT2Config {\n",
            "  \"activation_function\": \"gelu_new\",\n",
            "  \"architectures\": [\n",
            "    \"GPT2LMHeadModel\"\n",
            "  ],\n",
            "  \"attn_pdrop\": 0.1,\n",
            "  \"bos_token_id\": 50256,\n",
            "  \"embd_pdrop\": 0.1,\n",
            "  \"eos_token_id\": 50256,\n",
            "  \"gradient_checkpointing\": false,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"layer_norm_epsilon\": 1e-05,\n",
            "  \"model_type\": \"gpt2\",\n",
            "  \"n_ctx\": 1024,\n",
            "  \"n_embd\": 768,\n",
            "  \"n_head\": 12,\n",
            "  \"n_inner\": null,\n",
            "  \"n_layer\": 12,\n",
            "  \"n_positions\": 1024,\n",
            "  \"resid_pdrop\": 0.1,\n",
            "  \"scale_attn_weights\": true,\n",
            "  \"summary_activation\": null,\n",
            "  \"summary_first_dropout\": 0.1,\n",
            "  \"summary_proj_to_labels\": true,\n",
            "  \"summary_type\": \"cls_index\",\n",
            "  \"summary_use_proj\": true,\n",
            "  \"task_specific_params\": {\n",
            "    \"text-generation\": {\n",
            "      \"do_sample\": true,\n",
            "      \"max_length\": 50\n",
            "    }\n",
            "  },\n",
            "  \"transformers_version\": \"4.10.0.dev0\",\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50257\n",
            "}\n",
            "\n",
            "[INFO|tokenization_utils_base.py:1735] 2021-08-25 09:10:51,068 >> loading file https://huggingface.co/gpt2/resolve/main/vocab.json from cache at /root/.cache/huggingface/transformers/684fe667923972fb57f6b4dcb61a3c92763ad89882f3da5da9866baf14f2d60f.c7ed1f96aac49e745788faa77ba0a26a392643a50bb388b9c04ff469e555241f\n",
            "[INFO|tokenization_utils_base.py:1735] 2021-08-25 09:10:51,068 >> loading file https://huggingface.co/gpt2/resolve/main/merges.txt from cache at /root/.cache/huggingface/transformers/c0c761a63004025aeadd530c4c27b860ec4ecbe8a00531233de21d865a402598.5d12962c5ee615a4c803841266e9c3be9a691a924f72d395d3a6c6c81157788b\n",
            "[INFO|tokenization_utils_base.py:1735] 2021-08-25 09:10:51,068 >> loading file https://huggingface.co/gpt2/resolve/main/tokenizer.json from cache at /root/.cache/huggingface/transformers/16a2f78023c8dc511294f0c97b5e10fde3ef9889ad6d11ffaa2a00714e73926e.cf2d0ecb83b6df91b3dbb53f1d1e4c311578bfd3aa0e04934215a49bf9898df0\n",
            "[INFO|tokenization_utils_base.py:1735] 2021-08-25 09:10:51,068 >> loading file https://huggingface.co/gpt2/resolve/main/added_tokens.json from cache at None\n",
            "[INFO|tokenization_utils_base.py:1735] 2021-08-25 09:10:51,068 >> loading file https://huggingface.co/gpt2/resolve/main/special_tokens_map.json from cache at None\n",
            "[INFO|tokenization_utils_base.py:1735] 2021-08-25 09:10:51,069 >> loading file https://huggingface.co/gpt2/resolve/main/tokenizer_config.json from cache at None\n",
            "[INFO|configuration_utils.py:557] 2021-08-25 09:10:51,193 >> loading configuration file https://huggingface.co/gpt2/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/fc674cd6907b4c9e933cb42d67662436b89fa9540a1f40d7c919d0109289ad01.7d2e0efa5ca20cef4fb199382111e9d3ad96fd77b849e1d4bed13a66e1336f51\n",
            "[INFO|configuration_utils.py:594] 2021-08-25 09:10:51,194 >> Model config GPT2Config {\n",
            "  \"activation_function\": \"gelu_new\",\n",
            "  \"architectures\": [\n",
            "    \"GPT2LMHeadModel\"\n",
            "  ],\n",
            "  \"attn_pdrop\": 0.1,\n",
            "  \"bos_token_id\": 50256,\n",
            "  \"embd_pdrop\": 0.1,\n",
            "  \"eos_token_id\": 50256,\n",
            "  \"gradient_checkpointing\": false,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"layer_norm_epsilon\": 1e-05,\n",
            "  \"model_type\": \"gpt2\",\n",
            "  \"n_ctx\": 1024,\n",
            "  \"n_embd\": 768,\n",
            "  \"n_head\": 12,\n",
            "  \"n_inner\": null,\n",
            "  \"n_layer\": 12,\n",
            "  \"n_positions\": 1024,\n",
            "  \"resid_pdrop\": 0.1,\n",
            "  \"scale_attn_weights\": true,\n",
            "  \"summary_activation\": null,\n",
            "  \"summary_first_dropout\": 0.1,\n",
            "  \"summary_proj_to_labels\": true,\n",
            "  \"summary_type\": \"cls_index\",\n",
            "  \"summary_use_proj\": true,\n",
            "  \"task_specific_params\": {\n",
            "    \"text-generation\": {\n",
            "      \"do_sample\": true,\n",
            "      \"max_length\": 50\n",
            "    }\n",
            "  },\n",
            "  \"transformers_version\": \"4.10.0.dev0\",\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 50257\n",
            "}\n",
            "\n",
            "[INFO|modeling_utils.py:1275] 2021-08-25 09:10:51,407 >> loading weights file https://huggingface.co/gpt2/resolve/main/pytorch_model.bin from cache at /root/.cache/huggingface/transformers/752929ace039baa8ef70fe21cdf9ab9445773d20e733cf693d667982e210837e.323c769945a351daa25546176f8208b3004b6f563438a7603e7932bae9025925\n",
            "[INFO|modeling_utils.py:1520] 2021-08-25 09:10:53,395 >> All model checkpoint weights were used when initializing GPT2LMHeadModel.\n",
            "\n",
            "[INFO|modeling_utils.py:1529] 2021-08-25 09:10:53,396 >> All the weights of GPT2LMHeadModel were initialized from the model checkpoint at gpt2.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use GPT2LMHeadModel for predictions without further training.\n",
            "08/25/2021 09:10:53 - WARNING - datasets.arrow_dataset - Loading cached processed dataset at /root/.cache/huggingface/datasets/text/default-3165dcbb69aeec5c/0.0.0/e16f44aa1b321ece1f87b07977cc5d70be93d69b20486d6dacd62e12cf25c9a5/cache-42b04e036e82ec43.arrow\n",
            "08/25/2021 09:10:53 - WARNING - datasets.arrow_dataset - Loading cached processed dataset at /root/.cache/huggingface/datasets/text/default-3165dcbb69aeec5c/0.0.0/e16f44aa1b321ece1f87b07977cc5d70be93d69b20486d6dacd62e12cf25c9a5/cache-2a69ba27be35b289.arrow\n",
            "08/25/2021 09:10:53 - WARNING - datasets.arrow_dataset - Loading cached processed dataset at /root/.cache/huggingface/datasets/text/default-3165dcbb69aeec5c/0.0.0/e16f44aa1b321ece1f87b07977cc5d70be93d69b20486d6dacd62e12cf25c9a5/cache-8d3d4bc9bb115654.arrow\n",
            "08/25/2021 09:10:53 - WARNING - datasets.arrow_dataset - Loading cached processed dataset at /root/.cache/huggingface/datasets/text/default-3165dcbb69aeec5c/0.0.0/e16f44aa1b321ece1f87b07977cc5d70be93d69b20486d6dacd62e12cf25c9a5/cache-90f8e6c7eaee44cc.arrow\n",
            "[INFO|trainer.py:1164] 2021-08-25 09:10:55,842 >> ***** Running training *****\n",
            "[INFO|trainer.py:1165] 2021-08-25 09:10:55,842 >>   Num examples = 16662\n",
            "[INFO|trainer.py:1166] 2021-08-25 09:10:55,842 >>   Num Epochs = 5\n",
            "[INFO|trainer.py:1167] 2021-08-25 09:10:55,842 >>   Instantaneous batch size per device = 50\n",
            "[INFO|trainer.py:1168] 2021-08-25 09:10:55,842 >>   Total train batch size (w. parallel, distributed & accumulation) = 50\n",
            "[INFO|trainer.py:1169] 2021-08-25 09:10:55,843 >>   Gradient Accumulation steps = 1\n",
            "[INFO|trainer.py:1170] 2021-08-25 09:10:55,843 >>   Total optimization steps = 1670\n",
            "{'loss': 3.832, 'learning_rate': 3.973265321538069e-05, 'epoch': 1.5}\n",
            " 30% 500/1670 [05:15<12:23,  1.57it/s][INFO|trainer.py:1921] 2021-08-25 09:16:11,815 >> Saving model checkpoint to test-clm/checkpoint-500\n",
            "[INFO|configuration_utils.py:391] 2021-08-25 09:16:11,816 >> Configuration saved in test-clm/checkpoint-500/config.json\n",
            "[INFO|modeling_utils.py:1001] 2021-08-25 09:16:13,200 >> Model weights saved in test-clm/checkpoint-500/pytorch_model.bin\n",
            "[INFO|tokenization_utils_base.py:2014] 2021-08-25 09:16:13,201 >> tokenizer config file saved in test-clm/checkpoint-500/tokenizer_config.json\n",
            "[INFO|tokenization_utils_base.py:2020] 2021-08-25 09:16:13,201 >> Special tokens file saved in test-clm/checkpoint-500/special_tokens_map.json\n",
            "{'loss': 3.4712, 'learning_rate': 1.7364085661173347e-05, 'epoch': 2.99}\n",
            " 60% 1000/1670 [10:37<07:02,  1.58it/s][INFO|trainer.py:1921] 2021-08-25 09:21:33,179 >> Saving model checkpoint to test-clm/checkpoint-1000\n",
            "[INFO|configuration_utils.py:391] 2021-08-25 09:21:33,180 >> Configuration saved in test-clm/checkpoint-1000/config.json\n",
            "[INFO|modeling_utils.py:1001] 2021-08-25 09:21:34,773 >> Model weights saved in test-clm/checkpoint-1000/pytorch_model.bin\n",
            "[INFO|tokenization_utils_base.py:2014] 2021-08-25 09:21:34,773 >> tokenizer config file saved in test-clm/checkpoint-1000/tokenizer_config.json\n",
            "[INFO|tokenization_utils_base.py:2020] 2021-08-25 09:21:34,773 >> Special tokens file saved in test-clm/checkpoint-1000/special_tokens_map.json\n",
            "{'loss': 3.3039, 'learning_rate': 1.267564549716435e-06, 'epoch': 4.49}\n",
            " 90% 1500/1670 [15:57<01:46,  1.59it/s][INFO|trainer.py:1921] 2021-08-25 09:26:53,673 >> Saving model checkpoint to test-clm/checkpoint-1500\n",
            "[INFO|configuration_utils.py:391] 2021-08-25 09:26:53,674 >> Configuration saved in test-clm/checkpoint-1500/config.json\n",
            "[INFO|modeling_utils.py:1001] 2021-08-25 09:26:55,266 >> Model weights saved in test-clm/checkpoint-1500/pytorch_model.bin\n",
            "[INFO|tokenization_utils_base.py:2014] 2021-08-25 09:26:55,266 >> tokenizer config file saved in test-clm/checkpoint-1500/tokenizer_config.json\n",
            "[INFO|tokenization_utils_base.py:2020] 2021-08-25 09:26:55,267 >> Special tokens file saved in test-clm/checkpoint-1500/special_tokens_map.json\n",
            "[INFO|trainer.py:1997] 2021-08-25 09:27:02,483 >> Deleting older checkpoint [test-clm/checkpoint-500] due to args.save_total_limit\n",
            "100% 1670/1670 [17:54<00:00,  1.91it/s][INFO|trainer.py:1362] 2021-08-25 09:28:50,104 >> \n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n",
            "{'train_runtime': 1074.2915, 'train_samples_per_second': 77.549, 'train_steps_per_second': 1.555, 'train_loss': 3.5093976963066056, 'epoch': 5.0}\n",
            "100% 1670/1670 [17:54<00:00,  1.55it/s]\n",
            "[INFO|trainer.py:1921] 2021-08-25 09:28:50,147 >> Saving model checkpoint to test-clm\n",
            "[INFO|configuration_utils.py:391] 2021-08-25 09:28:50,148 >> Configuration saved in test-clm/config.json\n",
            "[INFO|modeling_utils.py:1001] 2021-08-25 09:28:51,493 >> Model weights saved in test-clm/pytorch_model.bin\n",
            "[INFO|tokenization_utils_base.py:2014] 2021-08-25 09:28:51,494 >> tokenizer config file saved in test-clm/tokenizer_config.json\n",
            "[INFO|tokenization_utils_base.py:2020] 2021-08-25 09:28:51,494 >> Special tokens file saved in test-clm/special_tokens_map.json\n",
            "***** train metrics *****\n",
            "  epoch                    =        5.0\n",
            "  train_loss               =     3.5094\n",
            "  train_runtime            = 0:17:54.29\n",
            "  train_samples            =      16662\n",
            "  train_samples_per_second =     77.549\n",
            "  train_steps_per_second   =      1.555\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eWuQMi-TGS17"
      },
      "source": [
        "generate using the fine-tuned model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "F3eqBuN8qiSi",
        "outputId": "17f682ab-01c7-4269-8c79-7485555bed5b"
      },
      "source": [
        "!python run_generation.py \\\n",
        "--help"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "usage: run_generation.py [-h] --model_type MODEL_TYPE --model_name_or_path\n",
            "                         MODEL_NAME_OR_PATH [--prompt PROMPT]\n",
            "                         [--length LENGTH] [--stop_token STOP_TOKEN]\n",
            "                         [--temperature TEMPERATURE]\n",
            "                         [--repetition_penalty REPETITION_PENALTY] [--k K]\n",
            "                         [--p P] [--prefix PREFIX]\n",
            "                         [--padding_text PADDING_TEXT]\n",
            "                         [--xlm_language XLM_LANGUAGE] [--seed SEED]\n",
            "                         [--no_cuda]\n",
            "                         [--num_return_sequences NUM_RETURN_SEQUENCES]\n",
            "                         [--fp16]\n",
            "\n",
            "optional arguments:\n",
            "  -h, --help            show this help message and exit\n",
            "  --model_type MODEL_TYPE\n",
            "                        Model type selected in the list: gpt2, ctrl, openai-\n",
            "                        gpt, xlnet, transfo-xl, xlm\n",
            "  --model_name_or_path MODEL_NAME_OR_PATH\n",
            "                        Path to pre-trained model or shortcut name selected in\n",
            "                        the list: gpt2, ctrl, openai-gpt, xlnet, transfo-xl,\n",
            "                        xlm\n",
            "  --prompt PROMPT\n",
            "  --length LENGTH\n",
            "  --stop_token STOP_TOKEN\n",
            "                        Token at which text generation is stopped\n",
            "  --temperature TEMPERATURE\n",
            "                        temperature of 1.0 has no effect, lower tend toward\n",
            "                        greedy sampling\n",
            "  --repetition_penalty REPETITION_PENALTY\n",
            "                        primarily useful for CTRL model; in that case, use 1.2\n",
            "  --k K\n",
            "  --p P\n",
            "  --prefix PREFIX       Text added prior to input.\n",
            "  --padding_text PADDING_TEXT\n",
            "                        Deprecated, the use of `--prefix` is preferred.\n",
            "  --xlm_language XLM_LANGUAGE\n",
            "                        Optional language when used with the XLM model.\n",
            "  --seed SEED           random seed for initialization\n",
            "  --no_cuda             Avoid using CUDA when available\n",
            "  --num_return_sequences NUM_RETURN_SEQUENCES\n",
            "                        The number of samples to generate.\n",
            "  --fp16                Whether to use 16-bit (mixed) precision (through\n",
            "                        NVIDIA apex) instead of 32-bit\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1OtRODSg36i_",
        "outputId": "a9923fd1-3c42-483b-c530-f538d51cac67"
      },
      "source": [
        "!python run_generation.py \\\n",
        "    --model_type=gpt2 \\\n",
        "    --model_name_or_path=/content/test-clm \\\n",
        "    --length 50 \\\n",
        "    --num_return_sequences 5 \\\n",
        "    --seed 0 \\\n",
        "    --prompt 'Fearsome monsters...'"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "08/25/2021 09:49:58 - WARNING - __main__ - device: cuda, n_gpu: 1, 16-bits training: False\n",
            "08/25/2021 09:50:03 - INFO - __main__ - Namespace(device=device(type='cuda'), fp16=False, k=0, length=50, model_name_or_path='/content/test-clm', model_type='gpt2', n_gpu=1, no_cuda=False, num_return_sequences=5, p=0.9, padding_text='', prefix='', prompt='Fearsome monsters...', repetition_penalty=1.0, seed=0, stop_token=None, temperature=1.0, xlm_language='')\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "=== GENERATED SEQUENCE 1 ===\n",
            "Fearsome monsters... Bad... Villains... Biscuit-san.I knew that from the start.How can you come up with a name for your son?!How can you move that arm?It was made of chains?Yeah, it was made of electricity\n",
            "=== GENERATED SEQUENCE 2 ===\n",
            "Fearsome monsters...Chimera Ants...Guns of the devil...Monsters that kill humans...Incredible monsters...Guns that make people sick...Villagers...And humans...All with the power of God.Brilliant monsters...Human chains...\n",
            "=== GENERATED SEQUENCE 3 ===\n",
            "Fearsome monsters...Treasure trove...I will not forget them!Enough.Knuckle has summoned a number of soldiers, right?With the current situation,he's probably already on the move.Perhaps they're young, but he must be in good shape.F\n",
            "=== GENERATED SEQUENCE 4 ===\n",
            "Fearsome monsters...Villagers...Bodies...Someone who you've only noticed...Well, this wasn't your big idea...I don't know anyone who likes to talk about...Do you know why?Because I alreadyknow...You're welcome here.That's\n",
            "=== GENERATED SEQUENCE 5 ===\n",
            "Fearsome monsters... More than humans...Crudely... Unexplored lands... Magician monsters... More than humans...Evil enclaves... There are...Evil enclaves...There are...Evil enclaves...It appears that Mito-san is a private\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RE_KWlA9tKB4",
        "outputId": "b7343349-e15a-454e-a8d8-e34ac8089f95"
      },
      "source": [
        "!python run_generation.py \\\n",
        "    --model_type=gpt2 \\\n",
        "    --model_name_or_path=/content/test-clm \\\n",
        "    --length 50 \\\n",
        "    --num_return_sequences 5 \\\n",
        "    --seed 0 \\\n",
        "    --prompt Killua"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "08/25/2021 09:48:56 - WARNING - __main__ - device: cuda, n_gpu: 1, 16-bits training: False\n",
            "08/25/2021 09:49:01 - INFO - __main__ - Namespace(device=device(type='cuda'), fp16=False, k=0, length=50, model_name_or_path='/content/test-clm', model_type='gpt2', n_gpu=1, no_cuda=False, num_return_sequences=5, p=0.9, padding_text='', prefix='', prompt='Killua', repetition_penalty=1.0, seed=0, stop_token=None, temperature=1.0, xlm_language='')\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "=== GENERATED SEQUENCE 1 ===\n",
            "Killua's friends.We don't know what they're up to.It seems Gon believes the Hunterskill them right here.Ah, that...Let's go find something good...If we can get them to bring some clean food,they might learn to\n",
            "=== GENERATED SEQUENCE 2 ===\n",
            "Killua?What?Did you say something earlier?Forget it...That was a fight I took a breakfrom.Gon's Razor was used for this fight.He doesn't require a blade.Shoot uses his legs to restrain himself.Yes\n",
            "=== GENERATED SEQUENCE 3 ===\n",
            "Killua, did you hear that Kurapika is a Phantom Troupe member?Yeah, he's been with the Troupe for over ten years now.So, we just hired him...What's this?Come on, Killua, don't worry\n",
            "=== GENERATED SEQUENCE 4 ===\n",
            "Killua, put on your best self!Look at the calendar.Did you need more time?Wait, I'm being stubborn.Get down to business, then.What about us?It's over.Uh-huh.I just need one more\n",
            "=== GENERATED SEQUENCE 5 ===\n",
            "Killua and Killua's HunterpediaBrush Your Teeth.Given your active role in the Hunter Exam,you should be able to demonstrate incredibleaura.You mustn't let your ego fool you.It appears that you've grown a fewtens\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KNsCpdUWtKqP",
        "outputId": "7090370c-10c0-42fc-a534-54ce10230a3b"
      },
      "source": [
        "!python run_generation.py \\\n",
        "    --model_type=gpt2 \\\n",
        "    --model_name_or_path=/content/test-clm \\\n",
        "    --length 50 \\\n",
        "    --num_return_sequences 5 \\\n",
        "    --seed 0 \\\n",
        "    --prompt Gon"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "08/25/2021 09:50:38 - WARNING - __main__ - device: cuda, n_gpu: 1, 16-bits training: False\n",
            "08/25/2021 09:50:42 - INFO - __main__ - Namespace(device=device(type='cuda'), fp16=False, k=0, length=50, model_name_or_path='/content/test-clm', model_type='gpt2', n_gpu=1, no_cuda=False, num_return_sequences=5, p=0.9, padding_text='', prefix='', prompt='Gon', repetition_penalty=1.0, seed=0, stop_token=None, temperature=1.0, xlm_language='')\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "=== GENERATED SEQUENCE 1 ===\n",
            "Gon's strategy is to avoid Bizeff.During that attack, Killua immediately summoned Gon.He has learned his secret...It's...That Ging used the Autonomous Regionto attack his father.He's discovered that Gon's power was strongerthan\n",
            "=== GENERATED SEQUENCE 2 ===\n",
            "Gon?What?Did you say something earlier?Forget it...That was a fight...Yeah, I remember.Gon's the only one I really cared about...And I was just so impressed.What's this?What was it?Kill\n",
            "=== GENERATED SEQUENCE 3 ===\n",
            "Gon, did you hear that?I hear Kurapika-sanis the name of a nearby charity auction.Not you...Your buddy.I'm putting together a list of places to auction items fromthat auction.I can sell them right here in\n",
            "=== GENERATED SEQUENCE 4 ===\n",
            "Gon! Hurry!You're alive, you little punk!Did you hear the sound?Wait up.Yeah, you're staying in the hotel, where you'll probably die.Yep.Okay, I'll return to Leol-san for an\n",
            "=== GENERATED SEQUENCE 5 ===\n",
            "Gon and Killua will win on that score.And their goalis to get Pouf, who's nowtroubled, to back down...And Pouf's best bet is to tell us the reward.Huh?What about him?Good grief\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-O0BM9WYtLRf",
        "outputId": "4e3ee409-737c-4540-8d9d-3f785d83129a"
      },
      "source": [
        "!python run_generation.py \\\n",
        "    --model_type=gpt2 \\\n",
        "    --model_name_or_path=/content/test-clm \\\n",
        "    --length 50 \\\n",
        "    --num_return_sequences 5 \\\n",
        "    --seed 0 \\\n",
        "    --prompt Hunter"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "08/25/2021 09:51:47 - WARNING - __main__ - device: cuda, n_gpu: 1, 16-bits training: False\n",
            "08/25/2021 09:51:51 - INFO - __main__ - Namespace(device=device(type='cuda'), fp16=False, k=0, length=50, model_name_or_path='/content/test-clm', model_type='gpt2', n_gpu=1, no_cuda=False, num_return_sequences=5, p=0.9, padding_text='', prefix='', prompt='Hunter', repetition_penalty=1.0, seed=0, stop_token=None, temperature=1.0, xlm_language='')\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "=== GENERATED SEQUENCE 1 ===\n",
            "Hunter's Necklace is activated.Next time: Bring Back Ants x And x Something.How can I do that?!Ah, that...That was good.Yes, that is indeed his ability.He is known as the King of Ants.\n",
            "=== GENERATED SEQUENCE 2 ===\n",
            "Hunter?What?Did you say something earlier?Yes, we've seen what happened to the Nostrade familyand they were arrested.They're still there...That's incredible.If this is an isolated incident,we're all living in another world.\n",
            "=== GENERATED SEQUENCE 3 ===\n",
            "Hunter, huh?What happened to the Scarlet Eyes?What happened to the Ten Dons?It all started with your father...Your father.The reason youbecame an assassin.You were born with ten faces.I was in need of someone to\n",
            "=== GENERATED SEQUENCE 4 ===\n",
            "Hunter)13) MachoMan x And x FeitanFourThirty-two people were found.Fourteen of them are Hunters,one is an Accompany player, and the remaining are Tutorial players.Don't worry.I just need one more\n",
            "=== GENERATED SEQUENCE 5 ===\n",
            "Hunter an item fromthe 50s that can give yourestricted slots.The Magician category, as well.Let's talk about thecards that...Illumi and Droll got.Illumi's card is a Heavens Arena card.It's a\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YhJuXseNtLqA",
        "outputId": "0683bb77-51e9-4f00-d93f-8486de265bea"
      },
      "source": [
        "!python run_generation.py \\\n",
        "    --model_type=gpt2 \\\n",
        "    --model_name_or_path=/content/test-clm \\\n",
        "    --length 50 \\\n",
        "    --num_return_sequences 5 \\\n",
        "    --seed 0 \\\n",
        "    --prompt Ging"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "08/25/2021 09:53:44 - WARNING - __main__ - device: cuda, n_gpu: 1, 16-bits training: False\n",
            "08/25/2021 09:53:48 - INFO - __main__ - Namespace(device=device(type='cuda'), fp16=False, k=0, length=50, model_name_or_path='/content/test-clm', model_type='gpt2', n_gpu=1, no_cuda=False, num_return_sequences=5, p=0.9, padding_text='', prefix='', prompt='Ging', repetition_penalty=1.0, seed=0, stop_token=None, temperature=1.0, xlm_language='')\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "=== GENERATED SEQUENCE 1 ===\n",
            "Ging's Phantom Troupe.Isn't that right?That's not true.Pitou is invincible.He's just a diversionary...Let's go find something good to do...Something good to do...I know.I'll have to go\n",
            "=== GENERATED SEQUENCE 2 ===\n",
            "Ging?What?Did you say something earlier?How's Gon?That was a coincidence...Yeah,I remember.Ging's the only one I really cared about...And I was just thinking that someone would talk to us.No!If you\n",
            "=== GENERATED SEQUENCE 3 ===\n",
            "Ging died.I've heard that there are four of us.Knuckle, Gon,and Killua had the third, fourth,and fifth victories, three of whomwon with their single hits.Each of them has ended up with six wins, four\n",
            "=== GENERATED SEQUENCE 4 ===\n",
            "Ging!So that he can learn from me!Kil...Did you need more training?Wait, how many days have passedsince the match began?Seven days...Four days?That's over two years.We are at the pinnacle of Greed\n",
            "=== GENERATED SEQUENCE 5 ===\n",
            "Ging!Ten days have passedsince that shooting.And theirstrength and defensehave beengetter than ever.Ging!We're back to business!Ten days?Pouf, my big brother and myselfwill be here, right?No...We\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CeF_gAuttMAf",
        "outputId": "d1ab43fc-3ceb-445b-a6b0-d077070a9e7d"
      },
      "source": [
        "!python run_generation.py \\\n",
        "    --model_type=gpt2 \\\n",
        "    --model_name_or_path=/content/test-clm \\\n",
        "    --length 50 \\\n",
        "    --num_return_sequences 5 \\\n",
        "    --seed 42 \\\n",
        "    --prompt Kite"
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "08/25/2021 09:55:45 - WARNING - __main__ - device: cuda, n_gpu: 1, 16-bits training: False\n",
            "08/25/2021 09:55:49 - INFO - __main__ - Namespace(device=device(type='cuda'), fp16=False, k=0, length=50, model_name_or_path='/content/test-clm', model_type='gpt2', n_gpu=1, no_cuda=False, num_return_sequences=5, p=0.9, padding_text='', prefix='', prompt='Kite', repetition_penalty=1.0, seed=42, stop_token=None, temperature=1.0, xlm_language='')\n",
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n",
            "=== GENERATED SEQUENCE 1 ===\n",
            "Kite, aren't you some kind of punk?!Bastard...If you stop here, I'll rip your corpse out of the ground!Wait!Kill him, and we'll return to NGL immediately!Stop talking, Bisky.Hurry\n",
            "=== GENERATED SEQUENCE 2 ===\n",
            "Kite!Be patient.To avoid my wrath, you must not wear the ring.No, sir.If you receive the favor, you must bring my three soldiers along.But if you lose,I will not grant you a single ticket.All right\n",
            "=== GENERATED SEQUENCE 3 ===\n",
            "Kite was already up there.I must catch a glimpse of Gon before he falls asleep.﻿Killua?Stop this!I can't say anything until Gon wakes up.﻿Gon and Killua can't tell when his brother\n",
            "=== GENERATED SEQUENCE 4 ===\n",
            "Kite?!What are you talking about?The nearest personwas recently taken,and he looked like hehad a rough time.He may have been captured by an assassin,but he didn't get away.We should keep this in mindwhen we decide what\n",
            "=== GENERATED SEQUENCE 5 ===\n",
            "Kite.It's the logical next step.You have a little over a yearuntil you become an examiner.You're one of the few applicants left.Why are you so careful to bringdown yourselfsire?Because if I can reproduce your ability\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}