NLP_HW5_2.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "NLP_HW5.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "gpuClass": "standard",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "ec84897dc6134b3a9d12232ef2cec79b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_9e2c183e18e341e4825e7aadeeffc3b5",
              "IPY_MODEL_6f1322075b0c46b1be5d7fb58569b306",
              "IPY_MODEL_aa51cb0ec204469986c2ca7eeed55811"
            ],
            "layout": "IPY_MODEL_1c268d420a5c4233a39c8aa66df4627d"
          }
        },
        "9e2c183e18e341e4825e7aadeeffc3b5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2b033acbcbcb45a7b2a3a3d8a91d3691",
            "placeholder": "​",
            "style": "IPY_MODEL_8e9a772ac2a9420a94b4c28309d531fc",
            "value": "Downloading tokenizer_config.json: 100%"
          }
        },
        "6f1322075b0c46b1be5d7fb58569b306": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3acb344aebcb464aa8a80f75487f42c2",
            "max": 292,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_6064d9daceb94882931d3199e3e5c085",
            "value": 292
          }
        },
        "aa51cb0ec204469986c2ca7eeed55811": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e4163d60c0f6439ca2f1c49309716c59",
            "placeholder": "​",
            "style": "IPY_MODEL_4761087cf87c49348211d48b62f802d3",
            "value": " 292/292 [00:00&lt;00:00, 1.75kB/s]"
          }
        },
        "1c268d420a5c4233a39c8aa66df4627d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2b033acbcbcb45a7b2a3a3d8a91d3691": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8e9a772ac2a9420a94b4c28309d531fc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3acb344aebcb464aa8a80f75487f42c2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6064d9daceb94882931d3199e3e5c085": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "e4163d60c0f6439ca2f1c49309716c59": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4761087cf87c49348211d48b62f802d3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "27064479d6d647428b50ece45fca966e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_aa6dbbbf21124990a67be1fb1d67789b",
              "IPY_MODEL_eff965e7fa604cd685cf37038e938789",
              "IPY_MODEL_94bb7ff233fa462b8ee1493788f1cb72"
            ],
            "layout": "IPY_MODEL_156e16b9fe004a02abc5551ceb3ac6b1"
          }
        },
        "aa6dbbbf21124990a67be1fb1d67789b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_511fc8f261fb4a33a34eea7b181b630d",
            "placeholder": "​",
            "style": "IPY_MODEL_38af5fa3fdce472cbf74fb10250ab8e4",
            "value": "Downloading config.json: 100%"
          }
        },
        "eff965e7fa604cd685cf37038e938789": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f9eede9491e940ca816bf5b1912a2fcb",
            "max": 565,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_96ebed29a4ed4d0bab420512ac068706",
            "value": 565
          }
        },
        "94bb7ff233fa462b8ee1493788f1cb72": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e9c9acb02f5c427aa4af6db29f27dff7",
            "placeholder": "​",
            "style": "IPY_MODEL_02434d44cba84ccd8b68098deb54b97a",
            "value": " 565/565 [00:00&lt;00:00, 3.31kB/s]"
          }
        },
        "156e16b9fe004a02abc5551ceb3ac6b1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "511fc8f261fb4a33a34eea7b181b630d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "38af5fa3fdce472cbf74fb10250ab8e4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f9eede9491e940ca816bf5b1912a2fcb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "96ebed29a4ed4d0bab420512ac068706": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "e9c9acb02f5c427aa4af6db29f27dff7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "02434d44cba84ccd8b68098deb54b97a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a703e05e1ba04b1e9581ceb7c410e85c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_699c60a97e75430dbe834e8805f0e06a",
              "IPY_MODEL_8b7ba2cd716b4e8a8a2cebf801273b2f",
              "IPY_MODEL_0c2324e7f4bd40a7938ee1a3881568e2"
            ],
            "layout": "IPY_MODEL_3fe66cf458304a16ae5499759bddcd81"
          }
        },
        "699c60a97e75430dbe834e8805f0e06a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2c8f12dcfeec4fc3a1ce0c1d9f6df5b0",
            "placeholder": "​",
            "style": "IPY_MODEL_78540b5c098244b2a729700add745d3e",
            "value": "Downloading vocab.txt: 100%"
          }
        },
        "8b7ba2cd716b4e8a8a2cebf801273b2f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b20e6d56d4c342dfad334cbd9789ccf6",
            "max": 426422,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_8fefa2796f4e4e96b936046976ff6ae1",
            "value": 426422
          }
        },
        "0c2324e7f4bd40a7938ee1a3881568e2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cf05e7791bc9442d8bf0fd9c4bde62ee",
            "placeholder": "​",
            "style": "IPY_MODEL_b569c54d4ebd469d8b870d5386207737",
            "value": " 416k/416k [00:00&lt;00:00, 753kB/s]"
          }
        },
        "3fe66cf458304a16ae5499759bddcd81": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2c8f12dcfeec4fc3a1ce0c1d9f6df5b0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "78540b5c098244b2a729700add745d3e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b20e6d56d4c342dfad334cbd9789ccf6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8fefa2796f4e4e96b936046976ff6ae1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "cf05e7791bc9442d8bf0fd9c4bde62ee": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b569c54d4ebd469d8b870d5386207737": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "11cbcc7e45a545c5b62496a85b3f6c1e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_e85e09290fa94175b2d9cedc776d5a8b",
              "IPY_MODEL_15a7ba6a5561496d95f514365f2c0ff6",
              "IPY_MODEL_0d30deac0d7646e3bcb1c2cedcc0d253"
            ],
            "layout": "IPY_MODEL_008d84f5b2f64fd0b7da3f33093bf3b1"
          }
        },
        "e85e09290fa94175b2d9cedc776d5a8b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1b030c1c9a434d0fb43080933c2b6195",
            "placeholder": "​",
            "style": "IPY_MODEL_be9e409f67ee43288222b48d081034af",
            "value": "Downloading tokenizer.json: 100%"
          }
        },
        "15a7ba6a5561496d95f514365f2c0ff6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8741b486b38b4c5fbab4886f6c631ac3",
            "max": 1108824,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_72565303c9f64bad84fc7683b63bc80d",
            "value": 1108824
          }
        },
        "0d30deac0d7646e3bcb1c2cedcc0d253": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5c970884b90e4efdb611d5b392156f7d",
            "placeholder": "​",
            "style": "IPY_MODEL_fc0b9985ed85428cb540a34d53d358cf",
            "value": " 1.06M/1.06M [00:00&lt;00:00, 1.18MB/s]"
          }
        },
        "008d84f5b2f64fd0b7da3f33093bf3b1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1b030c1c9a434d0fb43080933c2b6195": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "be9e409f67ee43288222b48d081034af": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8741b486b38b4c5fbab4886f6c631ac3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "72565303c9f64bad84fc7683b63bc80d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "5c970884b90e4efdb611d5b392156f7d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fc0b9985ed85428cb540a34d53d358cf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "37f406906a824680b5d9e5a7a62fa8c6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_d745b327cfa7427dbc4fc37ec68d6e54",
              "IPY_MODEL_29672d3c481f4fc8bf7d8f67a33f57cb",
              "IPY_MODEL_ae610e7d90824c8c88c970d0b04732a4"
            ],
            "layout": "IPY_MODEL_a4f3100df50c406d8b7c66b1c8c7170c"
          }
        },
        "d745b327cfa7427dbc4fc37ec68d6e54": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cfcf8486aa6c46aab8c10f93ef247e6c",
            "placeholder": "​",
            "style": "IPY_MODEL_d8e4fbf375464ab5ad64ab5ee85d40e2",
            "value": "Downloading special_tokens_map.json: 100%"
          }
        },
        "29672d3c481f4fc8bf7d8f67a33f57cb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_840c1c135ef54b04bd3d1df1885f9a4b",
            "max": 134,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e18e5fb0cfc64ab09ee8a7499603c003",
            "value": 134
          }
        },
        "ae610e7d90824c8c88c970d0b04732a4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1f076ccc9cd240288d069f0aef71b9bb",
            "placeholder": "​",
            "style": "IPY_MODEL_f09dee1990004f1e9ceefc38c57005e0",
            "value": " 134/134 [00:00&lt;00:00, 791B/s]"
          }
        },
        "a4f3100df50c406d8b7c66b1c8c7170c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cfcf8486aa6c46aab8c10f93ef247e6c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d8e4fbf375464ab5ad64ab5ee85d40e2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "840c1c135ef54b04bd3d1df1885f9a4b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e18e5fb0cfc64ab09ee8a7499603c003": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "1f076ccc9cd240288d069f0aef71b9bb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f09dee1990004f1e9ceefc38c57005e0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "168d5d51efc44eae89a18253565ccf01": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_2114b028a15f458bb9326e256397c999",
              "IPY_MODEL_0bb60baaaa674bf6a5e5f0ac33342942",
              "IPY_MODEL_045457c6228c4daf8254358dc0e4f589"
            ],
            "layout": "IPY_MODEL_836bfcb87f8e41a9b0c094f74cbc0731"
          }
        },
        "2114b028a15f458bb9326e256397c999": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_10902ca937e9436385e0b17899e617c0",
            "placeholder": "​",
            "style": "IPY_MODEL_7c9363a004e14789bab5d1bcf9053a60",
            "value": "100%"
          }
        },
        "0bb60baaaa674bf6a5e5f0ac33342942": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4a34e319a80646d9a2eb813ad931678b",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e9410248e64a485998efe818f5de2891",
            "value": 1
          }
        },
        "045457c6228c4daf8254358dc0e4f589": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_923195bc7b754378887900213accd871",
            "placeholder": "​",
            "style": "IPY_MODEL_c8f6be46c946452fa0ccb6b5ce49e059",
            "value": " 1/1 [00:00&lt;00:00,  3.40ba/s]"
          }
        },
        "836bfcb87f8e41a9b0c094f74cbc0731": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "10902ca937e9436385e0b17899e617c0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7c9363a004e14789bab5d1bcf9053a60": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4a34e319a80646d9a2eb813ad931678b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e9410248e64a485998efe818f5de2891": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "923195bc7b754378887900213accd871": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c8f6be46c946452fa0ccb6b5ce49e059": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "65a86757e06c429ebabe2abe21b47135": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_738703125b3f4daabb8cc51d53536909",
              "IPY_MODEL_be7feac07caf4d3384f513227b81a8b0",
              "IPY_MODEL_9f169abcdd9b413db8fc8cf76fa6ea7a"
            ],
            "layout": "IPY_MODEL_8c76c4a8350d46318931cc886bb8012d"
          }
        },
        "738703125b3f4daabb8cc51d53536909": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8af3e369f16742f28edd70b4b015301e",
            "placeholder": "​",
            "style": "IPY_MODEL_0bb78fd9ebc8479c84a3aeaaa90c78ad",
            "value": "100%"
          }
        },
        "be7feac07caf4d3384f513227b81a8b0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7f8e27e17ed4433fbcf091a1da593a04",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_389477fe499e44d29b80a2c9b81eb88b",
            "value": 1
          }
        },
        "9f169abcdd9b413db8fc8cf76fa6ea7a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b88e63c0cbcb4dcc97e31a02a090a7a3",
            "placeholder": "​",
            "style": "IPY_MODEL_f701162f7f8c4fd4a972a7634b3e8d04",
            "value": " 1/1 [00:00&lt;00:00,  3.79ba/s]"
          }
        },
        "8c76c4a8350d46318931cc886bb8012d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8af3e369f16742f28edd70b4b015301e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0bb78fd9ebc8479c84a3aeaaa90c78ad": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7f8e27e17ed4433fbcf091a1da593a04": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "389477fe499e44d29b80a2c9b81eb88b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b88e63c0cbcb4dcc97e31a02a090a7a3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f701162f7f8c4fd4a972a7634b3e8d04": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "83ec83544b5d42fdb65d61c7991e7dd6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_d686e4f374dd463cb786a92b568e07de",
              "IPY_MODEL_3869383e67164c31864e131579dff169",
              "IPY_MODEL_539291d1047348df8c414e69737cd39c"
            ],
            "layout": "IPY_MODEL_3f3b3df68173481a9d2c6bc851fc739d"
          }
        },
        "d686e4f374dd463cb786a92b568e07de": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b239b528dffd4c4f96e27e1fc1d9dc95",
            "placeholder": "​",
            "style": "IPY_MODEL_e4e5fc04433f419dadc9dee2db54c7a8",
            "value": "100%"
          }
        },
        "3869383e67164c31864e131579dff169": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cacbc753fe6542d9a0e3246289993dbf",
            "max": 5,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ccb07c2bd5fb46de8413ef3bb0f6e5b5",
            "value": 5
          }
        },
        "539291d1047348df8c414e69737cd39c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_40363b407acf4ef4878ae3414472fb3b",
            "placeholder": "​",
            "style": "IPY_MODEL_565febd28fc34798befa69fcd44921bf",
            "value": " 5/5 [00:03&lt;00:00,  1.04ba/s]"
          }
        },
        "3f3b3df68173481a9d2c6bc851fc739d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b239b528dffd4c4f96e27e1fc1d9dc95": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e4e5fc04433f419dadc9dee2db54c7a8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "cacbc753fe6542d9a0e3246289993dbf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ccb07c2bd5fb46de8413ef3bb0f6e5b5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "40363b407acf4ef4878ae3414472fb3b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "565febd28fc34798befa69fcd44921bf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "421389d8df1a48b7b7a23226a0773959": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_9d3aefb8d80640f399313798543dcb87",
              "IPY_MODEL_1c9cc2c992734c03afde9c252b8c3d1e",
              "IPY_MODEL_5b5da2aa2de540e0bdb80782a320aab3"
            ],
            "layout": "IPY_MODEL_b13651c05aa4489a951d25315f94829f"
          }
        },
        "9d3aefb8d80640f399313798543dcb87": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_808c2845d7c946dda19b983162faf3e7",
            "placeholder": "​",
            "style": "IPY_MODEL_cf57d12ee4ac4491a911ecdb075fc93b",
            "value": "Epoch 0:   0%"
          }
        },
        "1c9cc2c992734c03afde9c252b8c3d1e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_dd310b02826041bf9506f6abfdb5c863",
            "max": 1,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_565e9d4f1a134d0a8f8fd8412b6af346",
            "value": 0
          }
        },
        "5b5da2aa2de540e0bdb80782a320aab3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fd3111e8d3bb43f88d9107a16d852f17",
            "placeholder": "​",
            "style": "IPY_MODEL_92570bb14be545349604e86283a37c56",
            "value": " 0/1 [00:00&lt;?, ?it/s]"
          }
        },
        "b13651c05aa4489a951d25315f94829f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": "inline-flex",
            "flex": null,
            "flex_flow": "row wrap",
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "100%"
          }
        },
        "808c2845d7c946dda19b983162faf3e7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cf57d12ee4ac4491a911ecdb075fc93b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "dd310b02826041bf9506f6abfdb5c863": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": "2",
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "565e9d4f1a134d0a8f8fd8412b6af346": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "fd3111e8d3bb43f88d9107a16d852f17": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "92570bb14be545349604e86283a37c56": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/pam-lab/JupyterFiles/blob/main/NLP_HW5_2.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "pWENFIEONoKE"
      },
      "outputs": [],
      "source": [
        "from __future__ import absolute_import, division, print_function, unicode_literals\n",
        "\n",
        "try:\n",
        "    # %tensorflow_version only exists in Colab.\n",
        "    %tensorflow_version 2.x\n",
        "except Exception:\n",
        "    pass\n",
        "\n",
        "import tensorflow as tf"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -Uqq transformers\n",
        "!pip install -Uqq hazm\n",
        "!pip install -Uqq datasets\n",
        "!pip install -Uqq seqeval"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "k4FFyHSANxV-",
        "outputId": "f8f826be-4bff-41b8-aa5e-4f54a1f39266"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[K     |████████████████████████████████| 4.7 MB 8.5 MB/s \n",
            "\u001b[K     |████████████████████████████████| 101 kB 6.8 MB/s \n",
            "\u001b[K     |████████████████████████████████| 6.6 MB 43.7 MB/s \n",
            "\u001b[K     |████████████████████████████████| 596 kB 36.4 MB/s \n",
            "\u001b[K     |████████████████████████████████| 316 kB 9.6 MB/s \n",
            "\u001b[K     |████████████████████████████████| 1.4 MB 45.1 MB/s \n",
            "\u001b[K     |████████████████████████████████| 233 kB 39.2 MB/s \n",
            "\u001b[?25h  Building wheel for nltk (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for libwapiti (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[K     |████████████████████████████████| 365 kB 7.0 MB/s \n",
            "\u001b[K     |████████████████████████████████| 141 kB 53.0 MB/s \n",
            "\u001b[K     |████████████████████████████████| 212 kB 44.8 MB/s \n",
            "\u001b[K     |████████████████████████████████| 127 kB 41.9 MB/s \n",
            "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
            "datascience 0.10.6 requires folium==0.2.1, but you have folium 0.8.3 which is incompatible.\u001b[0m\n",
            "\u001b[K     |████████████████████████████████| 43 kB 1.4 MB/s \n",
            "\u001b[?25h  Building wheel for seqeval (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "import hazm\n",
        "\n",
        "import transformers \n",
        "from transformers import AutoTokenizer, AutoConfig\n",
        "from transformers import TFAutoModelForTokenClassification\n",
        "import itertools\n",
        "from datasets import Dataset\n",
        "from datasets import load_metric\n",
        "from transformers import TrainingArguments, Trainer\n",
        "from transformers import DataCollatorForTokenClassification\n",
        "\n",
        "\n",
        "import os\n",
        "import itertools\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "from datasets import Dataset\n",
        "from datasets import load_metric\n",
        "from transformers import AutoTokenizer\n",
        "from transformers import AutoModelForTokenClassification, TrainingArguments, Trainer\n",
        "from transformers import DataCollatorForTokenClassification\n",
        "import torch\n",
        "\n",
        "import os\n",
        "from IPython.display import display, HTML, clear_output\n",
        "from ipywidgets import widgets, Layout\n",
        "\n",
        "print()\n",
        "print('tensorflow', tf.__version__)\n",
        "print('transformers', transformers.__version__)\n",
        "print('numpy', np.__version__)\n",
        "print('pandas', pd.__version__)\n",
        "print()\n",
        "\n",
        "if tf.test.gpu_device_name() != '/device:GPU:0':\n",
        "    print()\n",
        "    print('WARNING: GPU device not found.')\n",
        "else:\n",
        "    print()\n",
        "    print('SUCCESS: Found GPU: {}'.format(tf.test.gpu_device_name()))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Jvnv1J0jN3p_",
        "outputId": "704722b4-8036-445f-9445-fe10a47ffceb"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "tensorflow 2.8.2\n",
            "transformers 4.21.0\n",
            "numpy 1.21.6\n",
            "pandas 1.3.5\n",
            "\n",
            "\n",
            "WARNING: GPU device not found.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Fine-tune model with our data"
      ],
      "metadata": {
        "id": "P4VswAICRKJp"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!gdown 1oJjoGK15-RD7IHm8-k0DhSXFlWDaq201\n",
        "!gdown 1AzXg9YVaXCAzalr57IpP1sNbWBBbZrqI\n",
        "!gdown 1LXZrlNFr8jKhVF5ETEu6OwnuwPfrIyLw"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tWv3wxXzYxtd",
        "outputId": "63b252f2-92f4-47d3-9e3b-94a93049048f"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=1oJjoGK15-RD7IHm8-k0DhSXFlWDaq201\n",
            "To: /content/ner-train.csv\n",
            "100% 6.85M/6.85M [00:00<00:00, 37.3MB/s]\n",
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=1AzXg9YVaXCAzalr57IpP1sNbWBBbZrqI\n",
            "To: /content/ner-test.csv\n",
            "100% 383k/383k [00:00<00:00, 52.0MB/s]\n",
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=1LXZrlNFr8jKhVF5ETEu6OwnuwPfrIyLw\n",
            "To: /content/ner-eval.csv\n",
            "100% 348k/348k [00:00<00:00, 65.2MB/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data_train = pd.read_csv('ner-train.csv')\n",
        "data_val = pd.read_csv('ner-eval.csv')\n",
        "data_test = pd.read_csv('ner-test.csv')"
      ],
      "metadata": {
        "id": "CpWNYP7DZP4W"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "len(data_train), len(data_val), len(data_test)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "c_kQf-xfo9mM",
        "outputId": "dd9ed899-16c5-48d3-ebbe-d59a19b61f26"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(4050, 225, 225)"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def clean_alt_list(list_):\n",
        "    list_ = list_.replace('[','').replace(']','').split(',')\n",
        "    return [item.replace(\"'\",\"\").strip() for item in list_]"
      ],
      "metadata": {
        "id": "_kOjjF8vqFTx"
      },
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def convert_to_list(data):\n",
        "    data['token'] = data['token'].apply(clean_alt_list)\n",
        "    data['labels'] = data['labels'].apply(clean_alt_list)"
      ],
      "metadata": {
        "id": "Yx08yK3tqHqD"
      },
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "convert_to_list(data_train)\n",
        "convert_to_list(data_val)\n",
        "convert_to_list(data_test)"
      ],
      "metadata": {
        "id": "k2ysn3FaZuwy"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data_train"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "id": "fw_lMFSeZ9PM",
        "outputId": "8273ea79-1342-45d4-f6f0-c0ab9410b219"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                                  token  \\\n",
              "0     [«صندلی, داغ», و, «شب, آرام», در, نوروزی, های,...   \n",
              "1     [به, گزارش, خبرگزاری, مهر, به, نقل, از, روابط,...   \n",
              "2     [به, گزارش, خبرگزاری, مهر, به, نقل, از, روابط,...   \n",
              "3     [نشان, درجه, یک, هنری, استاد, «محمدرضا, اسحاقی...   \n",
              "4     [به, گزارش, خبرنگار, خبرگزاری, شبستان, مازندرا...   \n",
              "...                                                 ...   \n",
              "4045  [اگر, این, تحریم, ها, موجب, تضعیف, اقتصاد, روس...   \n",
              "4046  [اگر, این, تحریم, ها, موجب, تضعیف, اقتصاد, روس...   \n",
              "4047  [هفته, هشتم, اسبدوانی, گنبدکاووس, با, حضور, سر...   \n",
              "4048  [به, گزارش, خبرگزاری, فارس, از, گرگان،, روز, د...   \n",
              "4049  [به, گزارش, خبرگزاری, فارس, از, گرگان،, روز, د...   \n",
              "\n",
              "                                                 labels  \n",
              "0                [o, o, o, o, o, o, B-DAT, I-DAT, o, o]  \n",
              "1     [o, o, B-ORG, I-ORG, I-ORG, o, o, B-ORG, I-ORG...  \n",
              "2     [o, o, B-ORG, I-ORG, I-ORG, o, o, B-ORG, I-ORG...  \n",
              "3     [o, o, o, o, o, o, B-PER, o, o, o, o, o, o, o, o]  \n",
              "4     [o, o, o, B-ORG, I-ORG, I-ORG, o, B-LOC, o, B-...  \n",
              "...                                                 ...  \n",
              "4045  [o, o, o, o, o, o, o, o, o, o, o, o, o, o, o, ...  \n",
              "4046  [o, o, o, o, o, o, o, o, o, o, o, o, o, o, o, ...  \n",
              "4047  [B-DAT, I-DAT, o, B-LOC, o, o, o, o, o, o, o, ...  \n",
              "4048  [o, o, B-ORG, I-ORG, o, B-LOC, o, B-DAT, I-DAT...  \n",
              "4049  [o, o, B-ORG, I-ORG, o, B-LOC, o, B-DAT, I-DAT...  \n",
              "\n",
              "[4050 rows x 2 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-4b31728b-0ed6-4dee-afb8-fc418c907926\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>token</th>\n",
              "      <th>labels</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>[«صندلی, داغ», و, «شب, آرام», در, نوروزی, های,...</td>\n",
              "      <td>[o, o, o, o, o, o, B-DAT, I-DAT, o, o]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>[به, گزارش, خبرگزاری, مهر, به, نقل, از, روابط,...</td>\n",
              "      <td>[o, o, B-ORG, I-ORG, I-ORG, o, o, B-ORG, I-ORG...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>[به, گزارش, خبرگزاری, مهر, به, نقل, از, روابط,...</td>\n",
              "      <td>[o, o, B-ORG, I-ORG, I-ORG, o, o, B-ORG, I-ORG...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>[نشان, درجه, یک, هنری, استاد, «محمدرضا, اسحاقی...</td>\n",
              "      <td>[o, o, o, o, o, o, B-PER, o, o, o, o, o, o, o, o]</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>[به, گزارش, خبرنگار, خبرگزاری, شبستان, مازندرا...</td>\n",
              "      <td>[o, o, o, B-ORG, I-ORG, I-ORG, o, B-LOC, o, B-...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4045</th>\n",
              "      <td>[اگر, این, تحریم, ها, موجب, تضعیف, اقتصاد, روس...</td>\n",
              "      <td>[o, o, o, o, o, o, o, o, o, o, o, o, o, o, o, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4046</th>\n",
              "      <td>[اگر, این, تحریم, ها, موجب, تضعیف, اقتصاد, روس...</td>\n",
              "      <td>[o, o, o, o, o, o, o, o, o, o, o, o, o, o, o, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4047</th>\n",
              "      <td>[هفته, هشتم, اسبدوانی, گنبدکاووس, با, حضور, سر...</td>\n",
              "      <td>[B-DAT, I-DAT, o, B-LOC, o, o, o, o, o, o, o, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4048</th>\n",
              "      <td>[به, گزارش, خبرگزاری, فارس, از, گرگان،, روز, د...</td>\n",
              "      <td>[o, o, B-ORG, I-ORG, o, B-LOC, o, B-DAT, I-DAT...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4049</th>\n",
              "      <td>[به, گزارش, خبرگزاری, فارس, از, گرگان،, روز, د...</td>\n",
              "      <td>[o, o, B-ORG, I-ORG, o, B-LOC, o, B-DAT, I-DAT...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>4050 rows × 2 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-4b31728b-0ed6-4dee-afb8-fc418c907926')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-4b31728b-0ed6-4dee-afb8-fc418c907926 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-4b31728b-0ed6-4dee-afb8-fc418c907926');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt\n",
        "labels_dist = []\n",
        "for item in data_train['labels']:\n",
        "    labels_dist = labels_dist + item\n",
        "\n",
        "plt.figure(figsize=(20,5))\n",
        "plt.hist(labels_dist, )"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 393
        },
        "id": "Vwq3ADGVyoqe",
        "outputId": "4a48ac69-12e4-43a5-e52b-a07e54c1b8be"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(array([3.60445e+05, 6.49000e+03, 7.49900e+03, 9.38800e+03, 2.31000e+02,\n",
              "        3.24000e+03, 7.48000e+02, 4.69000e+02, 2.46400e+03, 1.73900e+03]),\n",
              " array([ 0. ,  1.6,  3.2,  4.8,  6.4,  8. ,  9.6, 11.2, 12.8, 14.4, 16. ]),\n",
              " <a list of 10 Patch objects>)"
            ]
          },
          "metadata": {},
          "execution_count": 11
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 1440x360 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAABJcAAAEvCAYAAAD4qCBXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3df7BmdX0n+PdnQBImhoDSw7JAAptQ5aITW2WRmZhaRyrQmqqF1BjFrQkdi5JkxKrEGTNiMrUYDRvdSsKWO4YtElkgNQlSGkdiSBjKmElmdkFaRX6pQw9igEJpAWEYJxrws3883x4fbu7te+/p29yn+75eVU/dcz7ne875fp9f9/a7z4/q7gAAAADAFH9nszsAAAAAwMFLuAQAAADAZMIlAAAAACYTLgEAAAAwmXAJAAAAgMmESwAAAABMdvhmd2CjHXvssX3yySdvdjcAAAAADhmf+cxnvt7d25ZbdsiFSyeffHJ27dq12d0AAAAAOGRU1VdWWua0OAAAAAAmEy4BAAAAMJlwCQAAAIDJhEsAAAAATCZcAgAAAGAy4RIAAAAAkwmXAAAAAJhMuAQAAADAZMIlAAAAACYTLgEAAAAwmXAJAAAAgMkO3+wOsLKTL/njze4CS9z/vp/c7C4AAADAQnHkEgAAAACTCZcAAAAAmEy4BAAAAMBkq4ZLVfW9VfXpqvp8Vd1dVb866ldX1Zer6vbx2D7qVVUfqKrdVXVHVb18bls7q+re8dg5V39FVd051vlAVdWov6Cqbh7tb66qYzb+KQAAAABgqrUcufStJK/p7pcm2Z5kR1WdOZb9UndvH4/bR+21SU4dj4uSXJHMgqIklyZ5ZZIzklw6FxZdkeQtc+vtGPVLknyyu09N8skxDwAAAMCCWDVc6pmnxuzzxqP3scq5Sa4d692S5OiqOj7JOUlu7u7HuvvxJDdnFlQdn+So7r6luzvJtUnOm9vWNWP6mrk6AAAAAAtgTddcqqrDqur2JI9kFhDdOhZdNk59u7yqvmfUTkjywNzqD47avuoPLlNPkuO6++Ex/dUkx63Qv4uqaldV7dqzZ89ahgQAAADABlhTuNTdz3T39iQnJjmjql6S5F1JXpTkf0rygiTvPGC9nPWhs8IRU919ZXef3t2nb9u27UB2AwAAAIA567pbXHd/I8mnkuzo7ofHqW/fSvL/ZHYdpSR5KMlJc6udOGr7qp+4TD1JvjZOm8v4+ch6+gsAAADAgbWWu8Vtq6qjx/SRSX4iyRfnQp/K7FpId41Vbkhywbhr3JlJnhintt2U5OyqOmZcyPvsJDeNZU9W1ZljWxck+fjctvbeVW7nXB0AAACABXD4Gtocn+SaqjosszDq+u7+RFX9WVVtS1JJbk/y86P9jUlel2R3km8meXOSdPdjVfXeJLeNdu/p7sfG9FuTXJ3kyCR/Mh5J8r4k11fVhUm+kuQNUwcKAAAAwMZbNVzq7juSvGyZ+mtWaN9JLl5h2VVJrlqmvivJS5apP5rkrNX6CAAAAMDmWNc1lwAAAABgnnAJAAAAgMmESwAAAABMJlwCAAAAYDLhEgAAAACTCZcAAAAAmEy4BAAAAMBkwiUAAAAAJhMuAQAAADCZcAkAAACAyYRLAAAAAEwmXAIAAABgMuESAAAAAJMJlwAAAACYTLgEAAAAwGTCJQAAAAAmEy4BAAAAMJlwCQAAAIDJhEsAAAAATCZcAgAAAGAy4RIAAAAAkwmXAAAAAJhMuAQAAADAZMIlAAAAACYTLgEAAAAwmXAJAAAAgMmESwAAAABMtmq4VFXfW1WfrqrPV9XdVfWro35KVd1aVbur6sNVdcSof8+Y3z2Wnzy3rXeN+peq6py5+o5R211Vl8zVl90HAAAAAIthLUcufSvJa7r7pUm2J9lRVWcmeX+Sy7v7R5I8nuTC0f7CJI+P+uWjXarqtCTnJ3lxkh1JfruqDquqw5J8MMlrk5yW5E2jbfaxDwAAAAAWwKrhUs88NWafNx6d5DVJPjLq1yQ5b0yfO+Yzlp9VVTXq13X3t7r7y0l2JzljPHZ3933d/e0k1yU5d6yz0j4AAAAAWABruubSOMLo9iSPJLk5yX9K8o3ufno0eTDJCWP6hCQPJMlY/kSSF87Xl6yzUv2F+9jH0v5dVFW7qmrXnj171jIkAAAAADbAmsKl7n6mu7cnOTGzI41edEB7tU7dfWV3n97dp2/btm2zuwMAAACwZazrbnHd/Y0kn0ryD5IcXVWHj0UnJnloTD+U5KQkGct/IMmj8/Ul66xUf3Qf+wAAAABgAazlbnHbquroMX1kkp9I8oXMQqbXj2Y7k3x8TN8w5jOW/1l396ifP+4md0qSU5N8OsltSU4dd4Y7IrOLft8w1llpHwAAAAAsgMNXb5Ljk1wz7ur2d5Jc392fqKp7klxXVb+W5HNJPjTafyjJ71XV7iSPZRYWpbvvrqrrk9yT5OkkF3f3M0lSVW9LclOSw5Jc1d13j229c4V9AAAAALAAVg2XuvuOJC9bpn5fZtdfWlr/6yQ/vcK2Lkty2TL1G5PcuNZ9AAAAALAY1nXNJQAAAACYJ1wCAAAAYDLhEgAAAACTCZcAAAAAmEy4BAAAAMBkwiUAAAAAJhMuAQAAADCZcAkAAACAyYRLAAAAAEwmXAIAAABgMuESAAAAAJMJlwAAAACYTLgEAAAAwGTCJQAAAAAmEy4BAAAAMJlwCQAAAIDJhEsAAAAATCZcAgAAAGAy4RIAAAAAkwmXAAAAAJhMuAQAAADAZMIlAAAAACYTLgEAAAAwmXAJAAAAgMmESwAAAABMJlwCAAAAYLJVw6WqOqmqPlVV91TV3VX1C6P+7qp6qKpuH4/Xza3zrqraXVVfqqpz5uo7Rm13VV0yVz+lqm4d9Q9X1RGj/j1jfvdYfvJGDh4AAACA/bOWI5eeTvLPu/u0JGcmubiqThvLLu/u7eNxY5KMZecneXGSHUl+u6oOq6rDknwwyWuTnJbkTXPbef/Y1o8keTzJhaN+YZLHR/3y0Q4AAACABbFquNTdD3f3Z8f0f07yhSQn7GOVc5Nc193f6u4vJ9md5Izx2N3d93X3t5Ncl+Tcqqokr0nykbH+NUnOm9vWNWP6I0nOGu0BAAAAWADruubSOC3tZUluHaW3VdUdVXVVVR0zaickeWButQdHbaX6C5N8o7ufXlJ/1rbG8idGewAAAAAWwJrDpap6fpKPJvnF7n4yyRVJfjjJ9iQPJ/nNA9LDtfXtoqraVVW79uzZs1ndAAAAANhy1hQuVdXzMguW/nV3/2GSdPfXuvuZ7v5Okt/J7LS3JHkoyUlzq584aivVH01ydFUdvqT+rG2N5T8w2j9Ld1/Z3ad39+nbtm1by5AAAAAA2ABruVtcJflQki9092/N1Y+fa/ZTSe4a0zckOX/c6e2UJKcm+XSS25KcOu4Md0RmF/2+obs7yaeSvH6svzPJx+e2tXNMvz7Jn432AAAAACyAw1dvkh9L8jNJ7qyq20ftlzO729v2JJ3k/iQ/lyTdfXdVXZ/knszuNHdxdz+TJFX1tiQ3JTksyVXdfffY3juTXFdVv5bkc5mFWRk/f6+qdid5LLNACgAAAIAFsWq41N3/Pslyd2i7cR/rXJbksmXqNy63Xnffl++eVjdf/+skP71aHwEAAADYHOu6WxwAAAAAzBMuAQAAADCZcAkAAACAyYRLAAAAAEwmXAIAAABgMuESAAAAAJMJlwAAAACYTLgEAAAAwGTCJQAAAAAmEy4BAAAAMJlwCQAAAIDJhEsAAAAATCZcAgAAAGAy4RIAAAAAkwmXAAAAAJhMuAQAAADAZMIlAAAAACYTLgEAAAAwmXAJAAAAgMmESwAAAABMJlwCAAAAYDLhEgAAAACTCZcAAAAAmEy4BAAAAMBkwiUAAAAAJhMuAQAAADCZcAkAAACAyVYNl6rqpKr6VFXdU1V3V9UvjPoLqurmqrp3/Dxm1KuqPlBVu6vqjqp6+dy2do7291bVzrn6K6rqzrHOB6qq9rUPAAAAABbDWo5cejrJP+/u05KcmeTiqjotySVJPtndpyb55JhPktcmOXU8LkpyRTILipJcmuSVSc5IculcWHRFkrfMrbdj1FfaBwAAAAALYNVwqbsf7u7Pjun/nOQLSU5Icm6Sa0aza5KcN6bPTXJtz9yS5OiqOj7JOUlu7u7HuvvxJDcn2TGWHdXdt3R3J7l2ybaW2wcAAAAAC2Bd11yqqpOTvCzJrUmO6+6Hx6KvJjluTJ+Q5IG51R4ctX3VH1ymnn3sAwAAAIAFsOZwqaqen+SjSX6xu5+cXzaOOOoN7tuz7GsfVXVRVe2qql179uw5kN0AAAAAYM6awqWqel5mwdK/7u4/HOWvjVPaMn4+MuoPJTlpbvUTR21f9ROXqe9rH8/S3Vd29+ndffq2bdvWMiQAAAAANsBa7hZXST6U5Avd/Vtzi25IsveObzuTfHyufsG4a9yZSZ4Yp7bdlOTsqjpmXMj77CQ3jWVPVtWZY18XLNnWcvsAAAAAYAEcvoY2P5bkZ5LcWVW3j9ovJ3lfkuur6sIkX0nyhrHsxiSvS7I7yTeTvDlJuvuxqnpvkttGu/d092Nj+q1Jrk5yZJI/GY/sYx8AAAAALIBVw6Xu/vdJaoXFZy3TvpNcvMK2rkpy1TL1XUleskz90eX2AQAAAMBiWNfd4gAAAABgnnAJAAAAgMmESwAAAABMJlwCAAAAYDLhEgAAAACTCZcAAAAAmEy4BAAAAMBkwiUAAAAAJhMuAQAAADCZcAkAAACAyYRLAAAAAEwmXAIAAABgMuESAAAAAJMJlwAAAACYTLgEAAAAwGTCJQAAAAAmEy4BAAAAMJlwCQAAAIDJhEsAAAAATCZcAgAAAGAy4RIAAAAAkwmXAAAAAJhMuAQAAADAZMIlAAAAACYTLgEAAAAwmXAJAAAAgMmESwAAAABMtmq4VFVXVdUjVXXXXO3dVfVQVd0+Hq+bW/auqtpdVV+qqnPm6jtGbXdVXTJXP6Wqbh31D1fVEaP+PWN+91h+8kYNGgAAAICNsZYjl65OsmOZ+uXdvX08bkySqjotyflJXjzW+e2qOqyqDkvywSSvTXJakjeNtkny/rGtH0nyeJILR/3CJI+P+uWjHQAAAAALZNVwqbv/Islja9zeuUmu6+5vdfeXk+xOcsZ47O7u+7r720muS3JuVVWS1yT5yFj/miTnzW3rmjH9kSRnjfYAAAAALIj9uebS26rqjnHa3DGjdkKSB+baPDhqK9VfmOQb3f30kvqztjWWPzHaAwAAALAgpoZLVyT54STbkzyc5Dc3rEcTVNVFVbWrqnbt2bNnM7sCAAAAsKVMCpe6+2vd/Ux3fyfJ72R22luSPJTkpLmmJ47aSvVHkxxdVYcvqT9rW2P5D4z2y/Xnyu4+vbtP37Zt25QhAQAAADDBpHCpqo6fm/2pJHvvJHdDkvPHnd5OSXJqkk8nuS3JqePOcEdkdtHvG7q7k3wqyevH+juTfHxuWzvH9OuT/NloDwAAAMCCOHy1BlX1B0leneTYqnowyaVJXl1V25N0kvuT/FySdPfdVXV9knuSPJ3k4u5+ZmznbUluSnJYkqu6++6xi3cmua6qfi3J55J8aNQ/lOT3qmp3ZhcUP3+/RwsAAADAhlo1XOruNy1T/tAytb3tL0ty2TL1G5PcuEz9vnz3tLr5+l8n+enV+gcAAADA5tmfu8UBAAAAsMUJlwAAAACYTLgEAAAAwGTCJQAAAAAmEy4BAAAAMJlwCQAAAIDJhEsAAAAATCZcAgAAAGAy4RIAAAAAkwmXAAAAAJhMuAQAAADAZMIlAAAAACYTLgEAAAAwmXAJAAAAgMmESwAAAABMJlwCAAAAYDLhEgAAAACTCZcAAAAAmEy4BAAAAMBkwiUAAAAAJhMuAQAAADCZcAkAAACAyYRLAAAAAEwmXAIAAABgMuESAAAAAJMJlwAAAACYTLgEAAAAwGSrhktVdVVVPVJVd83VXlBVN1fVvePnMaNeVfWBqtpdVXdU1cvn1tk52t9bVTvn6q+oqjvHOh+oqtrXPgAAAABYHGs5cunqJDuW1C5J8snuPjXJJ8d8krw2yanjcVGSK5JZUJTk0iSvTHJGkkvnwqIrkrxlbr0dq+wDAAAAgAWxarjU3X+R5LEl5XOTXDOmr0ly3lz92p65JcnRVXV8knOS3Nzdj3X340luTrJjLDuqu2/p7k5y7ZJtLbcPAAAAABbE1GsuHdfdD4/pryY5bkyfkOSBuXYPjtq+6g8uU9/XPgAAAABYEPt9Qe9xxFFvQF8m76OqLqqqXVW1a8+ePQeyKwAAAADMmRoufW2c0pbx85FRfyjJSXPtThy1fdVPXKa+r338Ld19ZXef3t2nb9u2beKQAAAAAFivqeHSDUn23vFtZ5KPz9UvGHeNOzPJE+PUtpuSnF1Vx4wLeZ+d5Kax7MmqOnPcJe6CJdtabh8AAAAALIjDV2tQVX+Q5NVJjq2qBzO769v7klxfVRcm+UqSN4zmNyZ5XZLdSb6Z5M1J0t2PVdV7k9w22r2nu/deJPytmd2R7sgkfzIe2cc+AAAAAFgQq4ZL3f2mFRadtUzbTnLxCtu5KslVy9R3JXnJMvVHl9sHAAAAAItjvy/oDQAAAMDWJVwCAAAAYDLhEgAAAACTCZcAAAAAmEy4BAAAAMBkwiUAAAAAJhMuAQAAADCZcAkAAACAyYRLAAAAAEwmXAIAAABgMuESAAAAAJMJlwAAAACYTLgEAAAAwGTCJQAAAAAmEy4BAAAAMJlwCQAAAIDJhEsAAAAATCZcAgAAAGAy4RIAAAAAkwmXAAAAAJhMuAQAAADAZMIlAAAAACYTLgEAAAAwmXAJAAAAgMmESwAAAABMJlwCAAAAYDLhEgAAAACT7Ve4VFX3V9WdVXV7Ve0atRdU1c1Vde/4ecyoV1V9oKp2V9UdVfXyue3sHO3vraqdc/VXjO3vHuvW/vQXAAAAgI21EUcu/aPu3t7dp4/5S5J8srtPTfLJMZ8kr01y6nhclOSKZBZGJbk0ySuTnJHk0r2B1Gjzlrn1dmxAfwEAAADYIAfitLhzk1wzpq9Jct5c/dqeuSXJ0VV1fJJzktzc3Y919+NJbk6yYyw7qrtv6e5Ocu3ctgAAAABYAPsbLnWSf1tVn6mqi0btuO5+eEx/NclxY/qEJA/MrfvgqO2r/uAydQAAAAAWxOH7uf6ruvuhqvp7SW6uqi/OL+zurqrez32sagRbFyXJD/7gDx7o3QEAAAAw7NeRS9390Pj5SJKPZXbNpK+NU9oyfj4ymj+U5KS51U8ctX3VT1ymvlw/ruzu07v79G3btu3PkAAAAABYh8nhUlV9X1V9/97pJGcnuSvJDUn23vFtZ5KPj+kbklww7hp3ZpInxulzNyU5u6qOGRfyPjvJTWPZk1V15rhL3AVz2wIAAABgAezPaXHHJfnYLPfJ4Ul+v7v/tKpuS3J9VV2Y5CtJ3jDa35jkdUl2J/lmkjcnSXc/VlXvTXLbaPee7n5sTL81ydVJjkzyJ+MBAAAAwIKYHC51931JXrpM/dEkZy1T7yQXr7Ctq5JctUx9V5KXTO0jAAAAAAfW/t4tDgAAAIAtTLgEAAAAwGTCJQAAAAAmEy4BAAAAMJlwCQAAAIDJhEsAAAAATCZcAgAAAGAy4RIAAAAAkwmXAAAAAJhMuAQAAADAZMIlAAAAACYTLgEAAAAwmXAJAAAAgMmESwAAAABMJlwCAAAAYDLhEgAAAACTCZcAAAAAmEy4BAAAAMBkwiUAAAAAJhMuAQAAADCZcAkAAACAyYRLAAAAAEwmXAIAAABgMuESAAAAAJMdvtkdAODQc/Ilf7zZXWCJ+9/3k5vdBQAADlHCJeCgJ8gAAADYPMIlWAchBgAAADzbwl9zqap2VNWXqmp3VV2y2f0BAAAA4LsW+silqjosyQeT/ESSB5PcVlU3dPc9m9szAID954jYxeP6ZACwfgsdLiU5I8nu7r4vSarquiTnJhEuAQAAbBLh+GISkLNZFj1cOiHJA3PzDyZ55Sb1BQCAQ5x/MAMHM99hi2erBH6LHi6tSVVdlOSiMftUVX1pM/uzgY5N8vXN7sQmMfatayuPfyuPPdna4z/gY6/3H8it7zev/da0lceebO3xb+WxJ1t7/Ma+dW3Z8df7D6mx/9BKCxY9XHooyUlz8yeO2rN095VJrnyuOvVcqapd3X36ZvdjMxj71hx7srXHv5XHnmzt8W/lsSdbe/zGvjXHnmzt8W/lsSdbe/zGvjXHnmzt8W+VsS/63eJuS3JqVZ1SVUckOT/JDZvcJwAAAACGhT5yqbufrqq3JbkpyWFJruruuze5WwAAAAAMCx0uJUl335jkxs3uxyY55E71Wwdj37q28vi38tiTrT3+rTz2ZGuP39i3rq08/q089mRrj9/Yt66tPP4tMfbq7s3uAwAAAAAHqUW/5hIAAAAAC0y4xAFVVc9U1e1V9fmq+mxV/cMV2r27qh4abe+tqj+sqtOWtNleVV1VO8b8x0b73VX1xJi+faV9bJaqemqF+iE55rW+5qPtq6rq01X1xfG4aG7Z/PNzT1W9acm6/2ysc+fY129V1fMO5NjWY6XXfSw7ZMedTP7c31VV/8sy9b2Po6vq1XPv+y9W1W88tyNbuzV+7g+pMe+12utfVW+eG+O3x3v59qp6X1X9bFX9q9Hu3eP770fm1v3FUVvoO64sff0P9TGv8zP/jmXq51XVHVX1hfHcnLdk+TvG+//2qrqtqi44UGOZYp3j/2ZV/b252tL3ynnj9X7RmL91bPuvqmrP3Pvo5AM5pvVa5TvvoBjzWl/HDdjP79aSv/eWabPwn5V1vu+X/n7776vq0ao6aknbf1NVbxzfi3uWrLPP52wzrPK+PyTHPG+V8S/0+3eZvq34d/t+bveQ+LyvWXd7eBywR5Kn5qbPSfLvVmj37iTvmJt/Y5KvJtk2V3t/kr9Mcs2SdV+d5BObPda1PAdbYczreM3/uyR/leTlY/7YJJ9J8pNLn58kpyZ5MsnzxvzPJ/nTJEeP+SOSXJLkqM0e/xpe90N63Ot8D8yP9X9M8vXM/tPjWZ+Nufb/7X2f5MgkX0zyY5s93nW+/ofsmNf7+o/l9yc5dm7+Z5P8q7nn6o4k/3Ju+X9IcleS0zd7nFNe/0N1zFM+83O1lybZneSUMX/KmP/RMf/zmd3Y5agxf1SSnZs95v0Y/18lef9K75UkH87s9/6vLqn/t/fJIj5W+c47KMa8nu+u56AvC/9Z2Z/P/aj//nz/kvxAZr8T/+6iv9+Xew62ypinjn/R3r9rHctztO+D7vla6eHIpQVUsyMT7hqPX9zs/mygo5I8vpaG3f3hJP82yf+aJFVVSX46sy/en6iq7z1Afdw0h+iY9/WaX5zk6u7+bJJ099eT/IvMwpJn6e57k3wzyTGj9CtJ/ml3f2Ms/3Z3v6+7n9zg/h8IW23ca/rcd/cXkjydWdi2qu7+r0luT3LCfvVuE22RMa/5e38F/ybJuUlSVT+c5InM/hA/lB3sY17va/6OJP97d385ScbPX0/yS2P5L2f2vffkWP5kd1+zgf3daKuN/6okb6yqFyxdUFXPT/KqJBcmOf/AdG9THIxjXvF1rKqrq+qKqrqlqu6r2RGmV42jC66ea3dFVe2qqrur6lfn6n9e40jEqnqqqi4bR//cUlXH7aNPi/xZmfJd/wd59mv+U0lu6u5vblivFs9WHPNei/z+XZHP+/oIlxZMVb0iyZuTvDLJmUneUlUv29xe7Zcjx6F6X0zyu0neu451P5vkRWP6Hyb5cnf/pyR/nuQnN7SXi+NQGPNaX/MXZ3bEzrxdo/4sVfXyJPd29yPjcOLn7/2yPQhthXGv+3NfVa9M8p0ke0bp7XOHhX9qmfbHZHZk119sYL+fU4fwmPfne3+pJ5M8UFUvyewP8g9vRAcX3ME45v15zVf8Thzfe9/f3fdtUD8PlPWM/6nMwpZfWGbZuUn+tLv/Y5JHx9+Eh4KDZczreR2PSfIPkrw9yQ1JLs/svfz3q2r7aPMr3X16kh9N8j9X1Y8us53vS3JLd780s+/2t+xjn4v2WVnP87Xc77ebkry8ql445s/PLHzZ64317FPEjtz4IRxQW3HM+7Jo79/18HlfI+HS4nlVko9193/p7qeS/GGSH9/kPu2P/9rd27v7RUl2JLl2HJGzFvPt3pTkujF93Zg/FB0KY96f13ypt1fV3UluTXLZcg2q6pzxC/j+WqBrT+2ng33c63kPvL2qbk/yG0ne2OPY3iSXj21s7+5/NNf+x6vq80keyux/+756wEZx4BzqY97I74Bk9v13fpLzknxsIzp4EDjYxrzRr/nBZr3j/0CSnVX1/UvqB+vv/bU4GMa8ntfxj8Z3951Jvtbdd3b3d5LcneTk0eYNVfXZJJ/L7B+Ky1135dtJPjGmPzO37sFgPc/X3/r91t3fzuwf6q+vqmOTvCyz8GWvD8+ts30cvXsw2YpjPlT5vK+RcInnTHf/f5md/rFtHBJ4+/gH1kpeluQLVXVYkn+c5H+rqvuT/F9JdizzB8pC24pjXuU1vyfJ0v+hfEVmX9R7Xd7dL87sufhQVX3vOPzzqao6Zezjpu7entk1SY44kOOZYquOe681fO73/vH14939l2vY5F+O//F5cZIL5/7HaCFtxTHPm/C9v5xPJPmZJH+14KeA/i1bccwTXvMVvxPnvvf+hwPU3Q23lvGPU5t/P7PTpJMkNTtl7DVJfnf83v+lzP6xclCFdIfKmNfwOn5r/PzO3PTe+cPH7+p3JDmru380yR8nWe7yBn8z9x8MzyQ5fB/dWtjPyn581+89Tez1ST7e3X9zIPt5oGzFMc87lL7rfd73j3Bp8fxlkvOq6u9W1fdldi7uWv7xsfBqdheQw5I82t2/sjeVX6HtP05ydmZfwGcluaO7T+ruk7v7h5J8NLPn5qCxFce8ymv+wSQ/u/cfyuMQ4fcn+T+Wbqe7b8jsUNCdo/TrSa6oqqPHupXlv8Q33VYd917r+dyvxzg98H1J3rm/2zqQtuKY523E69+za1G8MyscxbfItuKYJ7zmv5HkXcbe6rIAAAH1SURBVDXuBDZ+/nKS3xzLfz3JB2vcYamqnl+LcEecFaxj/L+V5Ofy3X9cvD7J73X3D43f+ycl+XIOsqPXD5Uxb8B311FJ/kuSJ2p2XZXXbkC3Fvazsh/P159ndrr3xXn26WEHla045nmH0ne9z/v+2Vdaxibo7s/W7OJgnx6l3+3uz21il/bXkXPJb2V2FftnVmj79qr6J5mdj3pXktd0956a3Yp96WkBH03yT5NceyA6/Rw6FMe8pte8ux8eY/+dcURWJfk/u/uPVtjue5L8flX9TpIrMnvObq2qb2V2PYf/kNmhqAtti4x7PZ/7lez9bOx13jJt/u8k76iqk7v7/gn9XDSHypg34vV/lu6+bvVWh5aDbMzrec3/Zc3drKS7T6yqdyb5o6p6XpK/SfIvunvv9q5I8vwkt1XV34zlv7l0o5ts3e/57v56VX0ss2t4JLPTwd6/pNlHR33Rr7O2JgfBmDfsu6u7P19Vn8vsDp8PZPa7er0W/bMy5W/8vc7r7vu7+ztV9ZEkb0jy75as88aqetXc/Fu7+//dmK4/J7bimOct+vt3w2yRz/ua1HePzAIAAACA9XFaHAAAAACTCZcAAAAAmEy4BAAAAMBkwiUAAAAAJhMuAQAAADCZcAkAAACAyYRLAAAAAEwmXAIAAABgsv8fziUK5hV4Qj8AAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "labels, counts=np.unique(np.array(labels_dist),return_counts=True)\n",
        "label_counts = {}\n",
        "for label, count in zip(labels,counts):\n",
        "    label_counts[label] = count\n",
        "label_counts"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QNC6_rqizSj7",
        "outputId": "fb94218f-3317-4cb9-d443-4d311de3297c"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'B-DAT': 1434,\n",
              " 'B-EVE': 772,\n",
              " 'B-LOC': 2965,\n",
              " 'B-NAT': 598,\n",
              " 'B-ORG': 4622,\n",
              " 'B-PER': 4318,\n",
              " 'B-TIM': 231,\n",
              " 'B-mainLOC': 469,\n",
              " 'I-DAT': 1868,\n",
              " 'I-EVE': 1692,\n",
              " 'I-LOC': 1464,\n",
              " 'I-NAT': 150,\n",
              " 'I-ORG': 7499,\n",
              " 'I-PER': 5070,\n",
              " 'I-TIM': 275,\n",
              " 'I-mainLOC': 275,\n",
              " 'o': 359011}"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "from datasets import Dataset\n",
        "dataset_train = Dataset.from_pandas(data_train)\n",
        "dataset_val = Dataset.from_pandas(data_val)\n",
        "dataset_test = Dataset.from_pandas(data_test)"
      ],
      "metadata": {
        "id": "x59JYDzVDoIC"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### new Way"
      ],
      "metadata": {
        "id": "pVp9K3R-dQNf"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "labels = [\"B-DAT\", \"B-TIM\", \"B-PER\", \"B-ORG\", \"B-LOC\", \"B-EVE\", \"B-NAT\", \"B-mainLOC\", \"I-DAT\", \"I-TIM\", \"I-PER\", \"I-ORG\", \"I-LOC\", \"I-EVE\", \"I-NAT\", \"I-mainLOC\", \"o\"]\n",
        "id2label = {}\n",
        "label2id = {}\n",
        "for i, item in enumerate(labels):\n",
        "    id2label[i] = item\n",
        "    label2id[item] = i\n",
        "id2label"
      ],
      "metadata": {
        "id": "48iDx1KRdSqi",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "fa95015c-5052-4f52-f168-246abcdde86f"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{0: 'B-DAT',\n",
              " 1: 'B-TIM',\n",
              " 2: 'B-PER',\n",
              " 3: 'B-ORG',\n",
              " 4: 'B-LOC',\n",
              " 5: 'B-EVE',\n",
              " 6: 'B-NAT',\n",
              " 7: 'B-mainLOC',\n",
              " 8: 'I-DAT',\n",
              " 9: 'I-TIM',\n",
              " 10: 'I-PER',\n",
              " 11: 'I-ORG',\n",
              " 12: 'I-LOC',\n",
              " 13: 'I-EVE',\n",
              " 14: 'I-NAT',\n",
              " 15: 'I-mainLOC',\n",
              " 16: 'o'}"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "batch_size = 256\n",
        "task = \"ner\" \n",
        "model_name = 'HooshvareLab/bert-fa-zwnj-base'\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_name)"
      ],
      "metadata": {
        "id": "szsav97citAY",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 177,
          "referenced_widgets": [
            "ec84897dc6134b3a9d12232ef2cec79b",
            "9e2c183e18e341e4825e7aadeeffc3b5",
            "6f1322075b0c46b1be5d7fb58569b306",
            "aa51cb0ec204469986c2ca7eeed55811",
            "1c268d420a5c4233a39c8aa66df4627d",
            "2b033acbcbcb45a7b2a3a3d8a91d3691",
            "8e9a772ac2a9420a94b4c28309d531fc",
            "3acb344aebcb464aa8a80f75487f42c2",
            "6064d9daceb94882931d3199e3e5c085",
            "e4163d60c0f6439ca2f1c49309716c59",
            "4761087cf87c49348211d48b62f802d3",
            "27064479d6d647428b50ece45fca966e",
            "aa6dbbbf21124990a67be1fb1d67789b",
            "eff965e7fa604cd685cf37038e938789",
            "94bb7ff233fa462b8ee1493788f1cb72",
            "156e16b9fe004a02abc5551ceb3ac6b1",
            "511fc8f261fb4a33a34eea7b181b630d",
            "38af5fa3fdce472cbf74fb10250ab8e4",
            "f9eede9491e940ca816bf5b1912a2fcb",
            "96ebed29a4ed4d0bab420512ac068706",
            "e9c9acb02f5c427aa4af6db29f27dff7",
            "02434d44cba84ccd8b68098deb54b97a",
            "a703e05e1ba04b1e9581ceb7c410e85c",
            "699c60a97e75430dbe834e8805f0e06a",
            "8b7ba2cd716b4e8a8a2cebf801273b2f",
            "0c2324e7f4bd40a7938ee1a3881568e2",
            "3fe66cf458304a16ae5499759bddcd81",
            "2c8f12dcfeec4fc3a1ce0c1d9f6df5b0",
            "78540b5c098244b2a729700add745d3e",
            "b20e6d56d4c342dfad334cbd9789ccf6",
            "8fefa2796f4e4e96b936046976ff6ae1",
            "cf05e7791bc9442d8bf0fd9c4bde62ee",
            "b569c54d4ebd469d8b870d5386207737",
            "11cbcc7e45a545c5b62496a85b3f6c1e",
            "e85e09290fa94175b2d9cedc776d5a8b",
            "15a7ba6a5561496d95f514365f2c0ff6",
            "0d30deac0d7646e3bcb1c2cedcc0d253",
            "008d84f5b2f64fd0b7da3f33093bf3b1",
            "1b030c1c9a434d0fb43080933c2b6195",
            "be9e409f67ee43288222b48d081034af",
            "8741b486b38b4c5fbab4886f6c631ac3",
            "72565303c9f64bad84fc7683b63bc80d",
            "5c970884b90e4efdb611d5b392156f7d",
            "fc0b9985ed85428cb540a34d53d358cf",
            "37f406906a824680b5d9e5a7a62fa8c6",
            "d745b327cfa7427dbc4fc37ec68d6e54",
            "29672d3c481f4fc8bf7d8f67a33f57cb",
            "ae610e7d90824c8c88c970d0b04732a4",
            "a4f3100df50c406d8b7c66b1c8c7170c",
            "cfcf8486aa6c46aab8c10f93ef247e6c",
            "d8e4fbf375464ab5ad64ab5ee85d40e2",
            "840c1c135ef54b04bd3d1df1885f9a4b",
            "e18e5fb0cfc64ab09ee8a7499603c003",
            "1f076ccc9cd240288d069f0aef71b9bb",
            "f09dee1990004f1e9ceefc38c57005e0"
          ]
        },
        "outputId": "75b67edd-72b1-41c8-d3ea-467bbefaf707"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading tokenizer_config.json:   0%|          | 0.00/292 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "ec84897dc6134b3a9d12232ef2cec79b"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading config.json:   0%|          | 0.00/565 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "27064479d6d647428b50ece45fca966e"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading vocab.txt:   0%|          | 0.00/416k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "a703e05e1ba04b1e9581ceb7c410e85c"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading tokenizer.json:   0%|          | 0.00/1.06M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "11cbcc7e45a545c5b62496a85b3f6c1e"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading special_tokens_map.json:   0%|          | 0.00/134 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "37f406906a824680b5d9e5a7a62fa8c6"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "def tokenize_and_align_labels(examples):\n",
        "    label_all_tokens = True\n",
        "    tokenized_inputs = tokenizer(list(examples[\"token\"]), padding=\"max_length\",max_length= 128, truncation=True, is_split_into_words=True)\n",
        "\n",
        "    labels = []\n",
        "    for i, label in enumerate(examples['labels']):\n",
        "        word_ids = tokenized_inputs.word_ids(batch_index=i)\n",
        "        previous_word_idx = None\n",
        "        label_ids = []\n",
        "        for word_idx in word_ids:\n",
        "            if word_idx is None:\n",
        "                label_ids.append(16)\n",
        "            elif label[word_idx] == '0':\n",
        "                label_ids.append(0)\n",
        "            elif word_idx != previous_word_idx:\n",
        "                label_ids.append(label2id[label[word_idx]])\n",
        "            else:\n",
        "                label_ids.append(label2id[label[word_idx]] if label_all_tokens else 16)\n",
        "            previous_word_idx = word_idx\n",
        "        labels.append(label_ids)\n",
        "    \n",
        "    \n",
        "    tokenized_inputs[\"labels\"] = labels\n",
        "    return tokenized_inputs"
      ],
      "metadata": {
        "id": "5IwMko71jdGG"
      },
      "execution_count": 55,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "dataset_test_tokenized=dataset_test.map(tokenize_and_align_labels,batched=True)\n",
        "dataset_val_tokenized = dataset_val.map(tokenize_and_align_labels,batched=True)\n",
        "dataset_train_tokenized = dataset_train.map(tokenize_and_align_labels,batched=True)"
      ],
      "metadata": {
        "id": "nc4psMCgbYoM",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 113,
          "referenced_widgets": [
            "168d5d51efc44eae89a18253565ccf01",
            "2114b028a15f458bb9326e256397c999",
            "0bb60baaaa674bf6a5e5f0ac33342942",
            "045457c6228c4daf8254358dc0e4f589",
            "836bfcb87f8e41a9b0c094f74cbc0731",
            "10902ca937e9436385e0b17899e617c0",
            "7c9363a004e14789bab5d1bcf9053a60",
            "4a34e319a80646d9a2eb813ad931678b",
            "e9410248e64a485998efe818f5de2891",
            "923195bc7b754378887900213accd871",
            "c8f6be46c946452fa0ccb6b5ce49e059",
            "65a86757e06c429ebabe2abe21b47135",
            "738703125b3f4daabb8cc51d53536909",
            "be7feac07caf4d3384f513227b81a8b0",
            "9f169abcdd9b413db8fc8cf76fa6ea7a",
            "8c76c4a8350d46318931cc886bb8012d",
            "8af3e369f16742f28edd70b4b015301e",
            "0bb78fd9ebc8479c84a3aeaaa90c78ad",
            "7f8e27e17ed4433fbcf091a1da593a04",
            "389477fe499e44d29b80a2c9b81eb88b",
            "b88e63c0cbcb4dcc97e31a02a090a7a3",
            "f701162f7f8c4fd4a972a7634b3e8d04",
            "83ec83544b5d42fdb65d61c7991e7dd6",
            "d686e4f374dd463cb786a92b568e07de",
            "3869383e67164c31864e131579dff169",
            "539291d1047348df8c414e69737cd39c",
            "3f3b3df68173481a9d2c6bc851fc739d",
            "b239b528dffd4c4f96e27e1fc1d9dc95",
            "e4e5fc04433f419dadc9dee2db54c7a8",
            "cacbc753fe6542d9a0e3246289993dbf",
            "ccb07c2bd5fb46de8413ef3bb0f6e5b5",
            "40363b407acf4ef4878ae3414472fb3b",
            "565febd28fc34798befa69fcd44921bf"
          ]
        },
        "outputId": "334f621b-34a3-4d84-bd08-c60660f39993"
      },
      "execution_count": 56,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/1 [00:00<?, ?ba/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "168d5d51efc44eae89a18253565ccf01"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/1 [00:00<?, ?ba/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "65a86757e06c429ebabe2abe21b47135"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/5 [00:00<?, ?ba/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "83ec83544b5d42fdb65d61c7991e7dd6"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#dataset_test_tokenized.remove_columns(\"token\")\n",
        "#dataset_val_tokenized.remove_columns(\"token\")\n",
        "#dataset_train_tokenized.remove_columns(\"token\")"
      ],
      "metadata": {
        "id": "B4tTg1LLE9gm"
      },
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "plt.hist([len(item) for item in iter(dataset_train_tokenized['attention_mask'])])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 356
        },
        "id": "FgI_JrGwBCFp",
        "outputId": "8f553b44-1115-4c99-e28f-e812fc9e65aa"
      },
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(array([   0.,    0.,    0.,    0.,    0., 4050.,    0.,    0.,    0.,\n",
              "           0.]),\n",
              " array([127.5, 127.6, 127.7, 127.8, 127.9, 128. , 128.1, 128.2, 128.3,\n",
              "        128.4, 128.5]),\n",
              " <a list of 10 Patch objects>)"
            ]
          },
          "metadata": {},
          "execution_count": 34
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAUVklEQVR4nO3df5Bdd3nf8fcH+QcEEmTjjWtLmkoJYjKmnQh3I7tD2hJ7YstOJnJmgMhhQKXuKGXsDqQMiZ3O1AnEM7iQuHUHzCixiqAUoTpQaxwFI4wppTP+sQZhJBuPF9tEUoS1QbaJ64kTmad/3K+Yi9jV3t29e9fueb9m7uw5z/mec7/PyP7s3XPPvSdVhSSpG1621BOQJI2OoS9JHWLoS1KHGPqS1CGGviR1yClLPYGTOeuss2r16tVLPQ1Jekl54IEH/rqqxqbb9qIO/dWrVzMxMbHU05Ckl5Qk35lpm6d3JKlDDH1J6hBDX5I6xNCXpA4x9CWpQwx9SeoQQ1+SOsTQl6QOGTj0kyxL8vUkd7T1NUnuTTKZ5DNJTmv109v6ZNu+uu8Y17X6I0kuHXYzkqSTm8snct8NPAz8VFu/EbipqnYk+RhwFXBL+/lUVb02yaY27jeSnAdsAl4PnAt8McnrquqFIfUijdTqa/98SZ73iQ/+ypI8r/7/MNAr/SQrgV8B/rStB7gIuK0N2Q5c0ZY3tnXa9ovb+I3Ajqp6vqoeByaB9cNoQpI0mEFP7/wn4HeAH7T11wBPV9Wxtn4QWNGWVwAHANr2Z9r4H9an2eeHkmxJMpFkYmpqag6tSJJmM2voJ/lV4EhVPTCC+VBVW6tqvKrGx8am/ZI4SdI8DXJO/43AryW5HHg5vXP6/xlYnuSU9mp+JXCojT8ErAIOJjkFeDXwvb76cf37SJJGYNZX+lV1XVWtrKrV9N6I/VJVvQ24G3hzG7YZuL0t72rrtO1fqqpq9U3t6p41wFrgvqF1Ikma1UK+T/93gR1J/hD4OnBrq98KfDLJJHCU3i8Kqmp/kp3AQ8Ax4Gqv3JGk0ZpT6FfVl4Evt+XHmObqm6r6W+AtM+x/A3DDXCcpSRoOP5ErSR1i6EtShxj6ktQhhr4kdYihL0kdYuhLUocY+pLUIYa+JHWIoS9JHWLoS1KHGPqS1CGGviR1iKEvSR1i6EtShxj6ktQhhr4kdYihL0kdMmvoJ3l5kvuSfCPJ/iR/0OofT/J4kr3tsa7Vk+TmJJNJHkxyft+xNid5tD02z/SckqTFMcjtEp8HLqqqZ5OcCnw1yV+0be+rqttOGH8ZvZuerwUuAG4BLkhyJnA9MA4U8ECSXVX11DAakSTNbtZX+tXzbFs9tT3qJLtsBD7R9rsHWJ7kHOBSYE9VHW1BvwfYsLDpS5LmYqBz+kmWJdkLHKEX3Pe2TTe0Uzg3JTm91VYAB/p2P9hqM9VPfK4tSSaSTExNTc2xHUnSyQwU+lX1QlWtA1YC65P8I+A64OeAXwDOBH53GBOqqq1VNV5V42NjY8M4pCSpmdPVO1X1NHA3sKGqDrdTOM8D/xVY34YdAlb17bay1WaqS5JGZJCrd8aSLG/LrwB+GfhWO09PkgBXAPvaLruAd7SreC4Enqmqw8CdwCVJzkhyBnBJq0mSRmSQq3fOAbYnWUbvl8TOqrojyZeSjAEB9gL/po3fDVwOTALPAe8EqKqjST4A3N/Gvb+qjg6vFUnSbGYN/ap6EHjDNPWLZhhfwNUzbNsGbJvjHCVJQ+InciWpQwx9SeoQQ1+SOsTQl6QOMfQlqUMMfUnqEENfkjrE0JekDjH0JalDDH1J6hBDX5I6xNCXpA4x9CWpQwx9SeoQQ1+SOsTQl6QOGeR2iS9Pcl+SbyTZn+QPWn1NknuTTCb5TJLTWv30tj7Ztq/uO9Z1rf5IkksXqylJ0vQGeaX/PHBRVf08sA7Y0O59eyNwU1W9FngKuKqNvwp4qtVvauNIch6wCXg9sAH4aLsFoyRpRGYN/ep5tq2e2h4FXATc1urb6d0cHWBjW6dtv7jdPH0jsKOqnq+qx+ndQ3f9ULqQJA1koHP6SZYl2QscAfYA3waerqpjbchBYEVbXgEcAGjbnwFe01+fZp/+59qSZCLJxNTU1Nw7kiTNaKDQr6oXqmodsJLeq/OfW6wJVdXWqhqvqvGxsbHFehpJ6qQ5Xb1TVU8DdwP/FFie5JS2aSVwqC0fAlYBtO2vBr7XX59mH0nSCAxy9c5YkuVt+RXALwMP0wv/N7dhm4Hb2/Kutk7b/qWqqlbf1K7uWQOsBe4bViOSpNmdMvsQzgG2tyttXgbsrKo7kjwE7Ejyh8DXgVvb+FuBTyaZBI7Su2KHqtqfZCfwEHAMuLqqXhhuO5Kkk5k19KvqQeAN09QfY5qrb6rqb4G3zHCsG4Ab5j5NSdIw+IlcSeoQQ1+SOsTQl6QOMfQlqUMMfUnqEENfkjrE0JekDjH0JalDDH1J6hBDX5I6xNCXpA4x9CWpQwx9SeoQQ1+SOsTQl6QOMfQlqUMMfUnqkEHukbsqyd1JHkqyP8m7W/33kxxKsrc9Lu/b57okk0keSXJpX31Dq00muXZxWpIkzWSQe+QeA95bVV9L8pPAA0n2tG03VdWH+wcnOY/efXFfD5wLfDHJ69rmj9C7sfpB4P4ku6rqoWE0Ikma3SD3yD0MHG7Lf5PkYWDFSXbZCOyoqueBx9sN0o/fS3ey3VuXJDvaWENfkkZkTuf0k6ymd5P0e1vpmiQPJtmW5IxWWwEc6NvtYKvNVD/xObYkmUgyMTU1NZfpSZJmMXDoJ3kV8GfAe6rq+8AtwM8C6+j9JfBHw5hQVW2tqvGqGh8bGxvGISVJzSDn9ElyKr3A/1RVfRagqp7s2/4nwB1t9RCwqm/3la3GSeqSpBEY5OqdALcCD1fVH/fVz+kb9uvAvra8C9iU5PQka4C1wH3A/cDaJGuSnEbvzd5dw2lDkjSIQV7pvxF4O/DNJHtb7feAK5OsAwp4AvgtgKran2QnvTdojwFXV9ULAEmuAe4ElgHbqmr/EHuRJM1ikKt3vgpkmk27T7LPDcAN09R3n2w/SdLi8hO5ktQhhr4kdYihL0kdYuhLUocY+pLUIYa+JHWIoS9JHWLoS1KHGPqS1CGGviR1iKEvSR1i6EtShxj6ktQhhr4kdYihL0kdYuhLUocMcrvEVUnuTvJQkv1J3t3qZybZk+TR9vOMVk+Sm5NMJnkwyfl9x9rcxj+aZPPitSVJms4gr/SPAe+tqvOAC4Grk5wHXAvcVVVrgbvaOsBl9O6LuxbYAtwCvV8SwPXABcB64PrjvygkSaMxa+hX1eGq+lpb/hvgYWAFsBHY3oZtB65oyxuBT1TPPcDydhP1S4E9VXW0qp4C9gAbhtqNJOmk5nROP8lq4A3AvcDZVXW4bfoucHZbXgEc6NvtYKvNVJckjcjAoZ/kVcCfAe+pqu/3b6uqAmoYE0qyJclEkompqalhHFKS1AwU+klOpRf4n6qqz7byk+20De3nkVY/BKzq231lq81U/xFVtbWqxqtqfGxsbC69SJJmMcjVOwFuBR6uqj/u27QLOH4Fzmbg9r76O9pVPBcCz7TTQHcClyQ5o72Be0mrSZJG5JQBxrwReDvwzSR7W+33gA8CO5NcBXwHeGvbthu4HJgEngPeCVBVR5N8ALi/jXt/VR0dSheSpIHMGvpV9VUgM2y+eJrxBVw9w7G2AdvmMkFJ0vD4iVxJ6hBDX5I6xNCXpA4x9CWpQwx9SeoQQ1+SOsTQl6QOMfQlqUMMfUnqEENfkjrE0JekDjH0JalDDH1J6hBDX5I6xNCXpA4x9CWpQwx9SeqQQe6Ruy3JkST7+mq/n+RQkr3tcXnftuuSTCZ5JMmlffUNrTaZ5NrhtyJJms0gr/Q/DmyYpn5TVa1rj90ASc4DNgGvb/t8NMmyJMuAjwCXAecBV7axkqQRGuQeuV9JsnrA420EdlTV88DjSSaB9W3bZFU9BpBkRxv70JxnLEmat4Wc078myYPt9M8ZrbYCONA35mCrzVT/MUm2JJlIMjE1NbWA6UmSTjTf0L8F+FlgHXAY+KNhTaiqtlbVeFWNj42NDeuwkiQGOL0znap68vhykj8B7mirh4BVfUNXthonqUuSRmRer/STnNO3+uvA8St7dgGbkpyeZA2wFrgPuB9Ym2RNktPovdm7a/7TliTNx6yv9JN8GngTcFaSg8D1wJuSrAMKeAL4LYCq2p9kJ703aI8BV1fVC+041wB3AsuAbVW1f+jdSJJOapCrd66cpnzrScbfANwwTX03sHtOs5MkDZWfyJWkDjH0JalDDH1J6hBDX5I6xNCXpA4x9CWpQwx9SeoQQ1+SOsTQl6QOMfQlqUMMfUnqEENfkjrE0JekDjH0JalDDH1J6hBDX5I6xNCXpA6ZNfSTbEtyJMm+vtqZSfYkebT9PKPVk+TmJJNJHkxyft8+m9v4R5NsXpx2JEknM8gr/Y8DG06oXQvcVVVrgbvaOsBl9G6GvhbYAtwCvV8S9O6tewGwHrj++C8KSdLozBr6VfUV4OgJ5Y3A9ra8Hbiir/6J6rkHWJ7kHOBSYE9VHa2qp4A9/PgvEknSIpvvOf2zq+pwW/4ucHZbXgEc6Bt3sNVmqv+YJFuSTCSZmJqamuf0JEnTWfAbuVVVQA1hLsePt7WqxqtqfGxsbFiHlSQx/9B/sp22of080uqHgFV941a22kx1SdIIzTf0dwHHr8DZDNzeV39Hu4rnQuCZdhroTuCSJGe0N3AvaTVJ0gidMtuAJJ8G3gScleQgvatwPgjsTHIV8B3grW34buByYBJ4DngnQFUdTfIB4P427v1VdeKbw5KkRTZr6FfVlTNsuniasQVcPcNxtgHb5jQ7SdJQ+YlcSeoQQ1+SOsTQl6QOMfQlqUMMfUnqEENfkjrE0JekDjH0JalDDH1J6hBDX5I6xNCXpA4x9CWpQwx9SeoQQ1+SOsTQl6QOMfQlqUMWFPpJnkjyzSR7k0y02plJ9iR5tP08o9WT5OYkk0keTHL+MBqQJA1uGK/0f6mq1lXVeFu/FrirqtYCd7V1gMuAte2xBbhlCM8tSZqDxTi9sxHY3pa3A1f01T9RPfcAy5OcswjPL0mawUJDv4AvJHkgyZZWO7uqDrfl7wJnt+UVwIG+fQ+22o9IsiXJRJKJqampBU5PktRv1hujz+IXq+pQkp8G9iT5Vv/GqqokNZcDVtVWYCvA+Pj4nPaVJJ3cgl7pV9Wh9vMI8DlgPfDk8dM27eeRNvwQsKpv95WtJkkakXmHfpJXJvnJ48vAJcA+YBewuQ3bDNzelncB72hX8VwIPNN3GkiSNAILOb1zNvC5JMeP89+r6vNJ7gd2JrkK+A7w1jZ+N3A5MAk8B7xzAc8tSZqHeYd+VT0G/Pw09e8BF09TL+Dq+T6fJGnh/ESuJHWIoS9JHWLoS1KHGPqS1CGGviR1iKEvSR1i6EtShxj6ktQhhr4kdYihL0kdYuhLUocY+pLUIYa+JHWIoS9JHWLoS1KHGPqS1CGGviR1yMhDP8mGJI8kmUxy7aifX5K6bKShn2QZ8BHgMuA84Mok541yDpLUZaN+pb8emKyqx6rq74AdwMYRz0GSOmveN0afpxXAgb71g8AF/QOSbAG2tNVnkzwyorkN01nAXy/1JEbMnkckN476GX+E/84vDf9wpg2jDv1ZVdVWYOtSz2MhkkxU1fhSz2OU7Lkb7Pmlb9Sndw4Bq/rWV7aaJGkERh369wNrk6xJchqwCdg14jlIUmeN9PROVR1Lcg1wJ7AM2FZV+0c5hxF5SZ+emid77gZ7folLVS31HCRJI+InciWpQwx9SeoQQ38WSbYlOZJkX1/tQ0m+leTBJJ9LsrzV35Zkb9/jB0nWzXDcf9uOsT/JfxxVP4NYjJ6TrEtyTxszkWT9KHuazRx7PjXJ9iTfTPJwkutmOOaaJPe2rxz5TLt44UVjkXr+VPualX3t+KeOqp/ZLEa/fce5Ocmzi93DUFSVj5M8gH8OnA/s66tdApzSlm8Ebpxmv38MfHuGY/4S8EXg9Lb+00vd5wh6/gJwWVu+HPjyUvc5356B3wR2tOWfAJ4AVk9zzJ3Aprb8MeBdS93nCHq+HEh7fPrF1PNi9Nu2jwOfBJ5d6h4HefhKfxZV9RXg6Am1L1TVsbZ6D73PG5zoSnpfMzGddwEfrKrn2/GODGm6Q7FIPRfwU2351cBfDWGqQzPHngt4ZZJTgFcAfwd8v3/fJAEuAm5rpe3AFYsz+/kZds9t/93VAPcx/X8nS2Ix+m3fJ/Yh4HcWa97DZugv3L8C/mKa+m/Qe6UzndcB/6z96f+/kvzCos1uccyn5/cAH0pyAPgwcNI/l1+E+nu+Dfi/wGHgL4EPV9XRE8a/Bni6L1AO0vsakpeSufb8Q+20ztuBzy/2JIdoPv1eA+yqqsOjmeLCGfoLkOTfA8eAT51QvwB4rqr2Tbtj7/MRZwIXAu8DdrZXhi96C+j5XcBvV9Uq4LeBWxd1okM0Tc/rgReAc4E1wHuT/MwSTW9RDKHnjwJfqar/vagTHZL59JvkXOAtwH8Z4VQXzNCfpyT/EvhV4G3tT9l+m5j5FS/0XvV9tv0VfB/wA3pf6vSitsCeNwOfbcv/g97/VC96M/T8m8Dnq+rv26m5/0PvvG6/7wHL2+kBeAl95cgCej6+//XAGPDvRjDdBVtAv28AXgtMJnkC+Ikkk6OZ9fwZ+vOQZAO9c3i/VlXPnbDtZcBbmfncNsD/pPdmLkleB5zGi/xb/IbQ818B/6ItXwQ8uhjzHKaT9PyX9HogySvp/cX2rf59W3jcDby5lTYDty/2nBdqIT23bf8auBS4sqp+sPgzXpgF/hv/eVX9g6paXVWr6f2l+9rRzHwBlvqd5Bf7g96r18PA39N7hX4VMEnvK6L3tsfH+sa/CbhnmuP8KTDelk8D/huwD/gacNFS9zmCnn8ReAD4BnAv8E+Wus/59gy8it5fK/uBh4D39R1nN3BuW/4Zem9mTrbxpy91nyPo+Rjw7b79/8NS97mY/Z5w/JfE1Tt+DYMkdYindySpQwx9SeoQQ1+SOsTQl6QOMfQlqUMMfUnqEENfkjrk/wFupWsvWXR3LgAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model = AutoModelForTokenClassification.from_pretrained(model_name, num_labels=len(id2label))\n",
        "model.config.id2label = id2label\n",
        "model.config.label2id = label2id\n",
        "model.config._num_labels = len(id2label)\n",
        "model.config.num_labels = len(label2id)\n",
        "model"
      ],
      "metadata": {
        "id": "5VIQevS52d7a",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "bd059d2f-2e9d-4819-cc7f-0a9a62c2fde7"
      },
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at HooshvareLab/bert-fa-zwnj-base were not used when initializing BertForTokenClassification: ['cls.predictions.decoder.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.weight']\n",
            "- This IS expected if you are initializing BertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForTokenClassification were not initialized from the model checkpoint at HooshvareLab/bert-fa-zwnj-base and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "BertForTokenClassification(\n",
              "  (bert): BertModel(\n",
              "    (embeddings): BertEmbeddings(\n",
              "      (word_embeddings): Embedding(42000, 768, padding_idx=0)\n",
              "      (position_embeddings): Embedding(512, 768)\n",
              "      (token_type_embeddings): Embedding(2, 768)\n",
              "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (encoder): BertEncoder(\n",
              "      (layer): ModuleList(\n",
              "        (0): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (1): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (2): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (3): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (4): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (5): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (6): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (7): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (8): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (9): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (10): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (11): BertLayer(\n",
              "          (attention): BertAttention(\n",
              "            (self): BertSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): BertSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): BertIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): BertOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "  )\n",
              "  (dropout): Dropout(p=0.1, inplace=False)\n",
              "  (classifier): Linear(in_features=768, out_features=17, bias=True)\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# you should freeze all parameters except classifier\n",
        "# otherwise, you will be training the BERT!!\n",
        "for param in model.bert.parameters():\n",
        "    param.requires_grad = False\n",
        "\n",
        "args = TrainingArguments(\n",
        "    f\"test-{task}\",\n",
        "    evaluation_strategy = \"epoch\",\n",
        "    learning_rate=1e-4,\n",
        "    per_device_train_batch_size=batch_size,\n",
        "    per_device_eval_batch_size=batch_size,\n",
        "    num_train_epochs=10,\n",
        "    weight_decay=1e-5,\n",
        ")"
      ],
      "metadata": {
        "id": "8CMSsYmqmtz7"
      },
      "execution_count": 48,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data_collator = DataCollatorForTokenClassification(tokenizer)\n",
        "metric = load_metric(\"seqeval\")"
      ],
      "metadata": {
        "id": "nOcN9I0hm6Vk"
      },
      "execution_count": 49,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def compute_metrics(p):\n",
        "    predictions, labels = p\n",
        "    predictions = np.argmax(predictions, axis=2)\n",
        "\n",
        "    true_predictions = [[labels[p] for (p, l) in zip(prediction, label) if l != -100] for prediction, label in zip(predictions, labels)]\n",
        "    true_labels = [[labels[l] for (p, l) in zip(prediction, label) if l != -100] for prediction, label in zip(predictions, labels)]\n",
        "\n",
        "    results = metric.compute(predictions=true_predictions, references=true_labels)\n",
        "    return {\"precision\": results[\"overall_precision\"], \"recall\": results[\"overall_recall\"], \"f1\": results[\"overall_f1\"], \"accuracy\": results[\"overall_accuracy\"]}\n",
        "    "
      ],
      "metadata": {
        "id": "u7GxJMfDnEfG"
      },
      "execution_count": 50,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "dataset_train_tokenized"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oXb1YNJRSI_9",
        "outputId": "3587bfb6-654e-41d6-a50c-43f0df9ef148"
      },
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Dataset({\n",
              "    features: ['token', 'labels', 'input_ids', 'token_type_ids', 'attention_mask'],\n",
              "    num_rows: 4050\n",
              "})"
            ]
          },
          "metadata": {},
          "execution_count": 39
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "class NERDataset(torch.utils.data.Dataset):\n",
        "    def __init__(self, dataset):\n",
        "        self.labels = torch.tensor(dataset['labels'])\n",
        "        self.input_ids = torch.tensor(dataset['input_ids'])\n",
        "        self.token_type_ids = torch.tensor(dataset['token_type_ids'])\n",
        "        self.attention_mask = torch.tensor(dataset['attention_mask'])\n",
        "\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        item = {'labels': self.labels[idx], \n",
        "                'input_ids': self.input_ids[idx],\n",
        "                'token_type_ids': self.token_type_ids[idx],\n",
        "                'attention_mask': self.attention_mask[idx]}\n",
        "        return item\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.labels)"
      ],
      "metadata": {
        "id": "PpsohPxJQa2X"
      },
      "execution_count": 51,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "train_dataset = NERDataset(dataset_train_tokenized)\n",
        "val_dataset = NERDataset(dataset_val_tokenized)\n",
        "test_dataset = NERDataset(dataset_test_tokenized)"
      ],
      "metadata": {
        "id": "p1eV9x6CNI9F"
      },
      "execution_count": 57,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "Clk-tpOHU7BI"
      },
      "execution_count": 41,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install -Uqq pytorch_lightning"
      ],
      "metadata": {
        "id": "StLrobBP4PxI"
      },
      "execution_count": 42,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import pytorch_lightning as pl\n",
        "import torch.nn.functional as F\n",
        "from torch.utils.data import DataLoader\n",
        "import torchmetrics\n",
        "import torch\n",
        "import torchmetrics.functional as TF\n",
        "from torchmetrics import ConfusionMatrix\n",
        "\n",
        "# important\n",
        "from torch.utils.data import WeightedRandomSampler\n",
        "\n",
        "class LiteModel(pl.LightningModule):\n",
        "    def __init__(self,model, train_dataset, val_dataset, test_dataset, batch_size,num_classes):\n",
        "        super(LiteModel, self).__init__()\n",
        "        self.model = model\n",
        "        self.train_dataset = train_dataset\n",
        "        self.val_dataset = val_dataset\n",
        "        self.test_dataset = test_dataset\n",
        "        self.batch_size = batch_size\n",
        "        self.num_classes = num_classes\n",
        "\n",
        "        self.train_acc = torchmetrics.Accuracy()\n",
        "        self.train_f1 = torchmetrics.F1Score(num_classes=num_classes,mdmc_reduce='global')\n",
        "        self.train_auroc = torchmetrics.AUROC(num_classes=num_classes,mdmc_reduce='global')\n",
        "\n",
        "        self.val_acc = torchmetrics.Accuracy()\n",
        "        self.val_f1 = torchmetrics.F1Score(num_classes=num_classes)\n",
        "        self.val_auroc = torchmetrics.AUROC(num_classes=num_classes)\n",
        "        \n",
        "        self.test_acc = torchmetrics.Accuracy()\n",
        "        self.test_f1= torchmetrics.F1Score(num_classes=num_classes)\n",
        "        self.test_auroc = torchmetrics.AUROC(num_classes=num_classes)\n",
        "\n",
        "        self.confusion_matrix = ConfusionMatrix(num_classes=num_classes)\n",
        "\n",
        "\n",
        "    def forward(self, x):\n",
        "        return model(**x)\n",
        "\n",
        "    def configure_optimizers(self):\n",
        "        optimizer = torch.optim.Adam(self.parameters(), lr=1e-3)\n",
        "        lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,patience=2,verbose=True)\n",
        "        return {\n",
        "            'optimizer': optimizer,\n",
        "            'lr_scheduler': lr_scheduler,\n",
        "            'monitor': 'val_loss'\n",
        "        }\n",
        "\n",
        "    def training_step(self, batch, batch_idx):\n",
        "        output= self(batch)\n",
        "\n",
        "        softmax = output.logits.softmax(dim=1)\n",
        "        y_pred = torch.argmax(softmax, dim=1)\n",
        "        y_target = batch['labels'][:,0:y_pred.shape[1]]\n",
        "\n",
        "        acc = self.train_acc(y_pred, y_target)\n",
        "        f1 = self.train_f1(y_pred, y_target)\n",
        "        self.train_auroc.update(softmax, y_target)\n",
        "\n",
        "        self.log(\"train_loss\", output.loss,prog_bar=True)\n",
        "        self.log(\"train_accuracy\", acc,prog_bar=True)\n",
        "        self.log(\"train_f1\", f1,prog_bar=True)\n",
        "\n",
        "        return {'loss': output.loss}\n",
        "\n",
        "    def validation_step(self, batch, batch_idx):\n",
        "        output = self(batch)        \n",
        "\n",
        "        softmax = output.logits.softmax(dim=1)\n",
        "        y_pred = torch.argmax(softmax, dim=1)\n",
        "        y_target = batch['labels'][:,0:y_pred.shape[1]]\n",
        "        self.val_acc.update(y_pred, y_target)\n",
        "        self.val_f1.update(y_pred, y_target)\n",
        "        self.val_auroc.update(softmax, y_target)\n",
        "\n",
        "        return output.loss\n",
        "\n",
        "    def test_step(self, batch, batch_idx):\n",
        "        output = self(batch)        \n",
        "\n",
        "        softmax = output.logits.softmax(dim=1)\n",
        "        y_pred = torch.argmax(softmax, dim=1)\n",
        "       \n",
        "        y_target = batch['labels'][:,0:y_pred.shape[1]]\n",
        "\n",
        "        self.test_acc.update(y_pred, y_target)\n",
        "        self.test_f1.update(y_pred, y_target)\n",
        "        self.test_auroc.update(softmax, y_target)\n",
        "\n",
        "        matrix=self.confusion_matrix(y_pred, y_target)\n",
        "\n",
        "        return {'loss': output.loss,\n",
        "                'confusion_matrix': matrix }\n",
        "\n",
        "    def training_epoch_end(self, training_step_outputs):\n",
        "        train_loss = torch.tensor([item['loss'] for item in training_step_outputs]).mean()\n",
        "        train_accuracy = self.train_acc.compute()\n",
        "        train_f1 = self.train_f1.compute()\n",
        "        train_auroc = self.train_auroc.compute()\n",
        "\n",
        "        self.log(\"epoch_train_accuracy\", train_accuracy)\n",
        "        self.log(\"epoch_train_f1\", train_f1)\n",
        "\n",
        "        self.train_acc.reset()\n",
        "        self.train_f1.reset()\n",
        "        self.train_auroc.reset()\n",
        "        print(f\"\\ntraining accuracy: {train_accuracy:.4}, \"\\\n",
        "        f\"f1: {train_f1:.4}, auroc: {train_auroc:.4}\")\n",
        "\n",
        "    def validation_epoch_end(self, validation_step_outputs):\n",
        "        val_loss = torch.tensor(validation_step_outputs).mean()\n",
        "        val_accuracy = self.val_acc.compute()\n",
        "        val_f1 = self.val_f1.compute()\n",
        "        val_auroc = self.val_auroc.compute()\n",
        "        \n",
        "        self.log(\"val_accuracy\", val_accuracy)\n",
        "        self.log(\"val_loss\", val_loss)\n",
        "        self.log(\"val_f1\", val_f1)\n",
        "        self.log(\"val_auroc\", val_auroc)\n",
        "       \n",
        "        self.val_acc.reset()\n",
        "        self.val_f1.reset()\n",
        "        self.val_auroc.reset()\n",
        "        print(f\"\\nvalidation accuracy: {val_accuracy:.4} \"\\\n",
        "        f\"f1: {val_f1:.4}, auroc: {val_auroc:.4}\")\n",
        "\n",
        "        return {'val_loss': val_loss}\n",
        "    \n",
        "    def test_epoch_end(self, outputs) -> None:\n",
        "        loss = [item['loss'] for item in outputs]\n",
        "\n",
        "        test_loss = torch.tensor(loss).mean()\n",
        "        test_accuracy = self.test_acc.compute()\n",
        "        test_f1 = self.test_f1.compute()\n",
        "        test_auroc = self.test_auroc.compute()\n",
        "        \n",
        "        self.log(\"test_accuracy\", test_accuracy)\n",
        "        self.log(\"test_loss\", test_loss)\n",
        "        self.log(\"test_f1\", test_f1)\n",
        "        self.log(\"test_auroc\", test_auroc)\n",
        "       \n",
        "        self.test_acc.reset()\n",
        "        self.test_f1.reset()\n",
        "        self.test_auroc.reset()\n",
        "        print(f\"\\ntest accuracy: {test_accuracy:.4} \"\\\n",
        "        f\"f1: {test_f1:.4}, auroc: {test_auroc:.4}\")\n",
        "\n",
        "        final_confusion_matrix = torch.zeros(self.num_classes, self.num_classes,device=self.device)    \n",
        "        for item in outputs:\n",
        "            final_confusion_matrix += item['confusion_matrix']\n",
        "        \n",
        "        self.final_matrix = final_confusion_matrix\n",
        "\n",
        "    def train_dataloader(self)-> DataLoader:\n",
        "        return DataLoader(dataset= self.train_dataset, \n",
        "                          batch_size=self.batch_size)\n",
        "        len_dataset = len(self.train_dataset)\n",
        "        bincount=torch.tensor(self.train_dataset.labels).bincount()\n",
        "        dataset_weight=bincount/len_dataset\n",
        "        sampler_weight = 1/dataset_weight\n",
        "        weights = [sampler_weight[label].item() for label in self.train_dataset.labels]\n",
        "        sampler=WeightedRandomSampler(weights,num_samples=len_dataset)\n",
        "        return DataLoader(dataset = self.train_dataset,\n",
        "                          batch_size = self.batch_size,\n",
        "                          sampler=sampler)\n",
        "\n",
        "    def val_dataloader(self) -> DataLoader:\n",
        "        return DataLoader(dataset = self.val_dataset,\n",
        "                          batch_size=self.batch_size)\n",
        "\n",
        "    def test_dataloader(self) -> DataLoader:\n",
        "        return DataLoader(dataset= self.test_dataset,\n",
        "                          batch_size=self.batch_size)"
      ],
      "metadata": {
        "id": "dK-YN-VQ4Er8"
      },
      "execution_count": 72,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from pytorch_lightning import Trainer\n",
        "from pytorch_lightning.callbacks import ModelCheckpoint\n",
        "\n",
        "BATCH_SIZE = 32\n",
        "NUM_CLASSES= len(id2label)\n",
        "net = LiteModel(model, train_dataset, val_dataset, test_dataset, BATCH_SIZE,NUM_CLASSES)\n",
        "\n",
        "checkpoint_callback = ModelCheckpoint(dirpath='/content/models')\n",
        "\n",
        "\n",
        "trainer = Trainer(accelerator='auto',\n",
        "                  devices='auto',\n",
        "                  max_epochs=10,\n",
        "                  overfit_batches=1,\n",
        "                  log_every_n_steps=1,\n",
        "                  callbacks=[checkpoint_callback])\n",
        "\n",
        "\n",
        "#trainer = Trainer(\n",
        "#   net,\n",
        "#    args,\n",
        "#    train_dataset=train_dataloader,\n",
        "#    eval_dataset= dataset_val_tokenized,\n",
        "#    data_collator=data_collator,\n",
        "#    tokenizer=tokenizer,\n",
        "#    compute_metrics=compute_metrics\n",
        "#)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nKZmg5Os4XU1",
        "outputId": "89b28787-59bd-43b4-899c-17b47aa84812"
      },
      "execution_count": 73,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/torchmetrics/utilities/prints.py:36: UserWarning: Metric `AUROC` will save all targets and predictions in buffer. For large datasets this may lead to large memory footprint.\n",
            "  warnings.warn(*args, **kwargs)\n",
            "GPU available: False, used: False\n",
            "TPU available: False, using: 0 TPU cores\n",
            "IPU available: False, using: 0 IPUs\n",
            "HPU available: False, using: 0 HPUs\n",
            "`Trainer(overfit_batches=1)` was configured so 1 batch will be used.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "trainer.fit(model=net)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 734,
          "referenced_widgets": [
            "421389d8df1a48b7b7a23226a0773959",
            "9d3aefb8d80640f399313798543dcb87",
            "1c9cc2c992734c03afde9c252b8c3d1e",
            "5b5da2aa2de540e0bdb80782a320aab3",
            "b13651c05aa4489a951d25315f94829f",
            "808c2845d7c946dda19b983162faf3e7",
            "cf57d12ee4ac4491a911ecdb075fc93b",
            "dd310b02826041bf9506f6abfdb5c863",
            "565e9d4f1a134d0a8f8fd8412b6af346",
            "fd3111e8d3bb43f88d9107a16d852f17",
            "92570bb14be545349604e86283a37c56"
          ]
        },
        "id": "eUHynv1ynM7r",
        "outputId": "691f4770-f5be-42bb-94f0-9d42dc288eef"
      },
      "execution_count": 74,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Missing logger folder: /content/lightning_logs\n",
            "\n",
            "   | Name             | Type                       | Params\n",
            "-----------------------------------------------------------------\n",
            "0  | model            | BertForTokenClassification | 117 M \n",
            "1  | train_acc        | Accuracy                   | 0     \n",
            "2  | train_f1         | F1Score                    | 0     \n",
            "3  | train_auroc      | AUROC                      | 0     \n",
            "4  | val_acc          | Accuracy                   | 0     \n",
            "5  | val_f1           | F1Score                    | 0     \n",
            "6  | val_auroc        | AUROC                      | 0     \n",
            "7  | test_acc         | Accuracy                   | 0     \n",
            "8  | test_f1          | F1Score                    | 0     \n",
            "9  | test_auroc       | AUROC                      | 0     \n",
            "10 | confusion_matrix | ConfusionMatrix            | 0     \n",
            "-----------------------------------------------------------------\n",
            "13.1 K    Trainable params\n",
            "117 M     Non-trainable params\n",
            "117 M     Total params\n",
            "470.879   Total estimated model params size (MB)\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Training: 0it [00:00, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "421389d8df1a48b7b7a23226a0773959"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "error",
          "ename": "ValueError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-74-f04d79ae3318>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnet\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pytorch_lightning/trainer/trainer.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, model, train_dataloaders, val_dataloaders, datamodule, ckpt_path)\u001b[0m\n\u001b[1;32m    769\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstrategy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    770\u001b[0m         self._call_and_handle_interrupt(\n\u001b[0;32m--> 771\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_fit_impl\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_dataloaders\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_dataloaders\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdatamodule\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mckpt_path\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    772\u001b[0m         )\n\u001b[1;32m    773\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pytorch_lightning/trainer/trainer.py\u001b[0m in \u001b[0;36m_call_and_handle_interrupt\u001b[0;34m(self, trainer_fn, *args, **kwargs)\u001b[0m\n\u001b[1;32m    721\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstrategy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlauncher\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlaunch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrainer_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrainer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    722\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 723\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mtrainer_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    724\u001b[0m         \u001b[0;31m# TODO: treat KeyboardInterrupt as BaseException (delete the code below) in v1.7\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    725\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mKeyboardInterrupt\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mexception\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pytorch_lightning/trainer/trainer.py\u001b[0m in \u001b[0;36m_fit_impl\u001b[0;34m(self, model, train_dataloaders, val_dataloaders, datamodule, ckpt_path)\u001b[0m\n\u001b[1;32m    809\u001b[0m             \u001b[0mckpt_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel_provided\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel_connected\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlightning_module\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    810\u001b[0m         )\n\u001b[0;32m--> 811\u001b[0;31m         \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_run\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mckpt_path\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mckpt_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    812\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    813\u001b[0m         \u001b[0;32massert\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstopped\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pytorch_lightning/trainer/trainer.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, model, ckpt_path)\u001b[0m\n\u001b[1;32m   1234\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_checkpoint_connector\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mresume_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1235\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1236\u001b[0;31m         \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_run_stage\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1237\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1238\u001b[0m         \u001b[0mlog\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdetail\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"{self.__class__.__name__}: trainer tearing down\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pytorch_lightning/trainer/trainer.py\u001b[0m in \u001b[0;36m_run_stage\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1321\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredicting\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1322\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_run_predict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1323\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_run_train\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1324\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1325\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_pre_training_routine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pytorch_lightning/trainer/trainer.py\u001b[0m in \u001b[0;36m_run_train\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1351\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_loop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1352\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_detect_anomaly\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_detect_anomaly\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1353\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_loop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1354\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1355\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_run_evaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0m_EVALUATE_OUTPUT\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pytorch_lightning/loops/base.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    202\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    203\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_advance_start\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 204\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madvance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    205\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_advance_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    206\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_restarting\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pytorch_lightning/loops/fit_loop.py\u001b[0m in \u001b[0;36madvance\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    264\u001b[0m         )\n\u001b[1;32m    265\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprofiler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprofile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"run_training_epoch\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 266\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_outputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mepoch_loop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_data_fetcher\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    267\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    268\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mon_advance_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pytorch_lightning/loops/base.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    202\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    203\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_advance_start\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 204\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madvance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    205\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_advance_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    206\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_restarting\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pytorch_lightning/loops/epoch/training_epoch_loop.py\u001b[0m in \u001b[0;36madvance\u001b[0;34m(self, data_fetcher)\u001b[0m\n\u001b[1;32m    206\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    207\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprofiler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprofile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"run_training_batch\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 208\u001b[0;31m                 \u001b[0mbatch_output\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatch_loop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_idx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    209\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    210\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbatch_progress\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mincrement_processed\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pytorch_lightning/loops/base.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    202\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    203\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_advance_start\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 204\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madvance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    205\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_advance_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    206\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_restarting\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pytorch_lightning/loops/batch/training_batch_loop.py\u001b[0m in \u001b[0;36madvance\u001b[0;34m(self, batch, batch_idx)\u001b[0m\n\u001b[1;32m     86\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlightning_module\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautomatic_optimization\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     87\u001b[0m             \u001b[0moptimizers\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_get_active_optimizers\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizers\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizer_frequencies\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_idx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 88\u001b[0;31m             \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizer_loop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msplit_batch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizers\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_idx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     89\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     90\u001b[0m             \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmanual_loop\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msplit_batch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_idx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pytorch_lightning/loops/base.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    202\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    203\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_advance_start\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 204\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madvance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    205\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_advance_end\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    206\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_restarting\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pytorch_lightning/loops/optimization/optimizer_loop.py\u001b[0m in \u001b[0;36madvance\u001b[0;34m(self, batch, *args, **kwargs)\u001b[0m\n\u001b[1;32m    205\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_batch_idx\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    206\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_optimizers\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptim_progress\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizer_position\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 207\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizer_idx\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    208\u001b[0m         )\n\u001b[1;32m    209\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloss\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pytorch_lightning/loops/optimization/optimizer_loop.py\u001b[0m in \u001b[0;36m_run_optimization\u001b[0;34m(self, split_batch, batch_idx, optimizer, opt_idx)\u001b[0m\n\u001b[1;32m    254\u001b[0m         \u001b[0;31m# gradient update with accumulated gradients\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    255\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 256\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_optimizer_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mopt_idx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_idx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclosure\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    257\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    258\u001b[0m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclosure\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconsume_result\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pytorch_lightning/loops/optimization/optimizer_loop.py\u001b[0m in \u001b[0;36m_optimizer_step\u001b[0;34m(self, optimizer, opt_idx, batch_idx, train_step_and_backward_closure)\u001b[0m\n\u001b[1;32m    376\u001b[0m             \u001b[0mon_tpu\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0maccelerator\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mTPUAccelerator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    377\u001b[0m             \u001b[0musing_native_amp\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mamp_backend\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mAMPType\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mNATIVE\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 378\u001b[0;31m             \u001b[0musing_lbfgs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mis_lbfgs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    379\u001b[0m         )\n\u001b[1;32m    380\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pytorch_lightning/trainer/trainer.py\u001b[0m in \u001b[0;36m_call_lightning_module_hook\u001b[0;34m(self, hook_name, pl_module, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1593\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1594\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprofiler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprofile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"[LightningModule]{pl_module.__class__.__name__}.{hook_name}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1595\u001b[0;31m             \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1596\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1597\u001b[0m         \u001b[0;31m# restore current_fx when nested context\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pytorch_lightning/core/lightning.py\u001b[0m in \u001b[0;36moptimizer_step\u001b[0;34m(self, epoch, batch_idx, optimizer, optimizer_idx, optimizer_closure, on_tpu, using_native_amp, using_lbfgs)\u001b[0m\n\u001b[1;32m   1644\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1645\u001b[0m         \"\"\"\n\u001b[0;32m-> 1646\u001b[0;31m         \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclosure\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moptimizer_closure\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1647\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1648\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0moptimizer_zero_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_idx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mOptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer_idx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pytorch_lightning/core/optimizer.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, closure, **kwargs)\u001b[0m\n\u001b[1;32m    166\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    167\u001b[0m         \u001b[0;32massert\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_strategy\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 168\u001b[0;31m         \u001b[0mstep_output\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_strategy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizer_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_optimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_optimizer_idx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclosure\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    169\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    170\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_on_after_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pytorch_lightning/strategies/strategy.py\u001b[0m in \u001b[0;36moptimizer_step\u001b[0;34m(self, optimizer, opt_idx, closure, model, **kwargs)\u001b[0m\n\u001b[1;32m    191\u001b[0m         \"\"\"\n\u001b[1;32m    192\u001b[0m         \u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlightning_module\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 193\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprecision_plugin\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizer_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mopt_idx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclosure\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    194\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    195\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_setup_model_and_optimizers\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mModule\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizers\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mList\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mOptimizer\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mTuple\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mModule\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mList\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mOptimizer\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pytorch_lightning/plugins/precision/precision_plugin.py\u001b[0m in \u001b[0;36moptimizer_step\u001b[0;34m(self, model, optimizer, optimizer_idx, closure, **kwargs)\u001b[0m\n\u001b[1;32m    153\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpl\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mLightningModule\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    154\u001b[0m             \u001b[0mclosure\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpartial\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_wrap_closure\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer_idx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclosure\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 155\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclosure\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mclosure\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    156\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    157\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_track_grad_norm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrainer\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m\"pl.Trainer\"\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/optim/optimizer.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    107\u001b[0m                 \u001b[0mprofile_name\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"Optimizer.step#{}.step\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__class__\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    108\u001b[0m                 \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprofiler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrecord_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprofile_name\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 109\u001b[0;31m                     \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    110\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    111\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/autograd/grad_mode.py\u001b[0m in \u001b[0;36mdecorate_context\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     25\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mdecorate_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     26\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclone\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 27\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     28\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mcast\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mF\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdecorate_context\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     29\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/optim/adam.py\u001b[0m in \u001b[0;36mstep\u001b[0;34m(self, closure)\u001b[0m\n\u001b[1;32m    116\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mclosure\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    117\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menable_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 118\u001b[0;31m                 \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclosure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    119\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    120\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mgroup\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparam_groups\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pytorch_lightning/plugins/precision/precision_plugin.py\u001b[0m in \u001b[0;36m_wrap_closure\u001b[0;34m(self, model, optimizer, optimizer_idx, closure)\u001b[0m\n\u001b[1;32m    138\u001b[0m         \u001b[0mconsistent\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mthe\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0mPrecisionPlugin\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m \u001b[0msubclasses\u001b[0m \u001b[0mthat\u001b[0m \u001b[0mcannot\u001b[0m \u001b[0;32mpass\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclosure\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m \u001b[0mdirectly\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    139\u001b[0m         \"\"\"\n\u001b[0;32m--> 140\u001b[0;31m         \u001b[0mclosure_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclosure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    141\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_after_closure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer_idx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    142\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mclosure_result\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pytorch_lightning/loops/optimization/optimizer_loop.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    146\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    147\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mAny\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mAny\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mOptional\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 148\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclosure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    149\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_result\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    150\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pytorch_lightning/loops/optimization/optimizer_loop.py\u001b[0m in \u001b[0;36mclosure\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    132\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    133\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mclosure\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mAny\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mAny\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mClosureResult\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 134\u001b[0;31m         \u001b[0mstep_output\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_step_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    135\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    136\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mstep_output\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclosure_loss\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pytorch_lightning/loops/optimization/optimizer_loop.py\u001b[0m in \u001b[0;36m_training_step\u001b[0;34m(self, split_batch, batch_idx, opt_idx)\u001b[0m\n\u001b[1;32m    425\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    426\u001b[0m         \u001b[0;31m# manually capture logged metrics\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 427\u001b[0;31m         \u001b[0mtraining_step_output\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_strategy_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"training_step\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0mstep_kwargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    428\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrainer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstrategy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpost_training_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    429\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pytorch_lightning/trainer/trainer.py\u001b[0m in \u001b[0;36m_call_strategy_hook\u001b[0;34m(self, hook_name, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1763\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1764\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprofiler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprofile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf\"[Strategy]{self.strategy.__class__.__name__}.{hook_name}\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1765\u001b[0;31m             \u001b[0moutput\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1766\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1767\u001b[0m         \u001b[0;31m# restore current_fx when nested context\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/pytorch_lightning/strategies/strategy.py\u001b[0m in \u001b[0;36mtraining_step\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    331\u001b[0m         \"\"\"\n\u001b[1;32m    332\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprecision_plugin\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_step_context\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 333\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtraining_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    334\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    335\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mpost_training_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-72-20c363b95974>\u001b[0m in \u001b[0;36mtraining_step\u001b[0;34m(self, batch, batch_idx)\u001b[0m\n\u001b[1;32m     54\u001b[0m         \u001b[0my_target\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbatch\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'labels'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0my_pred\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     55\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 56\u001b[0;31m         \u001b[0macc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_acc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_pred\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_target\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     57\u001b[0m         \u001b[0mf1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_f1\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my_pred\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_target\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     58\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_auroc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msoftmax\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_target\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1128\u001b[0m         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m   1129\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1130\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1131\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1132\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torchmetrics/metric.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    243\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_cache\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_full_state_update\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    244\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 245\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_cache\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_reduce_state_update\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    246\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    247\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_cache\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torchmetrics/metric.py\u001b[0m in \u001b[0;36m_forward_reduce_state_update\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    307\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    308\u001b[0m         \u001b[0;31m# calculate batch state and compute batch value\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 309\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    310\u001b[0m         \u001b[0mbatch_val\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcompute\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    311\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torchmetrics/metric.py\u001b[0m in \u001b[0;36mwrapped_func\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    389\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_grad_enabled\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_enable_grad\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    390\u001b[0m                 \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 391\u001b[0;31m                     \u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    392\u001b[0m                 \u001b[0;32mexcept\u001b[0m \u001b[0mRuntimeError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0merr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    393\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0;34m\"Expected all tensors to be on\"\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mstr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0merr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torchmetrics/classification/accuracy.py\u001b[0m in \u001b[0;36mupdate\u001b[0;34m(self, preds, target)\u001b[0m\n\u001b[1;32m    246\u001b[0m                 \u001b[0mmulticlass\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmulticlass\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    247\u001b[0m                 \u001b[0mignore_index\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mignore_index\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 248\u001b[0;31m                 \u001b[0mmode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    249\u001b[0m             )\n\u001b[1;32m    250\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torchmetrics/functional/classification/accuracy.py\u001b[0m in \u001b[0;36m_accuracy_update\u001b[0;34m(preds, target, reduce, mdmc_reduce, threshold, num_classes, top_k, multiclass, ignore_index, mode)\u001b[0m\n\u001b[1;32m    115\u001b[0m         \u001b[0mmulticlass\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmulticlass\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    116\u001b[0m         \u001b[0mignore_index\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mignore_index\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 117\u001b[0;31m         \u001b[0mmode\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    118\u001b[0m     )\n\u001b[1;32m    119\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mtp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torchmetrics/functional/classification/stat_scores.py\u001b[0m in \u001b[0;36m_stat_scores_update\u001b[0;34m(preds, target, reduce, mdmc_reduce, num_classes, top_k, threshold, multiclass, ignore_index, mode)\u001b[0m\n\u001b[1;32m    171\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mmdmc_reduce\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    172\u001b[0m             raise ValueError(\n\u001b[0;32m--> 173\u001b[0;31m                 \u001b[0;34m\"When your inputs are multi-dimensional multi-class, you have to set the `mdmc_reduce` parameter\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    174\u001b[0m             )\n\u001b[1;32m    175\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mmdmc_reduce\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"global\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mValueError\u001b[0m: When your inputs are multi-dimensional multi-class, you have to set the `mdmc_reduce` parameter"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#trainer.evaluate()"
      ],
      "metadata": {
        "id": "yx1kco6Xm89D"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "trainer.save_model('un-ner.model')"
      ],
      "metadata": {
        "id": "lUhxnYGRoe28"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer = AutoTokenizer.from_pretrained('./un-ner.model/')"
      ],
      "metadata": {
        "id": "TbWyF-41oh54"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "paragraph = '''مدیرکل محیط سازمان زیست استان البرز با بیان اینکه با بیان اینکه موضوع شیرها به های زباله های انتقال یافته در منطقه موقعیت حلقه موقعیت دره موقعیت خطری برای این استان است ، گفت : در این مورد گزارشاتی در ۲۵تاریخ مردادتاریخ ۱۳۹۷تاریخ تقدیم مدیران استان شده است .'''\n",
        "tokens = tokenizer(paragraph)\n",
        "torch.tensor(tokens['input_ids']).unsqueeze(0).size()"
      ],
      "metadata": {
        "id": "L1cJR_NEox0P"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = AutoModelForTokenClassification.from_pretrained('./un-ner.model/')\n",
        "predictions = model.forward(input_ids=torch.tensor(tokens['input_ids']).unsqueeze(0), attention_mask=torch.tensor(tokens['attention_mask']).unsqueeze(0))\n",
        "predictions = torch.argmax(predictions.logits.squeeze(), axis=1)\n",
        "predictions = [labels[i] for i in predictions]"
      ],
      "metadata": {
        "id": "GirV-ahxpJI9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "words = tokenizer.batch_decode(tokens['input_ids'])\n",
        "pd.DataFrame({'ner': predictions, 'words': words}).to_csv('un_ner.csv')"
      ],
      "metadata": {
        "id": "wRP-ggnypS4v"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "answers = pd.read_csv('./un_ner.csv')\n",
        "answers"
      ],
      "metadata": {
        "id": "CjJ4UCLNpkkk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "bWPuMBugv2LH"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}