Part 2.3: Stanza_Biomedical_NLP_Demo.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Part 2.3: StanzaBiomedicalNLPDemo.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyN7uPyY6Qh9gnXzdIPdvrKy"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mQDeUe_Sv4hq"
      },
      "source": [
        "Stanza : https://stanfordnlp.github.io/stanza/\n",
        "\n",
        "Stanza Biomedical NLP : https://stanfordnlp.github.io/\n",
        "stanza/biomed.html\n",
        "\n",
        "Biomedical NLP Paper : https://arxiv.org/abs/2007.14640\n",
        "\n",
        "And video help is available on \"https://www.youtube.com/watch?v=iQ5kP2kOGNA\""
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z1ZwK8UXt5Zs"
      },
      "source": [
        "!pip install stanza"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7QnO9jlWtKHs"
      },
      "source": [
        "import stanza"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QSwUsoEWtw7L",
        "outputId": "2a3599c2-a2c6-40da-d566-07dfbdc08fc3"
      },
      "source": [
        "# download and initialize the CRAFT pipeline\n",
        "stanza.download('en', package='craft')\n",
        "nlp = stanza.Pipeline('en', package='craft')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading https://raw.githubusercontent.com/stanfordnlp/stanza-resources/master/resources_1.2.0.json: 128kB [00:00, 23.0MB/s]                    \n",
            "2021-04-08 18:26:26 INFO: Downloading these customized packages for language: en (English)...\n",
            "=======================\n",
            "| Processor | Package |\n",
            "-----------------------\n",
            "| tokenize  | craft   |\n",
            "| pos       | craft   |\n",
            "| lemma     | craft   |\n",
            "| depparse  | craft   |\n",
            "| pretrain  | craft   |\n",
            "=======================\n",
            "\n",
            "2021-04-08 18:26:26 INFO: File exists: /root/stanza_resources/en/tokenize/craft.pt.\n",
            "2021-04-08 18:26:26 INFO: File exists: /root/stanza_resources/en/pos/craft.pt.\n",
            "2021-04-08 18:26:26 INFO: File exists: /root/stanza_resources/en/lemma/craft.pt.\n",
            "2021-04-08 18:26:26 INFO: File exists: /root/stanza_resources/en/depparse/craft.pt.\n",
            "2021-04-08 18:26:27 INFO: File exists: /root/stanza_resources/en/pretrain/craft.pt.\n",
            "2021-04-08 18:26:27 INFO: Finished downloading models and saved to /root/stanza_resources.\n",
            "2021-04-08 18:26:27 INFO: Loading these models for language: en (English):\n",
            "=======================\n",
            "| Processor | Package |\n",
            "-----------------------\n",
            "| tokenize  | craft   |\n",
            "| pos       | craft   |\n",
            "| lemma     | craft   |\n",
            "| depparse  | craft   |\n",
            "=======================\n",
            "\n",
            "2021-04-08 18:26:27 INFO: Use device: cpu\n",
            "2021-04-08 18:26:27 INFO: Loading: tokenize\n",
            "2021-04-08 18:26:27 INFO: Loading: pos\n",
            "2021-04-08 18:26:27 INFO: Loading: lemma\n",
            "2021-04-08 18:26:27 INFO: Loading: depparse\n",
            "2021-04-08 18:26:27 INFO: Done loading processors!\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nv62ao-ctzrD"
      },
      "source": [
        "# annotate example text\n",
        "doc = nlp('A single-cell transcriptomic atlas characterizes ageing tissues in the mouse.')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LlBQcvv8uMeB",
        "outputId": "5d73e06e-2f0f-4800-fc82-128a71c250f8"
      },
      "source": [
        "# print out dependency tree\n",
        "doc.sentences[0].print_dependencies()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "('A', 6, 'det')\n",
            "('single', 4, 'amod')\n",
            "('-', 4, 'punct')\n",
            "('cell', 6, 'compound')\n",
            "('transcriptomic', 6, 'amod')\n",
            "('atlas', 7, 'nsubj')\n",
            "('characterizes', 0, 'root')\n",
            "('ageing', 9, 'compound')\n",
            "('tissues', 7, 'obj')\n",
            "('in', 12, 'case')\n",
            "('the', 12, 'det')\n",
            "('mouse', 7, 'obl')\n",
            "('.', 7, 'punct')\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "emIOIYv1uQUC",
        "outputId": "e3711dab-15ea-44cc-838e-1b01d49658e5"
      },
      "source": [
        "# download and initialize a mimic pipeline with an i2b2 NER model\n",
        "stanza.download('en', package='mimic', processors={'ner': 'i2b2'})\n",
        "nlp = stanza.Pipeline('en', package='mimic', processors={'ner': 'i2b2'})\n",
        "# annotate clinical text\n",
        "doc = nlp('The patient had a sore throat and was treated with Cepacol lozenges.')\n",
        "# print out all entities\n",
        "for ent in doc.entities:\n",
        "    print(f'{ent.text}\\t{ent.type}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading https://raw.githubusercontent.com/stanfordnlp/stanza-resources/master/resources_1.2.0.json: 128kB [00:00, 20.9MB/s]                    \n",
            "2021-04-08 18:26:28 INFO: Downloading these customized packages for language: en (English)...\n",
            "=============================\n",
            "| Processor       | Package |\n",
            "-----------------------------\n",
            "| tokenize        | mimic   |\n",
            "| pos             | mimic   |\n",
            "| lemma           | mimic   |\n",
            "| depparse        | mimic   |\n",
            "| ner             | i2b2    |\n",
            "| forward_charlm  | mimic   |\n",
            "| pretrain        | mimic   |\n",
            "| backward_charlm | mimic   |\n",
            "=============================\n",
            "\n",
            "2021-04-08 18:26:28 INFO: File exists: /root/stanza_resources/en/tokenize/mimic.pt.\n",
            "2021-04-08 18:26:28 INFO: File exists: /root/stanza_resources/en/pos/mimic.pt.\n",
            "2021-04-08 18:26:28 INFO: File exists: /root/stanza_resources/en/lemma/mimic.pt.\n",
            "2021-04-08 18:26:28 INFO: File exists: /root/stanza_resources/en/depparse/mimic.pt.\n",
            "2021-04-08 18:26:28 INFO: File exists: /root/stanza_resources/en/ner/i2b2.pt.\n",
            "2021-04-08 18:26:28 INFO: File exists: /root/stanza_resources/en/forward_charlm/mimic.pt.\n",
            "2021-04-08 18:26:29 INFO: File exists: /root/stanza_resources/en/pretrain/mimic.pt.\n",
            "2021-04-08 18:26:29 INFO: File exists: /root/stanza_resources/en/backward_charlm/mimic.pt.\n",
            "2021-04-08 18:26:29 INFO: Finished downloading models and saved to /root/stanza_resources.\n",
            "2021-04-08 18:26:29 INFO: Loading these models for language: en (English):\n",
            "=======================\n",
            "| Processor | Package |\n",
            "-----------------------\n",
            "| tokenize  | mimic   |\n",
            "| pos       | mimic   |\n",
            "| lemma     | mimic   |\n",
            "| depparse  | mimic   |\n",
            "| ner       | i2b2    |\n",
            "=======================\n",
            "\n",
            "2021-04-08 18:26:29 INFO: Use device: cpu\n",
            "2021-04-08 18:26:29 INFO: Loading: tokenize\n",
            "2021-04-08 18:26:29 INFO: Loading: pos\n",
            "2021-04-08 18:26:29 INFO: Loading: lemma\n",
            "2021-04-08 18:26:29 INFO: Loading: depparse\n",
            "2021-04-08 18:26:29 INFO: Loading: ner\n",
            "2021-04-08 18:26:30 INFO: Done loading processors!\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "a sore throat\tPROBLEM\n",
            "Cepacol lozenges\tTREATMENT\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jailLtCPua0d",
        "outputId": "b1302b71-a285-4667-c34e-fdf81faa5756"
      },
      "source": [
        "doc = nlp(\"Selegiline - induced postural hypotension in Parkinson ' s disease : a longitudinal study on the effects of drug withdrawal.The aims of this study were to confirm our previous findings in a separate cohort of patients and to determine the time course of the cardiovascular consequences of stopping selegiline in the expectation that this might shed light on the mechanisms by which the drug causes orthostatic hypotension\")\n",
        "# print out all entities\n",
        "for ent in doc.entities:\n",
        "    print(f'{ent.text}\\t{ent.type}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Selegiline\tTREATMENT\n",
            "postural hypotension\tPROBLEM\n",
            "Parkinson ' s disease\tPROBLEM\n",
            "a longitudinal study\tTEST\n",
            "drug withdrawal\tPROBLEM\n",
            "this study\tTEST\n",
            "selegiline\tTREATMENT\n",
            "orthostatic hypotension\tPROBLEM\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tZziKZkCuevk",
        "outputId": "ceca4971-5a50-4090-b4ad-88ab3bfb47d5"
      },
      "source": [
        "stanza.download('en', package='mimic', processors={'ner': 'anatem'})\n",
        "nlp = stanza.Pipeline('en', package='mimic', processors={'ner': 'anatem'})\n",
        "# annotate clinical text"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading https://raw.githubusercontent.com/stanfordnlp/stanza-resources/master/resources_1.2.0.json: 128kB [00:00, 27.7MB/s]                    \n",
            "2021-04-08 18:26:32 INFO: Downloading these customized packages for language: en (English)...\n",
            "=============================\n",
            "| Processor       | Package |\n",
            "-----------------------------\n",
            "| tokenize        | mimic   |\n",
            "| pos             | mimic   |\n",
            "| lemma           | mimic   |\n",
            "| depparse        | mimic   |\n",
            "| ner             | anatem  |\n",
            "| backward_charlm | pubmed  |\n",
            "| pretrain        | mimic   |\n",
            "| forward_charlm  | pubmed  |\n",
            "=============================\n",
            "\n",
            "2021-04-08 18:26:32 INFO: File exists: /root/stanza_resources/en/tokenize/mimic.pt.\n",
            "2021-04-08 18:26:32 INFO: File exists: /root/stanza_resources/en/pos/mimic.pt.\n",
            "2021-04-08 18:26:32 INFO: File exists: /root/stanza_resources/en/lemma/mimic.pt.\n",
            "2021-04-08 18:26:32 INFO: File exists: /root/stanza_resources/en/depparse/mimic.pt.\n",
            "2021-04-08 18:26:33 INFO: File exists: /root/stanza_resources/en/ner/anatem.pt.\n",
            "2021-04-08 18:26:33 INFO: File exists: /root/stanza_resources/en/backward_charlm/pubmed.pt.\n",
            "2021-04-08 18:26:33 INFO: File exists: /root/stanza_resources/en/pretrain/mimic.pt.\n",
            "2021-04-08 18:26:33 INFO: File exists: /root/stanza_resources/en/forward_charlm/pubmed.pt.\n",
            "2021-04-08 18:26:33 INFO: Finished downloading models and saved to /root/stanza_resources.\n",
            "2021-04-08 18:26:33 INFO: Loading these models for language: en (English):\n",
            "=======================\n",
            "| Processor | Package |\n",
            "-----------------------\n",
            "| tokenize  | mimic   |\n",
            "| pos       | mimic   |\n",
            "| lemma     | mimic   |\n",
            "| depparse  | mimic   |\n",
            "| ner       | anatem  |\n",
            "=======================\n",
            "\n",
            "2021-04-08 18:26:33 INFO: Use device: cpu\n",
            "2021-04-08 18:26:33 INFO: Loading: tokenize\n",
            "2021-04-08 18:26:33 INFO: Loading: pos\n",
            "2021-04-08 18:26:33 INFO: Loading: lemma\n",
            "2021-04-08 18:26:33 INFO: Loading: depparse\n",
            "2021-04-08 18:26:33 INFO: Loading: ner\n",
            "2021-04-08 18:26:34 INFO: Done loading processors!\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aikptUgtug2z",
        "outputId": "dbdd2c3d-abc5-4d61-d75b-8ed8df325439"
      },
      "source": [
        "doc = nlp(\"Selegiline - induced postural hypotension in Parkinson ' s disease : a longitudinal study on the effects of drug withdrawal.The aims of this study were to confirm our previous findings in a separate cohort of patients and to determine the time course of the cardiovascular consequences of stopping selegiline in the expectation that this might shed light on the mechanisms by which the drug causes orthostatic hypotension\")\n",
        "# print out all entities\n",
        "for ent in doc.entities:\n",
        "    print(f'{ent.text}\\t{ent.type}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "cardiovascular\tANATOMY\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hizkfW9SukCl",
        "outputId": "9297216a-9f26-4144-c701-f3dfc4db8c0c"
      },
      "source": [
        "doc = nlp(\"A 22-year-old man presented with progressive neurologic decline. A year before presentation, painless vision loss had occurred in the left eye, followed by the right eye. During the next 4 to 6 months, diffuse numbness, limb stiffness, whole-body weakness, and urinary incontinence had developed.\")\n",
        "# print out all entities\n",
        "for ent in doc.entities:\n",
        "    print(f'{ent.text}\\t{ent.type}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "neurologic\tANATOMY\n",
            "left eye\tANATOMY\n",
            "right eye\tANATOMY\n",
            "limb\tANATOMY\n",
            "body\tANATOMY\n",
            "urinary\tANATOMY\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "G2aZuzbcumFe",
        "outputId": "74da4400-57e7-4726-c487-f02a5d1ca467"
      },
      "source": [
        "nlp = stanza.Pipeline('en', package='mimic', processors={'ner':'i2b2'})\n",
        "doc = nlp(\"A 22-year-old man presented with progressive neurologic decline. A year before presentation, painless vision loss had occurred in the left eye, followed by the right eye. During the next 4 to 6 months, diffuse numbness, limb stiffness, whole-body weakness, and urinary incontinence had developed.\")\n",
        "# print out all entities\n",
        "for ent in doc.entities:\n",
        "    print(f'{ent.text}\\t{ent.type}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-04-08 18:26:37 INFO: Loading these models for language: en (English):\n",
            "=======================\n",
            "| Processor | Package |\n",
            "-----------------------\n",
            "| tokenize  | mimic   |\n",
            "| pos       | mimic   |\n",
            "| lemma     | mimic   |\n",
            "| depparse  | mimic   |\n",
            "| ner       | i2b2    |\n",
            "=======================\n",
            "\n",
            "2021-04-08 18:26:37 INFO: Use device: cpu\n",
            "2021-04-08 18:26:37 INFO: Loading: tokenize\n",
            "2021-04-08 18:26:37 INFO: Loading: pos\n",
            "2021-04-08 18:26:37 INFO: Loading: lemma\n",
            "2021-04-08 18:26:37 INFO: Loading: depparse\n",
            "2021-04-08 18:26:38 INFO: Loading: ner\n",
            "2021-04-08 18:26:38 INFO: Done loading processors!\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "progressive neurologic decline\tPROBLEM\n",
            "painless vision loss\tPROBLEM\n",
            "diffuse numbness\tPROBLEM\n",
            "limb stiffness\tPROBLEM\n",
            "whole-body weakness\tPROBLEM\n",
            "urinary incontinence\tPROBLEM\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1OMX-FJmuqc1",
        "outputId": "b715a949-2a9a-48dc-a6d0-c105bf76f34b"
      },
      "source": [
        "nlp = stanza.Pipeline('en', package='mimic', processors={'ner':'anatem'})\n",
        "doc = nlp(\"A 70-year-old man with bipolar disorder was evaluated because of depressed mood, suicidal ideation, and gait disturbance. One month earlier, he had been voluntarily hospitalized for depression. In the following weeks, he had worsening gait instability and urinary incontinence. Diagnostic tests were performed.\")\n",
        "# print out all entities\n",
        "for ent in doc.entities:\n",
        "    print(f'{ent.text}\\t{ent.type}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-04-08 18:26:40 INFO: Loading these models for language: en (English):\n",
            "=======================\n",
            "| Processor | Package |\n",
            "-----------------------\n",
            "| tokenize  | mimic   |\n",
            "| pos       | mimic   |\n",
            "| lemma     | mimic   |\n",
            "| depparse  | mimic   |\n",
            "| ner       | anatem  |\n",
            "=======================\n",
            "\n",
            "2021-04-08 18:26:40 INFO: Use device: cpu\n",
            "2021-04-08 18:26:40 INFO: Loading: tokenize\n",
            "2021-04-08 18:26:40 INFO: Loading: pos\n",
            "2021-04-08 18:26:40 INFO: Loading: lemma\n",
            "2021-04-08 18:26:40 INFO: Loading: depparse\n",
            "2021-04-08 18:26:40 INFO: Loading: ner\n",
            "2021-04-08 18:26:41 INFO: Done loading processors!\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "urinary\tANATOMY\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hxSh0ZCxuzPL",
        "outputId": "c5cdddff-71ef-4373-8294-d026e5eea3ce"
      },
      "source": [
        "nlp = stanza.Pipeline('en', package='mimic', processors={'ner':'i2b2'})\n",
        "doc = nlp(\"A 70-year-old man with bipolar disorder was evaluated because of depressed mood, suicidal ideation, and gait disturbance. One month earlier, he had been voluntarily hospitalized for depression. In the following weeks, he had worsening gait instability and urinary incontinence. Diagnostic tests were performed.\")\n",
        "# print out all entities\n",
        "for ent in doc.entities:\n",
        "    print(f'{ent.text}\\t{ent.type}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "2021-04-08 18:26:42 INFO: Loading these models for language: en (English):\n",
            "=======================\n",
            "| Processor | Package |\n",
            "-----------------------\n",
            "| tokenize  | mimic   |\n",
            "| pos       | mimic   |\n",
            "| lemma     | mimic   |\n",
            "| depparse  | mimic   |\n",
            "| ner       | i2b2    |\n",
            "=======================\n",
            "\n",
            "2021-04-08 18:26:42 INFO: Use device: cpu\n",
            "2021-04-08 18:26:42 INFO: Loading: tokenize\n",
            "2021-04-08 18:26:42 INFO: Loading: pos\n",
            "2021-04-08 18:26:42 INFO: Loading: lemma\n",
            "2021-04-08 18:26:42 INFO: Loading: depparse\n",
            "2021-04-08 18:26:43 INFO: Loading: ner\n",
            "2021-04-08 18:26:43 INFO: Done loading processors!\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "bipolar disorder\tPROBLEM\n",
            "depressed mood\tPROBLEM\n",
            "suicidal ideation\tPROBLEM\n",
            "gait disturbance\tPROBLEM\n",
            "depression\tPROBLEM\n",
            "worsening gait instability\tPROBLEM\n",
            "urinary incontinence\tPROBLEM\n",
            "Diagnostic tests\tTEST\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VbYDxA8Cu2zG"
      },
      "source": [
        "--------------------------------------"
      ]
    }
  ]
}