Lab03_Santiago_del_Rey_Juarez_Nikita_Belooussov.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Lab03-Santiago-del-Rey-Juarez_Nikita-Belooussov.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sCSOkW8W6rI4"
      },
      "source": [
        "# Lab 03\n",
        "Authors: Santiago del Rey Juárez and Nikita Belooussov\n",
        "\n",
        "## Session Excercise\n",
        "### 1. Read all pairs of sentences of the SMTeuroparl files of test set within the evaluation framework of the project.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZVZB59ja5TIb",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c424286c-b59e-4dda-e754-26f40e0a5a6f"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WvBfiVxD7A2X",
        "outputId": "159508e5-d114-4965-96fd-47dc83087634"
      },
      "source": [
        "import pandas as pd\n",
        "import nltk\n",
        "nltk.download('averaged_perceptron_tagger')\n",
        "nltk.download('wordnet')\n",
        "nltk.download('punkt')\n",
        "from nltk.metrics import jaccard_distance\n",
        "from nltk.stem import WordNetLemmatizer"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]     /root/nltk_data...\n",
            "[nltk_data]   Package averaged_perceptron_tagger is already up-to-\n",
            "[nltk_data]       date!\n",
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Package wordnet is already up-to-date!\n",
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "wsy2X5Xj5wPU",
        "outputId": "51e3467c-ee0b-4b11-b770-44058dc137bb"
      },
      "source": [
        "dt = pd.read_csv('/content/drive/My Drive/Colab Notebooks/IHLT/data/test-gold/STS.input.SMTeuroparl.txt',sep='\\t',header=None)\n",
        "dt.head()"
      ],
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>The leaders have now been given a new chance a...</td>\n",
              "      <td>The leaders benefit aujourd' hui of a new luck...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Amendment No 7 proposes certain changes in the...</td>\n",
              "      <td>Amendment No 7 is proposing certain changes in...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Let me remind you that our allies include ferv...</td>\n",
              "      <td>I would like to remind you that among our alli...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>The vote will take place today at 5.30 p.m.</td>\n",
              "      <td>The vote will take place at 5.30pm</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>The fishermen are inactive, tired and disappoi...</td>\n",
              "      <td>The fishermen are inactive, tired and disappoi...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                   0                                                  1\n",
              "0  The leaders have now been given a new chance a...  The leaders benefit aujourd' hui of a new luck...\n",
              "1  Amendment No 7 proposes certain changes in the...  Amendment No 7 is proposing certain changes in...\n",
              "2  Let me remind you that our allies include ferv...  I would like to remind you that among our alli...\n",
              "3        The vote will take place today at 5.30 p.m.                 The vote will take place at 5.30pm\n",
              "4  The fishermen are inactive, tired and disappoi...  The fishermen are inactive, tired and disappoi..."
            ]
          },
          "metadata": {},
          "execution_count": 29
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z50gZPHR6Grs"
      },
      "source": [
        "dt['gs'] = pd.read_csv('/content/drive/My Drive/Colab Notebooks/IHLT/data/test-gold/STS.gs.SMTeuroparl.txt',sep='\\t',header=None)"
      ],
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BCEoW0lP6jwd",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e6c6421b-91db-41f8-98a6-0eb949b59de7"
      },
      "source": [
        "dt.shape"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(459, 3)"
            ]
          },
          "metadata": {},
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7_lImQ1A6fkM",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "outputId": "4b1f890a-4d56-4ce9-c82c-912817d98748"
      },
      "source": [
        "dt.head()"
      ],
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>gs</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>The leaders have now been given a new chance a...</td>\n",
              "      <td>The leaders benefit aujourd' hui of a new luck...</td>\n",
              "      <td>4.50</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Amendment No 7 proposes certain changes in the...</td>\n",
              "      <td>Amendment No 7 is proposing certain changes in...</td>\n",
              "      <td>5.00</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Let me remind you that our allies include ferv...</td>\n",
              "      <td>I would like to remind you that among our alli...</td>\n",
              "      <td>4.25</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>The vote will take place today at 5.30 p.m.</td>\n",
              "      <td>The vote will take place at 5.30pm</td>\n",
              "      <td>4.50</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>The fishermen are inactive, tired and disappoi...</td>\n",
              "      <td>The fishermen are inactive, tired and disappoi...</td>\n",
              "      <td>5.00</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                   0  ...    gs\n",
              "0  The leaders have now been given a new chance a...  ...  4.50\n",
              "1  Amendment No 7 proposes certain changes in the...  ...  5.00\n",
              "2  Let me remind you that our allies include ferv...  ...  4.25\n",
              "3        The vote will take place today at 5.30 p.m.  ...  4.50\n",
              "4  The fishermen are inactive, tired and disappoi...  ...  5.00\n",
              "\n",
              "[5 rows x 3 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QGZsMD8t8kel"
      },
      "source": [
        "### 2. Compute their similarities by considering lemmas and Jaccard distance."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DIxX4ziNAclW"
      },
      "source": [
        "wnl = nltk.stem.WordNetLemmatizer()"
      ],
      "execution_count": 33,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JtRFnlmq9-_u"
      },
      "source": [
        "from nltk.corpus import wordnet\n",
        "def lemmatize(p):\n",
        "  if p[1][0] in {'N','V','J','R'}: #N- noun, V- verb, J- adjective, R-adverb\n",
        "    if p[1][0] is 'J':  #this is used due to wordnet using a different label for adjectives than one given by nltk\n",
        "      return wnl.lemmatize(p[0].lower(), pos=wordnet.ADJ)\n",
        "    return wnl.lemmatize(p[0].lower(), pos=p[1][0].lower())\n",
        "  return p[0]\n",
        "\n",
        "def compute_jaccard(sentence_0, sentence_1, type = 'lemma'):\n",
        "  s0 = nltk.word_tokenize(sentence_0)\n",
        "  s1 = nltk.word_tokenize(sentence_1)\n",
        "  if type == 'lemma':\n",
        "    pairs_s0 = nltk.pos_tag(s0)\n",
        "    pairs_s1 = nltk.pos_tag(s1)\n",
        "    s0 = [lemmatize(pair) for pair in pairs_s0]\n",
        "    s1 = [lemmatize(pair) for pair in pairs_s1]\n",
        "  \n",
        "  return 1 - jaccard_distance(set(s0), set(s1))"
      ],
      "execution_count": 34,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3WUoABHSAm0j",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "outputId": "b4e545d5-a8d4-4f70-989c-8ad46ef31eb6"
      },
      "source": [
        "dt['jaccard_lemma'] = pd.Series(\n",
        "    map(lambda s0, s1: compute_jaccard(s0, s1, 'lemma'), dt[0], dt[1])\n",
        ")\n",
        "dt.head()"
      ],
      "execution_count": 35,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>gs</th>\n",
              "      <th>jaccard_lemma</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>The leaders have now been given a new chance a...</td>\n",
              "      <td>The leaders benefit aujourd' hui of a new luck...</td>\n",
              "      <td>4.50</td>\n",
              "      <td>0.346154</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Amendment No 7 proposes certain changes in the...</td>\n",
              "      <td>Amendment No 7 is proposing certain changes in...</td>\n",
              "      <td>5.00</td>\n",
              "      <td>0.923077</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Let me remind you that our allies include ferv...</td>\n",
              "      <td>I would like to remind you that among our alli...</td>\n",
              "      <td>4.25</td>\n",
              "      <td>0.391304</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>The vote will take place today at 5.30 p.m.</td>\n",
              "      <td>The vote will take place at 5.30pm</td>\n",
              "      <td>4.50</td>\n",
              "      <td>0.545455</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>The fishermen are inactive, tired and disappoi...</td>\n",
              "      <td>The fishermen are inactive, tired and disappoi...</td>\n",
              "      <td>5.00</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                   0  ... jaccard_lemma\n",
              "0  The leaders have now been given a new chance a...  ...      0.346154\n",
              "1  Amendment No 7 proposes certain changes in the...  ...      0.923077\n",
              "2  Let me remind you that our allies include ferv...  ...      0.391304\n",
              "3        The vote will take place today at 5.30 p.m.  ...      0.545455\n",
              "4  The fishermen are inactive, tired and disappoi...  ...      1.000000\n",
              "\n",
              "[5 rows x 4 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2eR3B2aa9q_c"
      },
      "source": [
        "### 3. Compare the results with those in session 2 (document structure) in which words were considered."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Kjhuk1g6Blvy",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "outputId": "a8d8795e-7c6e-48be-baa6-831423068c26"
      },
      "source": [
        "dt['jaccard_word'] = pd.Series(\n",
        "    map(lambda s0, s1: compute_jaccard(s0, s1, 'word'), dt[0], dt[1])\n",
        ")\n",
        "dt.head()"
      ],
      "execution_count": 36,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>gs</th>\n",
              "      <th>jaccard_lemma</th>\n",
              "      <th>jaccard_word</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>The leaders have now been given a new chance a...</td>\n",
              "      <td>The leaders benefit aujourd' hui of a new luck...</td>\n",
              "      <td>4.50</td>\n",
              "      <td>0.346154</td>\n",
              "      <td>0.346154</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Amendment No 7 proposes certain changes in the...</td>\n",
              "      <td>Amendment No 7 is proposing certain changes in...</td>\n",
              "      <td>5.00</td>\n",
              "      <td>0.923077</td>\n",
              "      <td>0.785714</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Let me remind you that our allies include ferv...</td>\n",
              "      <td>I would like to remind you that among our alli...</td>\n",
              "      <td>4.25</td>\n",
              "      <td>0.391304</td>\n",
              "      <td>0.391304</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>The vote will take place today at 5.30 p.m.</td>\n",
              "      <td>The vote will take place at 5.30pm</td>\n",
              "      <td>4.50</td>\n",
              "      <td>0.545455</td>\n",
              "      <td>0.545455</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>The fishermen are inactive, tired and disappoi...</td>\n",
              "      <td>The fishermen are inactive, tired and disappoi...</td>\n",
              "      <td>5.00</td>\n",
              "      <td>1.000000</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                   0  ... jaccard_word\n",
              "0  The leaders have now been given a new chance a...  ...     0.346154\n",
              "1  Amendment No 7 proposes certain changes in the...  ...     0.785714\n",
              "2  Let me remind you that our allies include ferv...  ...     0.391304\n",
              "3        The vote will take place today at 5.30 p.m.  ...     0.545455\n",
              "4  The fishermen are inactive, tired and disappoi...  ...     1.000000\n",
              "\n",
              "[5 rows x 5 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WISa3mi7HnCx",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "outputId": "435cb18b-57a4-408e-a347-300b6636d486"
      },
      "source": [
        "dt.query('jaccard_lemma > jaccard_word')"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>gs</th>\n",
              "      <th>jaccard_lemma</th>\n",
              "      <th>jaccard_word</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Amendment No 7 proposes certain changes in the...</td>\n",
              "      <td>Amendment No 7 is proposing certain changes in...</td>\n",
              "      <td>5.00</td>\n",
              "      <td>0.923077</td>\n",
              "      <td>0.785714</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>The leaders have now been given a new chance a...</td>\n",
              "      <td>Leaders now have another chance to let them an...</td>\n",
              "      <td>5.00</td>\n",
              "      <td>0.318182</td>\n",
              "      <td>0.260870</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>We often pontificate here about being the repr...</td>\n",
              "      <td>We are proud often here to represent the citiz...</td>\n",
              "      <td>3.50</td>\n",
              "      <td>0.600000</td>\n",
              "      <td>0.500000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>I am one of those Members who attends sittings...</td>\n",
              "      <td>I belong to the Members who are present rather...</td>\n",
              "      <td>4.25</td>\n",
              "      <td>0.277778</td>\n",
              "      <td>0.210526</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>As I already explained during second reading, ...</td>\n",
              "      <td>As I have already explained, at second reading...</td>\n",
              "      <td>4.50</td>\n",
              "      <td>0.565217</td>\n",
              "      <td>0.500000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>438</th>\n",
              "      <td>That provision could open the door wide to arb...</td>\n",
              "      <td>This regulation opens the door to arbitrariness.</td>\n",
              "      <td>4.25</td>\n",
              "      <td>0.500000</td>\n",
              "      <td>0.384615</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>439</th>\n",
              "      <td>The European Union has got to do something and...</td>\n",
              "      <td>It is appropriate that the European Union is i...</td>\n",
              "      <td>4.00</td>\n",
              "      <td>0.388889</td>\n",
              "      <td>0.315789</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>440</th>\n",
              "      <td>Amendment No 7 proposes certain changes in the...</td>\n",
              "      <td>Amendment No 7 is proposing certain changes in...</td>\n",
              "      <td>5.00</td>\n",
              "      <td>0.923077</td>\n",
              "      <td>0.785714</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>443</th>\n",
              "      <td>Then perhaps we could have avoided a catastrophe.</td>\n",
              "      <td>We would perhaps have been able to then avoid ...</td>\n",
              "      <td>4.25</td>\n",
              "      <td>0.400000</td>\n",
              "      <td>0.235294</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>455</th>\n",
              "      <td>The vote will take place today at 5.30 p.m.</td>\n",
              "      <td>Vote will take place at 17 h 30.</td>\n",
              "      <td>4.75</td>\n",
              "      <td>0.461538</td>\n",
              "      <td>0.357143</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>145 rows × 5 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                     0  ... jaccard_word\n",
              "1    Amendment No 7 proposes certain changes in the...  ...     0.785714\n",
              "8    The leaders have now been given a new chance a...  ...     0.260870\n",
              "11   We often pontificate here about being the repr...  ...     0.500000\n",
              "14   I am one of those Members who attends sittings...  ...     0.210526\n",
              "17   As I already explained during second reading, ...  ...     0.500000\n",
              "..                                                 ...  ...          ...\n",
              "438  That provision could open the door wide to arb...  ...     0.384615\n",
              "439  The European Union has got to do something and...  ...     0.315789\n",
              "440  Amendment No 7 proposes certain changes in the...  ...     0.785714\n",
              "443  Then perhaps we could have avoided a catastrophe.  ...     0.235294\n",
              "455        The vote will take place today at 5.30 p.m.  ...     0.357143\n",
              "\n",
              "[145 rows x 5 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 37
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "3Wj7fsGijgmB",
        "outputId": "88b6fb9b-00ce-40a3-fd16-61692604ad46"
      },
      "source": [
        "dt.query('jaccard_lemma < jaccard_word')\n",
        "\n"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>gs</th>\n",
              "      <th>jaccard_lemma</th>\n",
              "      <th>jaccard_word</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>Amendment No 7 proposes certain changes in the...</td>\n",
              "      <td>Amendment No 7 proposes changes to certain par...</td>\n",
              "      <td>5.0</td>\n",
              "      <td>0.571429</td>\n",
              "      <td>0.833333</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>42</th>\n",
              "      <td>Amendment No 7 proposes certain changes in the...</td>\n",
              "      <td>Amendment 7 proposes certain modifications in ...</td>\n",
              "      <td>5.0</td>\n",
              "      <td>0.642857</td>\n",
              "      <td>0.769231</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>72</th>\n",
              "      <td>Amendment No 7 proposes certain changes in the...</td>\n",
              "      <td>Amendment 7 proposes certain modifications in ...</td>\n",
              "      <td>5.0</td>\n",
              "      <td>0.642857</td>\n",
              "      <td>0.769231</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>132</th>\n",
              "      <td>Amendment No 7 proposes certain changes in the...</td>\n",
              "      <td>Amendment 7 proposes certain modifications in ...</td>\n",
              "      <td>5.0</td>\n",
              "      <td>0.642857</td>\n",
              "      <td>0.769231</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>197</th>\n",
              "      <td>The fishermen are inactive, tired and disappoi...</td>\n",
              "      <td>The fishermen inactive, are tired and disappoi...</td>\n",
              "      <td>5.0</td>\n",
              "      <td>0.636364</td>\n",
              "      <td>1.000000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                                     0  ... jaccard_word\n",
              "19   Amendment No 7 proposes certain changes in the...  ...     0.833333\n",
              "42   Amendment No 7 proposes certain changes in the...  ...     0.769231\n",
              "72   Amendment No 7 proposes certain changes in the...  ...     0.769231\n",
              "132  Amendment No 7 proposes certain changes in the...  ...     0.769231\n",
              "197  The fishermen are inactive, tired and disappoi...  ...     1.000000\n",
              "\n",
              "[5 rows x 5 columns]"
            ]
          },
          "metadata": {},
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jhJj4WlW9rRl"
      },
      "source": [
        "### 4. Compare the results with gold standard by giving the pearson correlation between them."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AYjJYHVKG6nh",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "58d8096f-467e-4914-b0fc-590d76f5cde7"
      },
      "source": [
        "from scipy.stats import pearsonr\n",
        "corr_lemma = pearsonr(dt['gs'], dt['jaccard_lemma'])[0]\n",
        "corr_word = pearsonr(dt['gs'], dt['jaccard_word'])[0]\n",
        "print('Pearson correlation between gs and jaccard_lemma:', corr_lemma)\n",
        "print('Pearson correlation between gs and jaccard_word:', corr_word)"
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Pearson correlation between gs and jaccard_lemma: 0.4568471894478094\n",
            "Pearson correlation between gs and jaccard_word: 0.4504977169318684\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iemi4rKZ9rgu"
      },
      "source": [
        "### 5. Questions (justify the answers):\n",
        "  * Which is better: words or lemmas?\n",
        "  * Do you think that could perform better for any pair of texts?"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gGsPbYD-N1I1"
      },
      "source": [
        "According to our results, both methods are approximately equivalent. The lemmas method performs slightly better overall, but not by a significant enough amount when comparing large amounts of texts. A closer inspection shows that the lemmas tends to perform better if the sentence structure does not change between the two compared sentences, while if the structure is changed then the words method will perform better. \n",
        "<br><br>A good example for the lemmas case can be seen in the first sentence in the dataset. Where the sentence structure is not changed, only that the verb tense is changed between the two sentences by adding an is. Due to the lemmatization of the sentence the changed verb proposing changed back to propose, thus matching the original sentence more closely.\n",
        "<br><br>An example when to use the words method can be seen in sentence 197 in the dataset. In this sentence the sentence structure changes when the word inactive is moved to be before the verb. This causes some of the words in the sentence to be lemmatized differently, and not match the other sentence anymore. Since this does not happen when using the word method, it produces a better outcome. \n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HBNP9hMkM8nP",
        "outputId": "e033d519-1801-4abe-d668-64f0361f4c7a"
      },
      "source": [
        "#testing why the lemma performs better than words\n",
        "s0 = nltk.word_tokenize(dt.loc[1][0])\n",
        "s1 = nltk.word_tokenize(dt.loc[1][1])\n",
        "pairs_s0 = nltk.pos_tag(s0)\n",
        "pairs_s1 = nltk.pos_tag(s1)\n",
        "lemmas_s0 = [lemmatize(pair) for pair in pairs_s0]\n",
        "lemmas_s1 = [lemmatize(pair) for pair in pairs_s1]\n",
        "\n",
        "print(f\"Sentence 0 after tokenizing:\\n{s0}\")\n",
        "print(f\"Sentence 0 after tokenizing and pos tagging:\\n{pairs_s0}\")\n",
        "print(f\"Sentence 0 after tokenizing, pos tagging and lemmatizing:\\n{lemmas_s0}\")\n",
        "print()\n",
        "\n",
        "print(f\"Sentence 1 after tokenizing:\\n{s1}\")\n",
        "print(f\"Sentence 1 after tokenizing and pos tagging:\\n{pairs_s1}\")\n",
        "print(f\"Sentence 1 after tokenizing, pos tagging and lemmatizing:\\n{lemmas_s1}\")"
      ],
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Sentence 0 after tokenizing:\n",
            "['Amendment', 'No', '7', 'proposes', 'certain', 'changes', 'in', 'the', 'references', 'to', 'paragraphs', '.']\n",
            "Sentence 0 after tokenizing and pos tagging:\n",
            "[('Amendment', 'NNP'), ('No', 'NNP'), ('7', 'CD'), ('proposes', 'VBZ'), ('certain', 'JJ'), ('changes', 'NNS'), ('in', 'IN'), ('the', 'DT'), ('references', 'NNS'), ('to', 'TO'), ('paragraphs', 'VB'), ('.', '.')]\n",
            "Sentence 0 after tokenizing, pos tagging and lemmatizing:\n",
            "['amendment', 'no', '7', 'propose', 'certain', 'change', 'in', 'the', 'reference', 'to', 'paragraph', '.']\n",
            "\n",
            "Sentence 1 after tokenizing:\n",
            "['Amendment', 'No', '7', 'is', 'proposing', 'certain', 'changes', 'in', 'the', 'references', 'to', 'paragraphs', '.']\n",
            "Sentence 1 after tokenizing and pos tagging:\n",
            "[('Amendment', 'NNP'), ('No', 'NNP'), ('7', 'CD'), ('is', 'VBZ'), ('proposing', 'VBG'), ('certain', 'JJ'), ('changes', 'NNS'), ('in', 'IN'), ('the', 'DT'), ('references', 'NNS'), ('to', 'TO'), ('paragraphs', 'VB'), ('.', '.')]\n",
            "Sentence 1 after tokenizing, pos tagging and lemmatizing:\n",
            "['amendment', 'no', '7', 'be', 'propose', 'certain', 'change', 'in', 'the', 'reference', 'to', 'paragraph', '.']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZnlnJKg1D9zp",
        "outputId": "3182ed49-f802-4307-e082-4f1ccefa3d18"
      },
      "source": [
        "#testing why the word preforms better on some sentences than lemma\n",
        "s0 = nltk.word_tokenize(dt.loc[197][0])\n",
        "s1 = nltk.word_tokenize(dt.loc[197][1])\n",
        "pairs_s0 = nltk.pos_tag(s0)\n",
        "pairs_s1 = nltk.pos_tag(s1)\n",
        "lemmas_s0 = [lemmatize(pair) for pair in pairs_s0]\n",
        "lemmas_s1 = [lemmatize(pair) for pair in pairs_s1]\n",
        "\n",
        "print(f\"Sentence 0 after tokenizing:\\n{s0}\")\n",
        "print(f\"Sentence 0 after tokenizing and pos tagging:\\n{pairs_s0}\")\n",
        "print(f\"Sentence 0 after tokenizing, pos tagging and lemmatizing:\\n{lemmas_s0}\")\n",
        "print()\n",
        "\n",
        "print(f\"Sentence 1 after tokenizing:\\n{s1}\")\n",
        "print(f\"Sentence 1 after tokenizing and pos tagging:\\n{pairs_s1}\")\n",
        "print(f\"Sentence 1 after tokenizing, pos tagging and lemmatizing:\\n{lemmas_s1}\")"
      ],
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Sentence 0 after tokenizing:\n",
            "['The', 'fishermen', 'are', 'inactive', ',', 'tired', 'and', 'disappointed', '.']\n",
            "Sentence 0 after tokenizing and pos tagging:\n",
            "[('The', 'DT'), ('fishermen', 'NNS'), ('are', 'VBP'), ('inactive', 'JJ'), (',', ','), ('tired', 'JJ'), ('and', 'CC'), ('disappointed', 'JJ'), ('.', '.')]\n",
            "Sentence 0 after tokenizing, pos tagging and lemmatizing:\n",
            "['The', 'fisherman', 'be', 'inactive', ',', 'tired', 'and', 'disappointed', '.']\n",
            "\n",
            "Sentence 1 after tokenizing:\n",
            "['The', 'fishermen', 'inactive', ',', 'are', 'tired', 'and', 'disappointed', '.']\n",
            "Sentence 1 after tokenizing and pos tagging:\n",
            "[('The', 'DT'), ('fishermen', 'NNS'), ('inactive', 'JJ'), (',', ','), ('are', 'VBP'), ('tired', 'VBN'), ('and', 'CC'), ('disappointed', 'VBN'), ('.', '.')]\n",
            "Sentence 1 after tokenizing, pos tagging and lemmatizing:\n",
            "['The', 'fisherman', 'inactive', ',', 'be', 'tire', 'and', 'disappoint', '.']\n"
          ]
        }
      ]
    }
  ]
}