code.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Text summarization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# NLP Pkgs\n",
    "import spacy \n",
    "nlp = spacy.load('en_core_web_sm')\n",
    "# Pkgs for Normalizing Text\n",
    "from spacy.lang.en.stop_words import STOP_WORDS\n",
    "from string import punctuation\n",
    "# Import Heapq for Finding the Top N Sentences\n",
    "from heapq import nlargest\n",
    "\n",
    "\n",
    "\n",
    "def text_summarizer(raw_docx):\n",
    "    raw_text = raw_docx\n",
    "    docx = nlp(raw_text)\n",
    "    stopwords = list(STOP_WORDS)\n",
    "    # Build Word Frequency # word.text is tokenization in spacy\n",
    "    word_frequencies = {}  \n",
    "    for word in docx:  \n",
    "        if word.text not in stopwords:\n",
    "            if word.text not in word_frequencies.keys():\n",
    "                word_frequencies[word.text] = 1\n",
    "            else:\n",
    "                word_frequencies[word.text] += 1\n",
    "\n",
    "\n",
    "    maximum_frequncy = max(word_frequencies.values())\n",
    "\n",
    "    for word in word_frequencies.keys():  \n",
    "        word_frequencies[word] = (word_frequencies[word]/maximum_frequncy)\n",
    "    # Sentence Tokens\n",
    "    sentence_list = [ sentence for sentence in docx.sents ]\n",
    "\n",
    "    # Sentence Scores\n",
    "    sentence_scores = {}  \n",
    "    for sent in sentence_list:  \n",
    "        for word in sent:\n",
    "            if word.text.lower() in word_frequencies.keys():\n",
    "                if len(sent.text.split(' ')) < 30:\n",
    "                    if sent not in sentence_scores.keys():\n",
    "                        sentence_scores[sent] = word_frequencies[word.text.lower()]\n",
    "                    else:\n",
    "                        sentence_scores[sent] += word_frequencies[word.text.lower()]\n",
    "\n",
    "\n",
    "    summarized_sentences = nlargest(7, sentence_scores, key=sentence_scores.get)\n",
    "    final_sentences = [ w.text for w in summarized_sentences ]\n",
    "    summary = ' '.join(final_sentences)\n",
    "    return summary"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Text summarizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# NLP Pkgs\n",
    "import spacy \n",
    "nlp = spacy.load('en_core_web_sm')\n",
    "# Pkgs for Normalizing Text\n",
    "from spacy.lang.en.stop_words import STOP_WORDS\n",
    "from string import punctuation\n",
    "# Import Heapq for Finding the Top N Sentences\n",
    "from heapq import nlargest\n",
    "\n",
    "\n",
    "\n",
    "def text_summarizer(raw_docx):\n",
    "    raw_text = raw_docx\n",
    "    docx = nlp(raw_text)\n",
    "    stopwords = list(STOP_WORDS)\n",
    "    # Build Word Frequency # word.text is tokenization in spacy\n",
    "    word_frequencies = {}  \n",
    "    for word in docx:  \n",
    "        if word.text not in stopwords:\n",
    "            if word.text not in word_frequencies.keys():\n",
    "                word_frequencies[word.text] = 1\n",
    "            else:\n",
    "                word_frequencies[word.text] += 1\n",
    "\n",
    "\n",
    "    maximum_frequncy = max(word_frequencies.values())\n",
    "\n",
    "    for word in word_frequencies.keys():  \n",
    "        word_frequencies[word] = (word_frequencies[word]/maximum_frequncy)\n",
    "    # Sentence Tokens\n",
    "    sentence_list = [ sentence for sentence in docx.sents ]\n",
    "\n",
    "    # Sentence Scores\n",
    "    sentence_scores = {}  \n",
    "    for sent in sentence_list:  \n",
    "        for word in sent:\n",
    "            if word.text.lower() in word_frequencies.keys():\n",
    "                if len(sent.text.split(' ')) < 30:\n",
    "                    if sent not in sentence_scores.keys():\n",
    "                        sentence_scores[sent] = word_frequencies[word.text.lower()]\n",
    "                    else:\n",
    "                        sentence_scores[sent] += word_frequencies[word.text.lower()]\n",
    "\n",
    "\n",
    "    summarized_sentences = nlargest(7, sentence_scores, key=sentence_scores.get)\n",
    "    final_sentences = [ w.text for w in summarized_sentences ]\n",
    "    summary = ' '.join(final_sentences)\n",
    "    print(\"Original Document\\n\")\n",
    "    print(raw_docx)\n",
    "    print(\"Total Length:\",len(raw_docx))\n",
    "    print('\\n\\nSummarized Document\\n')\n",
    "    print(summary)\n",
    "    print(\"Total Length:\",len(summary))\n",
    "    \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Nltk Summarization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import word_tokenize, sent_tokenize\n",
    "import heapq  \n",
    "\n",
    "def nltk_summarizer(raw_text):\n",
    "    stopWords = set(stopwords.words(\"english\"))\n",
    "    word_frequencies = {}  \n",
    "    for word in nltk.word_tokenize(raw_text):  \n",
    "        if word not in stopWords:\n",
    "            if word not in word_frequencies.keys():\n",
    "                word_frequencies[word] = 1\n",
    "            else:\n",
    "                word_frequencies[word] += 1\n",
    "\n",
    "    maximum_frequncy = max(word_frequencies.values())\n",
    "\n",
    "    for word in word_frequencies.keys():  \n",
    "        word_frequencies[word] = (word_frequencies[word]/maximum_frequncy)\n",
    "\n",
    "    sentence_list = nltk.sent_tokenize(raw_text)\n",
    "    sentence_scores = {}  \n",
    "    for sent in sentence_list:  \n",
    "        for word in nltk.word_tokenize(sent.lower()):\n",
    "            if word in word_frequencies.keys():\n",
    "                if len(sent.split(' ')) < 30:\n",
    "                    if sent not in sentence_scores.keys():\n",
    "                        sentence_scores[sent] = word_frequencies[word]\n",
    "                    else:\n",
    "                        sentence_scores[sent] += word_frequencies[word]\n",
    "\n",
    "\n",
    "\n",
    "    summary_sentences = heapq.nlargest(7, sentence_scores, key=sentence_scores.get)\n",
    "\n",
    "    summary = ' '.join(summary_sentences)  \n",
    "    return summary"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Essay Grading"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import word_tokenize\n",
    "import re\n",
    "import pandas as pd\n",
    "stop_words=set(stopwords.words('english'))\n",
    "import numpy as np\n",
    "import nltk\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "import pickle\n",
    "count_fit=pd.read_csv(r'count_fit_data.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import word_tokenize\n",
    "import re\n",
    "import pandas as pd\n",
    "stop_words=set(stopwords.words('english'))\n",
    "import numpy as np\n",
    "import nltk\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "import pickle\n",
    "count_fit=pd.read_csv(r'count_fit_data.csv')\n",
    "\n",
    "def grade(essay):\n",
    "    \n",
    "    \n",
    "    essay=re.sub(\"[^A-Za-z ]\",\"\",essay)\n",
    "    x=[]\n",
    "    for i in essay.split():\n",
    "        \n",
    "        if i.startswith(\"@\") and i in stop_words:\n",
    "            continue\n",
    "        else:\n",
    "            x.append(i)        \n",
    "    essay=' '.join(x).lower()\n",
    "    essay_df=pd.DataFrame({'essay':[essay]})\n",
    "    #return essay_df\n",
    "\n",
    "\n",
    "    def sent2word(x):\n",
    "        x=re.sub(\"[^A-Za-z0-9]\",\" \",x)\n",
    "        words=nltk.word_tokenize(x)\n",
    "        return words\n",
    "\n",
    "    def essay2word(essay):\n",
    "        essay = essay.strip()\n",
    "        tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')\n",
    "        raw = tokenizer.tokenize(essay)\n",
    "        final_words=[]\n",
    "        for i in raw:\n",
    "            if(len(i)>0):\n",
    "                final_words.append(sent2word(i))\n",
    "        return final_words\n",
    "\n",
    "    def noOfWords(essay):\n",
    "        count=0\n",
    "        for i in essay2word(essay):\n",
    "            count=count+len(i)\n",
    "        return count\n",
    "\n",
    "    def noOfChar(essay):\n",
    "        count=0\n",
    "        for i in essay2word(essay):\n",
    "            for j in i:\n",
    "                count=count+len(j)\n",
    "        return count\n",
    "\n",
    "    def avg_word_len(essay):\n",
    "        return noOfChar(essay)/noOfWords(essay)\n",
    "\n",
    "    def noOfSent(essay):\n",
    "        return len(essay2word(essay))\n",
    "\n",
    "    def count_pos(essay):\n",
    "        sentences = essay2word(essay)\n",
    "        noun_count=0\n",
    "        adj_count=0\n",
    "        verb_count=0\n",
    "        adverb_count=0\n",
    "        for i in sentences:\n",
    "            pos_sentence = nltk.pos_tag(i)\n",
    "            for j in pos_sentence:\n",
    "                pos_tag = j[1]\n",
    "                if(pos_tag[0]=='N'):\n",
    "                    noun_count+=1\n",
    "                elif(pos_tag[0]=='V'):\n",
    "                    verb_count+=1\n",
    "                elif(pos_tag[0]=='J'):\n",
    "                    adj_count+=1\n",
    "                elif(pos_tag[0]=='R'):\n",
    "                    adverb_count+=1\n",
    "        return noun_count,verb_count,adj_count,adverb_count\n",
    "\n",
    "    pro_test = essay_df.copy()\n",
    "    pro_test['char_count'] = pro_test['essay'].apply(noOfChar)\n",
    "    pro_test['word_count'] = pro_test['essay'].apply(noOfWords)\n",
    "    pro_test['sent_count'] = pro_test['essay'].apply(noOfSent)\n",
    "    pro_test['avg_word_len'] = pro_test['essay'].apply(avg_word_len)\n",
    "\n",
    "    pro_test['noun_count'], pro_test['adj_count'], pro_test['verb_count'], pro_test['adv_count'] = zip(*pro_test['essay'].map(count_pos))\n",
    "    #pro_data.to_csv(\"Processed_data.csv\")\n",
    "\n",
    "    vectorizer = CountVectorizer(max_features = 10000, ngram_range=(1, 3), stop_words='english')\n",
    "    count_vectorfit = vectorizer.fit(count_fit['clean_essay'])\n",
    "    count_transform=vectorizer.transform(pro_test['essay'])\n",
    "    feature_names = vectorizer.get_feature_names()\n",
    "    x = count_transform.toarray()\n",
    "    X_test = np.concatenate((pro_test.iloc[:, 2:], pd.DataFrame(x)), axis = 1)\n",
    "\n",
    "    model=pickle.load(open(\"svr_pp\",'rb'))\n",
    "    ypred=model.predict(X_test)\n",
    "    return ypred\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Final app"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      
     ]
    }
   ],
   "source": [
    "from __future__ import unicode_literals\n",
    "from flask import Flask,render_template,url_for,request\n",
    "\n",
    "from spacy_summarization import text_summarizer\n",
    "from gensim.summarization import summarize\n",
    "from nltk_summarization import nltk_summarizer\n",
    "import time\n",
    "import spacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "nlp = spacy.load(\"en_core_web_sm\")\n",
    "app = Flask(__name__)\n",
    "\n",
    "# Web Scraping Pkg\n",
    "from bs4 import BeautifulSoup\n",
    "from urllib.request import urlopen\n",
    "#from urllib import urlopen\n",
    "\n",
    "# Sumy Pkg\n",
    "from sumy.parsers.plaintext import PlaintextParser\n",
    "from sumy.nlp.tokenizers import Tokenizer\n",
    "from sumy.summarizers.lex_rank import LexRankSummarizer\n",
    "\n",
    "# Sumy \n",
    "def sumy_summary(docx):\n",
    "    parser = PlaintextParser.from_string(docx,Tokenizer(\"english\"))\n",
    "    lex_summarizer = LexRankSummarizer()\n",
    "    summary = lex_summarizer(parser.document,3)\n",
    "    summary_list = [str(sentence) for sentence in summary]\n",
    "    result = ' '.join(summary_list)\n",
    "    return result\n",
    "\n",
    "\n",
    "# Reading Time\n",
    "def readingTime(mytext):\n",
    "    total_words = len([ token.text for token in nlp(mytext)])\n",
    "    estimatedTime = total_words/200.0\n",
    "    return estimatedTime\n",
    "\n",
    "# Fetch Text From Url\n",
    "def get_text(url):\n",
    "    page = urlopen(url)\n",
    "    soup = BeautifulSoup(page)\n",
    "    fetched_text = ' '.join(map(lambda p:p.text,soup.find_all('p')))\n",
    "    return fetched_text\n",
    "\n",
    "@app.route('/')\n",
    "def index():\n",
    "    return render_template('index.html')\n",
    "\n",
    "@app.route('/analyze',methods=['GET','POST'])\n",
    "def analyze():\n",
    "    start = time.time()\n",
    "    if request.method == 'POST':\n",
    "        rawtext = request.form['rawtext']\n",
    "        final_reading_time = readingTime(rawtext)\n",
    "        final_summary = text_summarizer(rawtext)\n",
    "        summary_reading_time = readingTime(final_summary)\n",
    "        end = time.time()\n",
    "        final_time = end-start\n",
    "    return render_template('index.html',ctext=rawtext,final_summary=final_summary,final_time=final_time,final_reading_time=final_reading_time,summary_reading_time=summary_reading_time)\n",
    "\n",
    "@app.route('/analyze_url',methods=['GET','POST'])\n",
    "def analyze_url():\n",
    "    start = time.time()\n",
    "    if request.method == 'POST':\n",
    "        raw_url = request.form['raw_url']\n",
    "        rawtext = get_text(raw_url)\n",
    "        final_reading_time = readingTime(rawtext)\n",
    "        final_summary = text_summarizer(rawtext)\n",
    "        summary_reading_time = readingTime(final_summary)\n",
    "        end = time.time()\n",
    "        final_time = end-start\n",
    "    return render_template('index.html',ctext=rawtext,final_summary=final_summary,final_time=final_time,final_reading_time=final_reading_time,summary_reading_time=summary_reading_time)\n",
    "\n",
    "\n",
    "\n",
    "@app.route('/compare_summary')\n",
    "def compare_summary():\n",
    "    return render_template('compare_summary.html')\n",
    "\n",
    "#import Pegasus\n",
    "\n",
    "@app.route('/comparer',methods=['GET','POST'])\n",
    "def comparer():\n",
    "    start = time.time()\n",
    "    if request.method == 'POST':\n",
    "        rawtext = request.form['rawtext']\n",
    "        final_reading_time = readingTime(rawtext)\n",
    "        final_summary_spacy = text_summarizer(rawtext)\n",
    "        summary_reading_time = readingTime(final_summary_spacy)\n",
    "        # Gensim Summarizer\n",
    "        final_summary_gensim = summarize(rawtext)\n",
    "        summary_reading_time_gensim = readingTime(final_summary_gensim)\n",
    "        # NLTK\n",
    "        final_summary_nltk = nltk_summarizer(rawtext)\n",
    "        summary_reading_time_nltk = readingTime(final_summary_nltk)\n",
    "#         batch = tokenizer.prepare_seq2seq_batch(rawtext, truncation=True, padding='longest').to(torch_device)\n",
    "#         translated = model.generate(**batch)\n",
    "#         final_summary_pegasus = tokenizer.batch_decode(translated, skip_special_tokens=True)\n",
    "#         summary_reading_time_pegasus = readingTime(final_summary_pegasus)\n",
    "        # Sumy\n",
    "        final_summary_sumy = sumy_summary(rawtext)\n",
    "        summary_reading_time_sumy = readingTime(final_summary_sumy) \n",
    "\n",
    "        end = time.time()\n",
    "        final_time = end-start\n",
    "        return render_template('compare_summary.html',ctext=rawtext,final_summary_spacy=final_summary_spacy,final_summary_gensim=final_summary_gensim,final_summary_nltk=final_summary_nltk,final_time=final_time,final_reading_time=final_reading_time,summary_reading_time=summary_reading_time,summary_reading_time_gensim=summary_reading_time_gensim,final_summary_sumy=final_summary_sumy,summary_reading_time_sumy=summary_reading_time_sumy,summary_reading_time_nltk=summary_reading_time_nltk)\n",
    "\n",
    "\n",
    "@app.route('/grading')\n",
    "def grading():\n",
    "    return render_template('grading.html')\n",
    "\n",
    "@app.route('/grader',methods=['GET','POST'])\n",
    "def grader():\n",
    "    if request.method == 'POST':\n",
    "        essay = request.form['essay']\n",
    "        predicted_score = grade(essay)\n",
    "        return render_template('grading.html',ctext=essay,predicted_score=predicted_score)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " * Serving Flask app \"__main__\" (lazy loading)\n",
      " * Environment: production\n",
      "   WARNING: This is a development server. Do not use it in a production deployment.\n",
      "   Use a production WSGI server instead.\n",
      " * Debug mode: on\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " * Running on http://127.0.0.1:5000/ (Press CTRL+C to quit)\n",
      "127.0.0.1 - - [13/Jul/2021 09:43:38] \"\u001b[37mGET / HTTP/1.1\u001b[0m\" 200 -\n",
      "127.0.0.1 - - [13/Jul/2021 09:43:38] \"\u001b[33mGET /static/css/style.css HTTP/1.1\u001b[0m\" 404 -\n",
      "127.0.0.1 - - [13/Jul/2021 09:43:38] \"\u001b[36mGET /static/css/materialize.css HTTP/1.1\u001b[0m\" 304 -\n",
      "127.0.0.1 - - [13/Jul/2021 09:43:38] \"\u001b[33mGET /backgroundblue.png HTTP/1.1\u001b[0m\" 404 -\n",
      "127.0.0.1 - - [13/Jul/2021 09:43:38] \"\u001b[37mGET /static/js/init.js HTTP/1.1\u001b[0m\" 200 -\n",
      "127.0.0.1 - - [13/Jul/2021 09:43:38] \"\u001b[33mGET /backgroundpurple.png HTTP/1.1\u001b[0m\" 404 -\n",
      "127.0.0.1 - - [13/Jul/2021 09:43:39] \"\u001b[37mGET /static/js/materialize.js HTTP/1.1\u001b[0m\" 200 -\n",
      "127.0.0.1 - - [13/Jul/2021 09:43:39] \"\u001b[33mGET /backgroundblue.png HTTP/1.1\u001b[0m\" 404 -\n",
      "127.0.0.1 - - [13/Jul/2021 09:43:39] \"\u001b[33mGET /backgroundpurple.png HTTP/1.1\u001b[0m\" 404 -\n",
      "127.0.0.1 - - [13/Jul/2021 09:43:39] \"\u001b[33mGET /favicon.ico HTTP/1.1\u001b[0m\" 404 -\n",
      "127.0.0.1 - - [13/Jul/2021 09:43:41] \"\u001b[37mGET / HTTP/1.1\u001b[0m\" 200 -\n",
      "127.0.0.1 - - [13/Jul/2021 09:43:41] \"\u001b[33mGET /static/css/style.css HTTP/1.1\u001b[0m\" 404 -\n",
      "127.0.0.1 - - [13/Jul/2021 09:43:41] \"\u001b[33mGET /backgroundblue.png HTTP/1.1\u001b[0m\" 404 -\n",
      "127.0.0.1 - - [13/Jul/2021 09:43:41] \"\u001b[33mGET /backgroundpurple.png HTTP/1.1\u001b[0m\" 404 -\n",
      "127.0.0.1 - - [13/Jul/2021 09:43:44] \"\u001b[37mGET /compare_summary HTTP/1.1\u001b[0m\" 200 -\n",
      "127.0.0.1 - - [13/Jul/2021 09:43:44] \"\u001b[33mGET /static/css/style.css HTTP/1.1\u001b[0m\" 404 -\n",
      "127.0.0.1 - - [13/Jul/2021 09:43:44] \"\u001b[33mGET /backgroundblue.png HTTP/1.1\u001b[0m\" 404 -\n",
      "127.0.0.1 - - [13/Jul/2021 09:43:44] \"\u001b[33mGET /backgroundpurple.png HTTP/1.1\u001b[0m\" 404 -\n",
      "127.0.0.1 - - [13/Jul/2021 09:43:48] \"\u001b[37mGET / HTTP/1.1\u001b[0m\" 200 -\n",
      "127.0.0.1 - - [13/Jul/2021 09:43:48] \"\u001b[33mGET /static/css/style.css HTTP/1.1\u001b[0m\" 404 -\n",
      "127.0.0.1 - - [13/Jul/2021 09:43:48] \"\u001b[33mGET /backgroundblue.png HTTP/1.1\u001b[0m\" 404 -\n",
      "127.0.0.1 - - [13/Jul/2021 09:43:48] \"\u001b[33mGET /backgroundpurple.png HTTP/1.1\u001b[0m\" 404 -\n"
     ]
    }
   ],
   "source": [
    "app.run(debug=True, use_reloader=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}