scraper.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "from lxml import etree, html\n",
    "import time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Helper function to generate Links using year and stock"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_url(stock, page, year):\n",
    "    return 'http://www.moneycontrol.com/stocks/company_info/stock_news.php?sc_id={0}&scat=&pageno={1}&next=0&durationType=Y&duration=1&news_type=&Year={2}'.format(stock, page, year)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extracting date from string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fix_date(s):\n",
    "    time = ''\n",
    "    date = ''\n",
    "    end = 0\n",
    "    flag = 0\n",
    "    for i, x in enumerate(s):\n",
    "        if x == '|':\n",
    "            if flag == 0:\n",
    "                time = s[0:i].strip()\n",
    "                end = i+1\n",
    "                flag = 1\n",
    "            else:\n",
    "                date = s[end:i].strip()\n",
    "                return date, time\n",
    "    \n",
    "    date = s[end: ]\n",
    "    return date, time"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extracting stock news and date from html using BeautifulSoup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "stock = 'IT'\n",
    "ans = []\n",
    "xpathselector = '//*[@id=\"mc_mainWrapper\"]/div[3]/div[2]/div[3]/div[2]/div[2]/div/div[3]/div[1]/div[2]/div[2]/a[1]'\n",
    "headers = {\n",
    "    'User-Agent': 'Mozilla/5.0',\n",
    "}\n",
    "\n",
    "for year in range(2018, 2010, -1):\n",
    "    count=0\n",
    "    for i in range(1, 30):\n",
    "        if count==3:break\n",
    "        res = requests.get(get_url(stock, i, year), headers)\n",
    "\n",
    "        if not res.status_code == 200:\n",
    "            print('oh noooooo')\n",
    "            break\n",
    "\n",
    "        content = res.content\n",
    "        soup = BeautifulSoup(content, \"lxml\")\n",
    "        arr = soup.select('.MT15 .FL a strong')\n",
    "        p = soup.select('.MT15 .FL p.a_10dgry')\n",
    "        #print('wassss')\n",
    "        texts = []\n",
    "        if len(arr) == 0:\n",
    "            count+=1\n",
    "            #print(p, arr, content)\n",
    "            #print('done with arrrrrr')\n",
    "            continue\n",
    "            \n",
    "        if len(p) == 0:\n",
    "            #print(arr)\n",
    "            continue\n",
    "            \n",
    "        else:\n",
    "            arrText = [x.text for x in arr]\n",
    "            pText = [fix_date(x.text) for x in p]\n",
    "            texts = [{\n",
    "                'text': x[0],\n",
    "                'date': x[1][0],\n",
    "                'time': x[1][1],\n",
    "            } for x in zip(arrText, pText)]\n",
    "        print('{}-{}'.format(year, i))\n",
    "#         time.sleep(2)\n",
    "        ans.extend(texts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2410"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(ans)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extracted data from html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'date': '16 Mar 2018',\n",
       "  'text': 'Infosys to announce March quarter earnings, final dividend on April 13',\n",
       "  'time': '10.17 am'},\n",
       " {'date': '16 Mar 2018',\n",
       "  'text': 'Infosys to open technology & innovation hub in Connecticut',\n",
       "  'time': '9.28 am'},\n",
       " {'date': '16 Mar 2018',\n",
       "  'text': 'Infosys to consider March quarter results on April 13, 2018',\n",
       "  'time': '9.28 am'},\n",
       " {'date': '16 Mar 2018',\n",
       "  'text': 'Stocks in the news: JP Associates, Aurobindo Pharma, Dilip Buildcon, Infosys, Gravita',\n",
       "  'time': '7.43 am'},\n",
       " {'date': '13 Mar 2018',\n",
       "  'text': 'Nifty likely to head towards 10,440-10,470; Infosys can give 5% return in short term',\n",
       "  'time': '8.03 am'},\n",
       " {'date': '12 Mar 2018',\n",
       "  'text': 'Buy United Spirits, Jubilant Foodworks, NIIT Technologies: Ashwani Gujral',\n",
       "  'time': '12.22 pm'},\n",
       " {'date': '09 Mar 2018',\n",
       "  'text': 'Betting on tech theme? Top 4 stocks which investors could add in their portfolio',\n",
       "  'time': '1.25 pm'},\n",
       " {'date': '09 Mar 2018',\n",
       "  'text': 'Stocks in the news: Infosys, Vedanta, TCS, GTL Infra, IDFC Bank, Filatex India, SpiceJet',\n",
       "  'time': '7.55 am'},\n",
       " {'date': '08 Mar 2018',\n",
       "  'text': 'Buy Infosys; short BPCL, M&M Financial Services: Sudarshan Sukhani',\n",
       "  'time': '12.15 pm'},\n",
       " {'date': '07 Mar 2018',\n",
       "  'text': 'Approach market with caution; focus on fundamentals, earnings: Gautam Duggad',\n",
       "  'time': '12.21 pm'}]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ans[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "thefile = open('/home/jitendra/Desktop/audio/IT.txt', 'w')\n",
    "for item in ans:\n",
    "  thefile.write(\"%s\\n\" % item)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_list = list(ans)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(new_list)\n",
    "# df.apply(lambda x: str.replace(r^'/;|/'))\n",
    "df.to_csv('/home/jitendra/Desktop/audio/IT.csv', index=False, sep='|')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}