Proceso_Datos.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Objetivo"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Me interesa saber la distribución de los costos de productos adquiridos por la secretaría de salud. Esto es, ¿Cuánto de más o cuánto de menos se ha estado pagando? ¿Podemos detectar corrupción? ¿Comportamiento por distribuidores?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Obtener la data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "No logre encontrar los datos abiertos que alimentan: http://compras.imss.gob.mx/?P=search_alt Voy a proceder a scrappearlo."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Alto, están las compras consolidadas: Según HHealth (Gracias!) no son todas las compras. pero está chido para hacerme de un diccionario.\n",
    "http://www.imss.gob.mx/compraconsolidada2016"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import requests as r\n",
    "import os\n",
    "import json\n",
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def products(field,page):\n",
    "    result = os.popen(\"curl 'http://buscador.compras.imss.gob.mx/index.php' -H 'Pragma: no-cache' -H 'Origin: http://buscador.compras.imss.gob.mx' -H 'Accept-Encoding: gzip, deflate' -H 'Accept-Language: es-ES,es;q=0.8,en;q=0.6' -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36' -H 'Content-Type: application/x-www-form-urlencoded' -H 'Accept: */*' -H 'Cache-Control: no-cache' -H 'X-Requested-With: WAJAF::Ajax - WebAbility(r) v5' -H 'Cookie: __utma=37030712.222899114.1488655887.1488656746.1488656746.1; __utmb=37030712.1.10.1488656746; __utmc=37030712; __utmz=37030712.1488656746.1.1.utmcsr=(direct)|utmccn=(direct)|utmcmd=(none); __asc=9819a67f15a9ace2b0599c2b83a; __auc=9819a67f15a9ace2b0599c2b83a; _ga=GA1.3.222899114.1488655887' -H 'Connection: keep-alive' -H 'Referer: http://buscador.compras.imss.gob.mx/wrap/index.html' --data '&type=compras&message=X&filtered=1&descripcion=\"+field+\"&proveedor=&numcompra=&delegacion=values%3D&fecha=min%3D%3Bmax%3D&procedimiento=values%3D&exact=false&numperpage=20&page=\" + str(page)+\"&order=fecha%20desc' --compressed\").read()\n",
    "    return(json.loads(result))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "first_page  =search_page(\" \",1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['near', 'page', 'exact', 'time', 'quantity', 'id', 'data', 'numperpage', 'nquery'])"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "first_page[\"result\"].keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2136708"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "int(first_page[\"result\"][\"quantity\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'clave': '12086929',\n",
       " 'delegacion': None,\n",
       " 'descripcion': '176',\n",
       " 'fecha': '',\n",
       " 'monto': '0.00',\n",
       " 'numcompra': None,\n",
       " 'procedimiento': None,\n",
       " 'proveedor': None}"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "first_page[\"result\"][\"data\"][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def transaction(transaction_number):\n",
    "    transaction_page = \"http://compras.imss.gob.mx/?P=imsscomprofich&f=\"+ str(transaction_number)\n",
    "    content = r.get(transaction_page)\n",
    "    soup = BeautifulSoup(content.content)\n",
    "    local_data = {}\n",
    "    for span in soup.findAll(\"span\", {\"class\":\"txtdesccaja\"}):\n",
    "        split_span = span.text.replace(\"\\n\",\"\").strip().split(\":\")\n",
    "        name = split_span[0]\n",
    "        content = \" \".join(split_span[1:])\n",
    "        local_data[name]=content\n",
    "    return(local_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'# Factura': ' D17159',\n",
       " 'COMERCIAL HOSPITALARIA S.A DE C.V.': '',\n",
       " 'Cantidad recibida': ' 30',\n",
       " 'Cantidad solicitada': ' 30',\n",
       " 'Delegación del IMSS': ' Delegación Sonora',\n",
       " 'Descripción': ' TIRA REACTIVA PARA DETERMINACION SEMICUANTITATIVA DE GLUCOSA EN SANGRE CON   LIMITES DE DETECCION QUE VAN DE 20 A 800 MG/DL. FRASCO CON 50 TIRAS. TA.',\n",
       " 'Estado de la República': ' Sonora',\n",
       " 'Fecha factura': ' 17/02/17',\n",
       " 'IVA': ' 16',\n",
       " 'No. Procedimiento': ' 230',\n",
       " 'Precio': ' 294.00',\n",
       " 'Procedimiento de compra': 'Compra emergente en unidades de servicio',\n",
       " 'RFC': ' CHO011105F23 [Ver ficha del proveedor]',\n",
       " 'Subprocedimiento de compra': ' N/A',\n",
       " 'Unidad': ' TRA'}"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "transaction(12086888)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'# Factura': ' D17094',\n",
       " 'COMERCIAL HOSPITALARIA S.A DE C.V.': '',\n",
       " 'Cantidad recibida': ' 15',\n",
       " 'Cantidad solicitada': ' 15',\n",
       " 'Delegación del IMSS': ' Delegación Sonora',\n",
       " 'Descripción': ' RIFAMPICINA SUSPENSION ORAL CADA 5 ML CONTIENEN  RIFAMPICINA 100 MG ENVASE CON   120 ML Y DOSIFICADOR.',\n",
       " 'Estado de la República': ' Sonora',\n",
       " 'Fecha factura': ' 13/02/17',\n",
       " 'IVA': ' No contiene datos',\n",
       " 'No. Procedimiento': ' 171',\n",
       " 'Precio': ' 438.90',\n",
       " 'Procedimiento de compra': 'Compra emergente en unidades de servicio',\n",
       " 'RFC': ' CHO011105F23 [Ver ficha del proveedor]',\n",
       " 'Subprocedimiento de compra': ' N/A',\n",
       " 'Unidad': ' ENV'}"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "transaction(12086887)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def get_all_data():\n",
    "    first_page = search_page(\" \",1)\n",
    "    data = first_page[\"result\"][\"data\"]\n",
    "    hits = int(first_page[\"result\"][\"quantity\"])\n",
    "    pages = floor(hits/20) + 1\n",
    "    i = 1\n",
    "    while i<=pages:\n",
    "        try:\n",
    "            page_data = search_page(\" \",i)\n",
    "            data = data + page_data[\"result\"][\"data\"]\n",
    "            i = i +1\n",
    "        except:\n",
    "            None\n",
    "    return(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "106836"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import math\n",
    "math.floor(int(results[\"result\"][\"quantity\"])/20)+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'result': {'data': [{'clave': '12086929',\n",
       "    'delegacion': None,\n",
       "    'descripcion': '176',\n",
       "    'fecha': '',\n",
       "    'monto': '0.00',\n",
       "    'numcompra': None,\n",
       "    'procedimiento': None,\n",
       "    'proveedor': None},\n",
       "   {'clave': '12086928',\n",
       "    'delegacion': None,\n",
       "    'descripcion': '125',\n",
       "    'fecha': '',\n",
       "    'monto': '0.00',\n",
       "    'numcompra': None,\n",
       "    'procedimiento': None,\n",
       "    'proveedor': None},\n",
       "   {'clave': '12086927',\n",
       "    'delegacion': None,\n",
       "    'descripcion': '9',\n",
       "    'fecha': '',\n",
       "    'monto': '0.00',\n",
       "    'numcompra': None,\n",
       "    'procedimiento': None,\n",
       "    'proveedor': None},\n",
       "   {'clave': '12086926',\n",
       "    'delegacion': None,\n",
       "    'descripcion': '90',\n",
       "    'fecha': '',\n",
       "    'monto': '0.00',\n",
       "    'numcompra': None,\n",
       "    'procedimiento': None,\n",
       "    'proveedor': None},\n",
       "   {'clave': '12086925',\n",
       "    'delegacion': None,\n",
       "    'descripcion': '96',\n",
       "    'fecha': '',\n",
       "    'monto': '0.00',\n",
       "    'numcompra': None,\n",
       "    'procedimiento': None,\n",
       "    'proveedor': None},\n",
       "   {'clave': '12086924',\n",
       "    'delegacion': None,\n",
       "    'descripcion': '107',\n",
       "    'fecha': '',\n",
       "    'monto': '0.00',\n",
       "    'numcompra': None,\n",
       "    'procedimiento': None,\n",
       "    'proveedor': None},\n",
       "   {'clave': '12086923',\n",
       "    'delegacion': None,\n",
       "    'descripcion': '80',\n",
       "    'fecha': '',\n",
       "    'monto': '0.00',\n",
       "    'numcompra': None,\n",
       "    'procedimiento': None,\n",
       "    'proveedor': None},\n",
       "   {'clave': '12086922',\n",
       "    'delegacion': None,\n",
       "    'descripcion': '71',\n",
       "    'fecha': '',\n",
       "    'monto': '0.00',\n",
       "    'numcompra': None,\n",
       "    'procedimiento': None,\n",
       "    'proveedor': None},\n",
       "   {'clave': '12086921',\n",
       "    'delegacion': None,\n",
       "    'descripcion': '81',\n",
       "    'fecha': '',\n",
       "    'monto': '0.00',\n",
       "    'numcompra': None,\n",
       "    'procedimiento': None,\n",
       "    'proveedor': None},\n",
       "   {'clave': '12086920',\n",
       "    'delegacion': None,\n",
       "    'descripcion': '79',\n",
       "    'fecha': '',\n",
       "    'monto': '0.00',\n",
       "    'numcompra': None,\n",
       "    'procedimiento': None,\n",
       "    'proveedor': None},\n",
       "   {'clave': '12086919',\n",
       "    'delegacion': None,\n",
       "    'descripcion': '20',\n",
       "    'fecha': '',\n",
       "    'monto': '0.00',\n",
       "    'numcompra': None,\n",
       "    'procedimiento': None,\n",
       "    'proveedor': None},\n",
       "   {'clave': '12086918',\n",
       "    'delegacion': None,\n",
       "    'descripcion': '73',\n",
       "    'fecha': '',\n",
       "    'monto': '0.00',\n",
       "    'numcompra': None,\n",
       "    'procedimiento': None,\n",
       "    'proveedor': None},\n",
       "   {'clave': '12086917',\n",
       "    'delegacion': None,\n",
       "    'descripcion': '57',\n",
       "    'fecha': '',\n",
       "    'monto': '0.00',\n",
       "    'numcompra': None,\n",
       "    'procedimiento': None,\n",
       "    'proveedor': None},\n",
       "   {'clave': '12086916',\n",
       "    'delegacion': None,\n",
       "    'descripcion': '176',\n",
       "    'fecha': '',\n",
       "    'monto': '0.00',\n",
       "    'numcompra': None,\n",
       "    'procedimiento': None,\n",
       "    'proveedor': None},\n",
       "   {'clave': '12086915',\n",
       "    'delegacion': None,\n",
       "    'descripcion': '98',\n",
       "    'fecha': '',\n",
       "    'monto': '0.00',\n",
       "    'numcompra': None,\n",
       "    'procedimiento': None,\n",
       "    'proveedor': None},\n",
       "   {'clave': '12086914',\n",
       "    'delegacion': None,\n",
       "    'descripcion': '105',\n",
       "    'fecha': '',\n",
       "    'monto': '0.00',\n",
       "    'numcompra': None,\n",
       "    'procedimiento': None,\n",
       "    'proveedor': None},\n",
       "   {'clave': '12086913',\n",
       "    'delegacion': None,\n",
       "    'descripcion': '105',\n",
       "    'fecha': '',\n",
       "    'monto': '0.00',\n",
       "    'numcompra': None,\n",
       "    'procedimiento': None,\n",
       "    'proveedor': None},\n",
       "   {'clave': '12086912',\n",
       "    'delegacion': None,\n",
       "    'descripcion': '104',\n",
       "    'fecha': '',\n",
       "    'monto': '0.00',\n",
       "    'numcompra': None,\n",
       "    'procedimiento': None,\n",
       "    'proveedor': None},\n",
       "   {'clave': '12086911',\n",
       "    'delegacion': None,\n",
       "    'descripcion': '108',\n",
       "    'fecha': '',\n",
       "    'monto': '0.00',\n",
       "    'numcompra': None,\n",
       "    'procedimiento': None,\n",
       "    'proveedor': None},\n",
       "   {'clave': '12086910',\n",
       "    'delegacion': None,\n",
       "    'descripcion': '16',\n",
       "    'fecha': '',\n",
       "    'monto': '0.00',\n",
       "    'numcompra': None,\n",
       "    'procedimiento': None,\n",
       "    'proveedor': None}],\n",
       "  'exact': 'false',\n",
       "  'id': 'compras',\n",
       "  'near': False,\n",
       "  'nquery': None,\n",
       "  'numperpage': '20',\n",
       "  'page': '1',\n",
       "  'quantity': '2136708',\n",
       "  'time': 4}}"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "results = search_page(\" \",3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'result': {'data': [{'clave': '12086889',\n",
       "    'delegacion': None,\n",
       "    'descripcion': '196',\n",
       "    'fecha': '',\n",
       "    'monto': '0.00',\n",
       "    'numcompra': None,\n",
       "    'procedimiento': None,\n",
       "    'proveedor': None},\n",
       "   {'clave': '12086888',\n",
       "    'delegacion': None,\n",
       "    'descripcion': '230',\n",
       "    'fecha': '',\n",
       "    'monto': '0.00',\n",
       "    'numcompra': None,\n",
       "    'procedimiento': None,\n",
       "    'proveedor': None},\n",
       "   {'clave': '12086887',\n",
       "    'delegacion': None,\n",
       "    'descripcion': '171',\n",
       "    'fecha': '',\n",
       "    'monto': '0.00',\n",
       "    'numcompra': None,\n",
       "    'procedimiento': None,\n",
       "    'proveedor': None},\n",
       "   {'clave': '12086886',\n",
       "    'delegacion': None,\n",
       "    'descripcion': '231',\n",
       "    'fecha': '',\n",
       "    'monto': '0.00',\n",
       "    'numcompra': None,\n",
       "    'procedimiento': None,\n",
       "    'proveedor': None},\n",
       "   {'clave': '12086885',\n",
       "    'delegacion': None,\n",
       "    'descripcion': '224',\n",
       "    'fecha': '',\n",
       "    'monto': '0.00',\n",
       "    'numcompra': None,\n",
       "    'procedimiento': None,\n",
       "    'proveedor': None},\n",
       "   {'clave': '12086884',\n",
       "    'delegacion': None,\n",
       "    'descripcion': '64',\n",
       "    'fecha': '',\n",
       "    'monto': '0.00',\n",
       "    'numcompra': None,\n",
       "    'procedimiento': None,\n",
       "    'proveedor': None},\n",
       "   {'clave': '12086883',\n",
       "    'delegacion': None,\n",
       "    'descripcion': '55',\n",
       "    'fecha': '',\n",
       "    'monto': '0.00',\n",
       "    'numcompra': None,\n",
       "    'procedimiento': None,\n",
       "    'proveedor': None},\n",
       "   {'clave': '12086882',\n",
       "    'delegacion': None,\n",
       "    'descripcion': '52',\n",
       "    'fecha': '',\n",
       "    'monto': '0.00',\n",
       "    'numcompra': None,\n",
       "    'procedimiento': None,\n",
       "    'proveedor': None},\n",
       "   {'clave': '12086881',\n",
       "    'delegacion': None,\n",
       "    'descripcion': '258',\n",
       "    'fecha': '',\n",
       "    'monto': '0.00',\n",
       "    'numcompra': None,\n",
       "    'procedimiento': None,\n",
       "    'proveedor': None},\n",
       "   {'clave': '12086880',\n",
       "    'delegacion': None,\n",
       "    'descripcion': '267',\n",
       "    'fecha': '',\n",
       "    'monto': '0.00',\n",
       "    'numcompra': None,\n",
       "    'procedimiento': None,\n",
       "    'proveedor': None},\n",
       "   {'clave': '12086879',\n",
       "    'delegacion': None,\n",
       "    'descripcion': '8',\n",
       "    'fecha': '',\n",
       "    'monto': '0.00',\n",
       "    'numcompra': None,\n",
       "    'procedimiento': None,\n",
       "    'proveedor': None},\n",
       "   {'clave': '12086878',\n",
       "    'delegacion': None,\n",
       "    'descripcion': '114',\n",
       "    'fecha': '',\n",
       "    'monto': '0.00',\n",
       "    'numcompra': None,\n",
       "    'procedimiento': None,\n",
       "    'proveedor': None},\n",
       "   {'clave': '12086877',\n",
       "    'delegacion': None,\n",
       "    'descripcion': '88',\n",
       "    'fecha': '',\n",
       "    'monto': '0.00',\n",
       "    'numcompra': None,\n",
       "    'procedimiento': None,\n",
       "    'proveedor': None},\n",
       "   {'clave': '12086876',\n",
       "    'delegacion': None,\n",
       "    'descripcion': '91',\n",
       "    'fecha': '',\n",
       "    'monto': '0.00',\n",
       "    'numcompra': None,\n",
       "    'procedimiento': None,\n",
       "    'proveedor': None},\n",
       "   {'clave': '12086875',\n",
       "    'delegacion': None,\n",
       "    'descripcion': '52',\n",
       "    'fecha': '',\n",
       "    'monto': '0.00',\n",
       "    'numcompra': None,\n",
       "    'procedimiento': None,\n",
       "    'proveedor': None},\n",
       "   {'clave': '12086874',\n",
       "    'delegacion': None,\n",
       "    'descripcion': '74',\n",
       "    'fecha': '',\n",
       "    'monto': '0.00',\n",
       "    'numcompra': None,\n",
       "    'procedimiento': None,\n",
       "    'proveedor': None},\n",
       "   {'clave': '12086873',\n",
       "    'delegacion': None,\n",
       "    'descripcion': '67',\n",
       "    'fecha': '',\n",
       "    'monto': '0.00',\n",
       "    'numcompra': None,\n",
       "    'procedimiento': None,\n",
       "    'proveedor': None},\n",
       "   {'clave': '12086872',\n",
       "    'delegacion': None,\n",
       "    'descripcion': '151',\n",
       "    'fecha': '',\n",
       "    'monto': '0.00',\n",
       "    'numcompra': None,\n",
       "    'procedimiento': None,\n",
       "    'proveedor': None},\n",
       "   {'clave': '12086871',\n",
       "    'delegacion': None,\n",
       "    'descripcion': '146',\n",
       "    'fecha': '',\n",
       "    'monto': '0.00',\n",
       "    'numcompra': None,\n",
       "    'procedimiento': None,\n",
       "    'proveedor': None},\n",
       "   {'clave': '12086870',\n",
       "    'delegacion': None,\n",
       "    'descripcion': '120',\n",
       "    'fecha': '',\n",
       "    'monto': '0.00',\n",
       "    'numcompra': None,\n",
       "    'procedimiento': None,\n",
       "    'proveedor': None}],\n",
       "  'exact': 'false',\n",
       "  'id': 'compras',\n",
       "  'near': False,\n",
       "  'nquery': None,\n",
       "  'numperpage': '20',\n",
       "  'page': '3',\n",
       "  'quantity': '2136708',\n",
       "  'time': 33}}"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}