ipcc-wip.ipynb

{
 "cells": [
  {
   "cell_type": "raw",
   "metadata": {
    "vscode": {
     "languageId": "raw"
    }
   },
   "source": [
    "---\n",
    "title: \"IPCC WIP\"\n",
    "jupyter: python3\n",
    "code-fold: true \n",
    "execute:\n",
    "  echo: false\n",
    "  output: asis \n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "import sys\n",
    "import time\n",
    "from SPARQLWrapper import SPARQLWrapper, JSON\n",
    "import requests\n",
    "from PIL import Image\n",
    "import html\n",
    "from markdownify import markdownify \n",
    "\n",
    "endpoint_url = \"https://computational-publishing-service.wikibase.cloud/query/sparql\"\n",
    "\n",
    "query_txt = \"\"\"PREFIX cps: <https://computational-publishing-service.wikibase.cloud/entity/>\n",
    "PREFIX cpss: <https://computational-publishing-service.wikibase.cloud/entity/statement/>\n",
    "PREFIX cpsv: <https://computational-publishing-service.wikibase.cloud/value/>\n",
    "PREFIX cpspt: <https://computational-publishing-service.wikibase.cloud/prop/direct/>\n",
    "PREFIX cpsp: <https://computational-publishing-service.wikibase.cloud/prop/>\n",
    "PREFIX cpsps: <https://computational-publishing-service.wikibase.cloud/prop/statement/>\n",
    "PREFIX cpspq: <https://computational-publishing-service.wikibase.cloud/prop/qualifier/>\n",
    "\n",
    "SELECT ?textItem ?kuratorLabel ?textUrl\n",
    "WHERE\n",
    "{\n",
    "  <placeholder>\n",
    "  ?textItem cpsp:P46 ?kuratorStatement. \n",
    "  ?kuratorStatement cpsps:P46 ?kuratorItem. \n",
    "  ?kuratorItem rdfs:label ?kuratorLabel.\n",
    "  ?textItem cpsp:P57 ?urlstatement. \n",
    "  ?urlstatement cpsps:P57 ?textUrl. \n",
    "}\"\"\"\n",
    "\n",
    "query_img = \"\"\"PREFIX cps: <https://computational-publishing-service.wikibase.cloud/entity/>\n",
    "PREFIX cpss: <https://computational-publishing-service.wikibase.cloud/entity/statement/>\n",
    "PREFIX cpsv: <https://computational-publishing-service.wikibase.cloud/value/>\n",
    "PREFIX cpspt: <https://computational-publishing-service.wikibase.cloud/prop/direct/>\n",
    "PREFIX cpsp: <https://computational-publishing-service.wikibase.cloud/prop/>\n",
    "PREFIX cpsps: <https://computational-publishing-service.wikibase.cloud/prop/statement/>\n",
    "PREFIX cpspq: <https://computational-publishing-service.wikibase.cloud/prop/qualifier/>\n",
    "\n",
    "SELECT DISTINCT ?itemLabel ?itemDescr ?imgItem ?imgUrl ?publishDate \n",
    "WHERE\n",
    "{\n",
    "  ?imgItem cpsp:P107 ?urlStatement. \n",
    "  ?urlStatement cpsps:P107 ?imgUrl. \n",
    "  ?imgItem cpsp:P60 ?dateStatement. \n",
    "  ?dateStatement cpsps:P60 ?publishDate.\n",
    "  ?imgItem cpsp:P6 ?partOfStatement.\n",
    "  ?partOfStatement cpsps:P6 ?partOfItem.\n",
    "  <placeholder> \n",
    "  SERVICE wikibase:label {\n",
    "      bd:serviceParam wikibase:language \"de,en\".\n",
    "      ?imgItem rdfs:label ?itemLabel.\n",
    "      ?imgItem schema:description ?itemDescr.\n",
    "    }\n",
    "}\"\"\"\n",
    "query_graph = \"\"\"PREFIX cps: <https://computational-publishing-service.wikibase.cloud/entity/>\n",
    "PREFIX cpss: <https://computational-publishing-service.wikibase.cloud/entity/statement/>\n",
    "PREFIX cpsv: <https://computational-publishing-service.wikibase.cloud/value/>\n",
    "PREFIX cpspt: <https://computational-publishing-service.wikibase.cloud/prop/direct/>\n",
    "PREFIX cpsp: <https://computational-publishing-service.wikibase.cloud/prop/>\n",
    "PREFIX cpsps: <https://computational-publishing-service.wikibase.cloud/prop/statement/>\n",
    "PREFIX cpspq: <https://computational-publishing-service.wikibase.cloud/prop/qualifier/>\n",
    "\n",
    "SELECT ?x ?y\n",
    "WHERE\n",
    "{\n",
    "  ?a cpsp:P2 ?c.\n",
    "  ?c cpsps:P2 ?d.\n",
    "  ?a rdfs:label ?x.\n",
    "  ?d rdfs:label ?y.\n",
    "\n",
    "}LIMIT 1\"\"\"\n",
    "\n",
    "query_graph2 = \"\"\"\n",
    "SELECT ?a ?b ?c\n",
    "WHERE\n",
    "{\n",
    "  ?a rdfs:label ?c\n",
    "}LIMIT 100\"\"\"\n",
    "\n",
    "\n",
    "def run_query(endpoint_url, query):\n",
    "    user_agent = \"WDQS-example Python/%s.%s\" % (sys.version_info[0], sys.version_info[1])\n",
    "    # TODO adjust user agent; see https://w.wiki/CX6\n",
    "    sparql = SPARQLWrapper(endpoint_url, agent=user_agent)\n",
    "    sparql.setQuery(query)\n",
    "    sparql.setMethod(\"POST\") #this NEEDS to be added to get results (not included in the wikibase python example code)\n",
    "    sparql.setReturnFormat(JSON)\n",
    "    return sparql.query().convert()\n",
    "\n",
    "def get_text(textitem_id):\n",
    "    q = \"\"\n",
    "    if textitem_id:\n",
    "        q = query_txt.replace(\"<placeholder>\", \"cps:\"+textitem_id+\" cpsp:P46 ?kuratorStatement.\")\n",
    "    else:\n",
    "        q = query_txt.replace(\"<placeholder>\",\"\")\n",
    "\n",
    "    results_txt = run_query(endpoint_url, q)\n",
    "    for item in results_txt[\"results\"][\"bindings\"]:\n",
    "        print('Wikibase link: ' + '[' + item['textItem']['value'] + ']' + '(' + item['textItem']['value'] + ')' + '\\n')\n",
    "        print('Kurator: ' + item['kuratorLabel']['value'] + '\\n')\n",
    "        headers = {'User-Agent': 'Ex_Books_conference_bot/0.0 (https://github.com/SimonXIX/Experimental_Books_workshop; ad7588@coventry.ac.uk)'}\n",
    "        r = requests.get(item['textUrl']['value'], headers=headers, stream=True)\n",
    "        text = str(r.text) #changed from r.content\n",
    "        text = text.removeprefix(\"<!DOCTYPE html>\") #changed from \"b'<!DOCTYPE html>\"\n",
    "        print(markdownify(text)) #changed\n",
    "\n",
    "def get_delay(date):\n",
    "    try:\n",
    "        date = datetime.datetime.strptime(date, '%a, %d %b %Y %H:%M:%S GMT')\n",
    "        timeout = int((date - datetime.datetime.now()).total_seconds())\n",
    "    except ValueError:\n",
    "        timeout = int(date)\n",
    "    return timeout\n",
    "\n",
    "def fetch_image_by_url(url, headers):\n",
    "    r = requests.get(url, headers=headers, stream=True)\n",
    "    if r.status_code == 200:\n",
    "        im = Image.open(r.raw)\n",
    "        return im\n",
    "    if r.status_code == 500:\n",
    "        return None\n",
    "    if r.status_code == 403:\n",
    "        return None\n",
    "    if r.status_code == 429:\n",
    "        timeout = get_delay(r.headers['retry-after'])\n",
    "        print('Timeout {} m {} s'.format(timeout // 60, timeout % 60))\n",
    "        time.sleep(timeout)\n",
    "        fetch_image_by_url(url, headers)\n",
    "\n",
    "def get_img(partOfItem_id):\n",
    "    q = \"\"\n",
    "    if partOfItem_id:\n",
    "        q = query_img.replace(\"<placeholder>\", \"?partOfStatement cpsps:P6 cps:\"+partOfItem_id+\".\")\n",
    "    else:\n",
    "        q = query_img.replace(\"<placeholder>\",\"\")\n",
    "    results_img = run_query(endpoint_url, q)\n",
    "    for item in results_img[\"results\"][\"bindings\"]:    \n",
    "      #print(item)\n",
    "      print('Wikibase link: ' + '[' + item['imgItem']['value'] + ']' + '(' + item['imgItem']['value'] + ')' + '\\n')\n",
    "      print('Title: ' + item['itemLabel']['value'] + '\\n')\n",
    "      print('Year: ' + item['publishDate']['value'] + '\\n')\n",
    "      print('Description: ' + html.unescape(item['itemDescr']['value']) + '\\n')\n",
    "\n",
    "      # get image from image URL and display resized version\n",
    "      image_url=item['imgUrl']['value']\n",
    "      headers = {'User-Agent': 'Ex_Books_conference_bot/0.0 (https://github.com/SimonXIX/Experimental_Books_workshop; ad7588@coventry.ac.uk)'}\n",
    "      im = fetch_image_by_url(image_url, headers)\n",
    "      im.thumbnail((500, 500), Image.Resampling.LANCZOS)\n",
    "      display(im)\n",
    "      print('\\n\\n')\n",
    "\n",
    "def get_graph():\n",
    "    import VizKG.visualize as vkg\n",
    "    results_graph1 = run_query(endpoint_url, query_graph)\n",
    "    #print(results_graph1)\n",
    "    #print('---')\n",
    "    results_graph2 = run_query(endpoint_url, query_graph2)\n",
    "    #print(results_graph2)\n",
    "\n",
    "    chart = vkg(sparql_query=query_graph2, sparql_service_url=endpoint_url, chart='wordcloud')\n",
    "    chart.plot()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_text(\"Q209\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_graph()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}