From a42c685de3b4fb558a8f77a119f4aae0af313ab2 Mon Sep 17 00:00:00 2001
From: birgits <birgit.schachler@rl-institut.de>
Date: Mon, 30 Oct 2023 15:11:44 +0100
Subject: [PATCH] Bug fix beautiful soup not needed anymore as github returns
 json instead of html

---
 examples/electromobility_example.ipynb | 22 +++++++++++-----------
 rtd_requirements.txt                   |  1 -
 setup.py                               |  1 -
 3 files changed, 11 insertions(+), 13 deletions(-)

diff --git a/examples/electromobility_example.ipynb b/examples/electromobility_example.ipynb
index c2170bf8..70738b6a 100644
--- a/examples/electromobility_example.ipynb
+++ b/examples/electromobility_example.ipynb
@@ -43,7 +43,7 @@
    "outputs": [],
    "source": [
     "import os\n",
-    "\n",
+    "import json\n",
     "import geopandas as gpd\n",
     "import pandas as pd\n",
     "import requests\n",
@@ -53,7 +53,6 @@
     "\n",
     "from copy import deepcopy\n",
     "from pathlib import Path\n",
-    "from bs4 import BeautifulSoup\n",
     "\n",
     "from edisgo.edisgo import EDisGo\n",
     "from edisgo.tools.logger import setup_logger\n",
@@ -415,11 +414,6 @@
    "source": [
     "# Download SimBEV data\n",
     "\n",
-    "def listFD(url, ext=\"\"):\n",
-    "    page = requests.get(url).text\n",
-    "    soup = BeautifulSoup(page, \"html.parser\")\n",
-    "    return [node.get(\"href\").split(\"/\")[-1] for node in soup.find_all(\"a\") if node.get(\"href\").endswith(ext)]\n",
-    "\n",
     "def download_simbev_example_data():\n",
     "\n",
     "    raw_url = (\"https://raw.githubusercontent.com/openego/eDisGo/dev/\" +\n",
@@ -435,7 +429,9 @@
     "    # download files\n",
     "    url = (f\"https://github.com/openego/eDisGo/tree/dev/\" +\n",
     "           f\"tests/data/simbev_example_scenario/{ags}/\")\n",
-    "    filenames = [f for f in listFD(url, \"csv\")]\n",
+    "    page = requests.get(url).text\n",
+    "    items = json.loads(page)[\"payload\"][\"tree\"][\"items\"]\n",
+    "    filenames = [f[\"name\"] for f in items if \"csv\" in f[\"name\"]]\n",
     "\n",
     "    for file in filenames:\n",
     "        req = requests.get(f\"{raw_url}/{ags}/{file}\")\n",
@@ -473,7 +469,9 @@
     "    # download files\n",
     "    url = (\"https://github.com/openego/eDisGo/tree/dev/\" +\n",
     "           \"tests/data/tracbev_example_scenario/\")\n",
-    "    filenames = [f for f in listFD(url, \"gpkg\")]\n",
+    "    page = requests.get(url).text\n",
+    "    items = json.loads(page)[\"payload\"][\"tree\"][\"items\"]\n",
+    "    filenames = [f[\"name\"] for f in items if \"gpkg\" in f[\"name\"]]\n",
     "\n",
     "    for file in filenames:\n",
     "        req = requests.get(\n",
@@ -493,7 +491,9 @@
    "cell_type": "code",
    "execution_count": null,
    "id": "1d65e6d6",
-   "metadata": {},
+   "metadata": {
+    "scrolled": true
+   },
    "outputs": [],
    "source": [
     "edisgo.import_electromobility(\n",
@@ -776,7 +776,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.5"
+   "version": "3.8.18"
   },
   "toc": {
    "base_numbering": 1,
diff --git a/rtd_requirements.txt b/rtd_requirements.txt
index 96640084..560dc821 100644
--- a/rtd_requirements.txt
+++ b/rtd_requirements.txt
@@ -1,4 +1,3 @@
-beautifulsoup4
 dash < 2.9.0
 demandlib
 docutils == 0.16.0
diff --git a/setup.py b/setup.py
index 680d01cd..967ce07d 100644
--- a/setup.py
+++ b/setup.py
@@ -31,7 +31,6 @@ def read(fname):
 
 
 requirements = [
-    "beautifulsoup4",
     "contextily",
     "dash < 2.9.0",
     "demandlib",