Features/#450 add metadata kh

CHANGELOG.rst

@@ -494,12 +494,12 @@ Changed
   `#1073 <https://github.com/openego/eGon-data/issues/1073>`_
 * Change hgv data source to use database
   `#1086 <https://github.com/openego/eGon-data/issues/1086>`_
+* Rename eMob MIT carrier names (use underscores)
+  `#1105 <https://github.com/openego/eGon-data/issues/1105>`_
 * Change desposit ID for data_bundle download from zenodo sandbox
   `#1110 <https://github.com/openego/eGon-data/issues/1110>`_
 * Use MaStR geocoding results for pv rooftop to buildings mapping workflow
   `#1095 <https://github.com/openego/eGon-data/issues/1095>`_
-* Rename eMob MIT carrier names (use underscores)
-  `#1105 <https://github.com/openego/eGon-data/issues/1105>`_
 
 .. _#799: https://github.com/openego/eGon-data/issues/799
 

src/egon/data/datasets/DSM_cts_ind.py

@@ -8,6 +8,10 @@
 sanity checks. See [#1120](https://github.com/openego/eGon-data/issues/1120)
 for updates.
 """
+import datetime
+import json
+
+from omi.dialects import get_dialect
 from sqlalchemy import ARRAY, Column, Float, Integer, String
 from sqlalchemy.ext.declarative import declarative_base
 import geopandas as gpd
@@ -18,6 +22,15 @@
 from egon.data.datasets import Dataset
 from egon.data.datasets.electricity_demand.temporal import calc_load_curve
 from egon.data.datasets.industry.temporal import identify_bus
+from egon.data.metadata import (
+    context,
+    contributors,
+    generate_resource_fields_from_db_table,
+    license_odbl,
+    meta_metadata,
+    meta_metadata,
+    sources,
+)
 
 # CONSTANTS
 # TODO: move to datasets.yml
@@ -128,7 +141,7 @@ def __init__(self, dependencies):
             name=self.name,
             version=self.version,
             dependencies=self.dependencies,
-            tasks=(dsm_cts_ind_processing),
+            tasks=(dsm_cts_ind_processing,),
         )
 
 
@@ -205,6 +218,195 @@ class EgonSitesIndLoadCurvesIndividualDsmTimeseries(Base):
     e_min = Column(ARRAY(Float))
 
 
+def add_metadata_individual():
+    targets = config.datasets()["DSM_CTS_industry"]["targets"]
+
+    targets = {
+        k: v for k, v in targets.items() if "dsm_timeseries" in v["table"]
+    }
+
+    title_dict = {
+        "egon_etrago_electricity_cts_dsm_timeseries": (
+            "DSM flexibility band time series for CTS"
+        ),
+        "egon_osm_ind_load_curves_individual_dsm_timeseries": (
+            "DSM flexibility band time series for OSM industry sites"
+        ),
+        "egon_demandregio_sites_ind_electricity_dsm_timeseries": (
+            "DSM flexibility band time series for demandregio industry sites"
+        ),
+        "egon_sites_ind_load_curves_individual_dsm_timeseries": (
+            "DSM flexibility band time series for other industry sites"
+        ),
+    }
+
+    description_dict = {
+        "egon_etrago_electricity_cts_dsm_timeseries": (
+            "DSM flexibility band time series for CTS in 1 h resolution "
+            "including available store capacity and power potential"
+        ),
+        "egon_osm_ind_load_curves_individual_dsm_timeseries": (
+            "DSM flexibility band time series for OSM industry sites in 1 h "
+            "resolution including available store capacity and power potential"
+        ),
+        "egon_demandregio_sites_ind_electricity_dsm_timeseries": (
+            "DSM flexibility band time series for demandregio industry sites "
+            "in 1 h resolution including available store capacity and power "
+            "potential"
+        ),
+        "egon_sites_ind_load_curves_individual_dsm_timeseries": (
+            "DSM flexibility band time series for other industry sites in 1 h "
+            "resolution including available store capacity and power potential"
+        ),
+    }
+
+    keywords_dict = {
+        "egon_etrago_electricity_cts_dsm_timeseries": ["cts"],
+        "egon_osm_ind_load_curves_individual_dsm_timeseries": [
+            "osm",
+            "industry",
+        ],
+        "egon_demandregio_sites_ind_electricity_dsm_timeseries": [
+            "demandregio",
+            "industry",
+        ],
+        "egon_sites_ind_load_curves_individual_dsm_timeseries": ["industry"],
+    }
+
+    primaryKey_dict = {
+        "egon_etrago_electricity_cts_dsm_timeseries": ["bus"],
+        "egon_osm_ind_load_curves_individual_dsm_timeseries": ["osm_id"],
+        "egon_demandregio_sites_ind_electricity_dsm_timeseries": [
+            "industrial_sites_id",
+        ],
+        "egon_sites_ind_load_curves_individual_dsm_timeseries": ["site_id"],
+    }
+
+    sources_dict = {
+        "egon_etrago_electricity_cts_dsm_timeseries": [
+            sources()["nep2021"],
+            sources()["zensus"],
+        ],
+        "egon_osm_ind_load_curves_individual_dsm_timeseries": [
+            sources()["hotmaps_industrial_sites"],
+            sources()["schmidt"],
+            sources()["seenergies"],
+        ],
+        "egon_demandregio_sites_ind_electricity_dsm_timeseries": [
+            sources()["openstreetmap"],
+        ],
+        "egon_sites_ind_load_curves_individual_dsm_timeseries": [
+            sources()["hotmaps_industrial_sites"],
+            sources()["openstreetmap"],
+            sources()["schmidt"],
+            sources()["seenergies"],
+        ],
+    }
+
+    contris = contributors(["kh", "kh"])
+
+    contris[0]["date"] = "2023-03-17"
+
+    contris[0]["object"] = "metadata"
+    contris[1]["object"] = "dataset"
+
+    contris[0]["comment"] = "Add metadata to dataset."
+    contris[1]["comment"] = "Add workflow to generate dataset."
+
+    for t_dict in targets.values():
+        schema = t_dict["schema"]
+        table = t_dict["table"]
+        name = f"{schema}.{table}"
+
+        meta = {
+            "name": name,
+            "title": title_dict[table],
+            "id": "WILL_BE_SET_AT_PUBLICATION",
+            "description": description_dict[table],
+            "language": "en-US",
+            "keywords": ["dsm", "timeseries"] + keywords_dict[table],
+            "publicationDate": datetime.date.today().isoformat(),
+            "context": context(),
+            "spatial": {
+                "location": "none",
+                "extent": "Germany",
+                "resolution": "none",
+            },
+            "temporal": {
+                "referenceDate": "2011-01-01",
+                "timeseries": {
+                    "start": "2011-01-01",
+                    "end": "2011-12-31",
+                    "resolution": "1 h",
+                    "alignment": "left",
+                    "aggregationType": "average",
+                },
+            },
+            "sources": [
+                sources()["egon-data"],
+                sources()["vg250"],
+                sources()["demandregio"],
+            ]
+            + sources_dict[table],
+            "licenses": [license_odbl("© eGon development team")],
+            "contributors": contris,
+            "resources": [
+                {
+                    "profile": "tabular-data-resource",
+                    "name": name,
+                    "path": "None",
+                    "format": "PostgreSQL",
+                    "encoding": "UTF-8",
+                    "schema": {
+                        "fields": generate_resource_fields_from_db_table(
+                            schema,
+                            table,
+                        ),
+                        "primaryKey": ["scn_name"] + primaryKey_dict[table],
+                    },
+                    "dialect": {"delimiter": "", "decimalSeparator": ""},
+                }
+            ],
+            "review": {"path": "", "badge": ""},
+            "metaMetadata": meta_metadata(),
+            "_comment": {
+                "metadata": (
+                    "Metadata documentation and explanation (https://"
+                    "github.com/OpenEnergyPlatform/oemetadata/blob/master/"
+                    "metadata/v141/metadata_key_description.md)"
+                ),
+                "dates": (
+                    "Dates and time must follow the ISO8601 including time "
+                    "zone (YYYY-MM-DD or YYYY-MM-DDThh:mm:ss±hh)"
+                ),
+                "units": "Use a space between numbers and units (100 m)",
+                "languages": (
+                    "Languages must follow the IETF (BCP47) format (en-GB, "
+                    "en-US, de-DE)"
+                ),
+                "licenses": (
+                    "License name must follow the SPDX License List "
+                    "(https://spdx.org/licenses/)"
+                ),
+                "review": (
+                    "Following the OEP Data Review (https://github.com/"
+                    "OpenEnergyPlatform/data-preprocessing/wiki)"
+                ),
+                "none": "If not applicable use (none)",
+            },
+        }
+
+        dialect = get_dialect(meta_metadata()["metadataVersion"])()
+
+        meta = dialect.compile_and_render(dialect.parse(json.dumps(meta)))
+
+        db.submit_comment(
+            f"'{json.dumps(meta)}'",
+            schema,
+            table,
+        )
+
+
 # Code
 def cts_data_import(cts_cool_vent_ac_share):
     """
@@ -999,10 +1201,10 @@ def delete_dsm_entries(carrier):
 
     # buses
 
-    sql = f"""
-    DELETE FROM {targets["bus"]["schema"]}.{targets["bus"]["table"]} b
-    WHERE (b.carrier LIKE '{carrier}');
-    """
+    sql = (
+        f"DELETE FROM {targets['bus']['schema']}.{targets['bus']['table']} b "
+        f"WHERE (b.carrier LIKE '{carrier}');"
+    )
     db.execute_sql(sql)
 
     # links
@@ -1621,3 +1823,5 @@ def dsm_cts_ind_processing():
     dsm_cts_ind()
 
     dsm_cts_ind_individual()
+
+    add_metadata_individual()

src/egon/data/datasets/emobility/motorized_individual_travel/__init__.py

@@ -31,6 +31,7 @@
     EgonEvMvGridDistrict,
     EgonEvPool,
     EgonEvTrip,
+    add_metadata,
 )
 from egon.data.datasets.emobility.motorized_individual_travel.ev_allocation import (  # noqa: E501
     allocate_evs_numbers,
@@ -475,5 +476,6 @@ def generate_model_data_tasks(scenario_name):
                     *generate_model_data_tasks(scenario_name="eGon2035"),
                     *generate_model_data_tasks(scenario_name="eGon100RE"),
                 },
+                add_metadata,
             ),
         )