Merge pull request #14 from PySport/feature/pass-loader-kwargs

Pass loader kwargs to loader

ingestify/application/loader.py

@@ -84,9 +84,15 @@ def load_file(
             file_data_serialization_format=file_resource.data_serialization_format
             or "txt",
             **http_options,
+            **file_resource.loader_kwargs,
         )
     else:
-        return file_resource.file_loader(file_resource, current_file)
+        return file_resource.file_loader(
+            file_resource,
+            current_file,
+            # TODO: check how to fix this with typehints
+            **file_resource.loader_kwargs,
+        )
 
 
 class UpdateDatasetTask(Task):

ingestify/domain/models/resources/dataset_resource.py

@@ -1,4 +1,4 @@
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from datetime import datetime
 from typing import Optional, Callable, TYPE_CHECKING
 
@@ -29,6 +29,7 @@ class FileResource:
     file_loader: Optional[
         Callable[["FileResource", Optional["File"]], Optional["DraftFile"]]
     ] = None
+    loader_kwargs: dict = field(default_factory=dict)
 
     def __post_init__(self):
         if self.json_content is None and not self.url and not self.file_loader:
@@ -75,6 +76,7 @@ def add_file(
                 Optional["DraftFile"],
             ]
         ] = None,
+        loader_kwargs: Optional[dict] = None,
     ):
         file_id = f"{data_feed_key}__{data_spec_version}"
         if file_id in self.files:
@@ -91,6 +93,7 @@ def add_file(
             http_options=http_options,
             data_serialization_format=data_serialization_format,
             file_loader=file_loader,
+            loader_kwargs=loader_kwargs or {},
         )
 
         self.files[file_id] = file_resource

ingestify/tests/test_engine.py

@@ -36,7 +36,11 @@ def add_extract_job(engine: IngestionEngine, source: Source, **selector):
     )
 
 
-def file_loader(file_resource, current_file):
+def file_loader(file_resource, current_file, some_extract_config=None):
+    if some_extract_config is not None and some_extract_config != "test123":
+        # Test loader_kwargs are passed correctly
+        raise Exception(f"Incorrect value for this test value: {some_extract_config}")
+
     if file_resource.file_id == "file1__v1":
         if not current_file:
             return DraftFile.from_input(
@@ -66,7 +70,7 @@ def find_datasets(
         dataset_collection_metadata: DatasetCollectionMetadata,
         competition_id,
         season_id,
-        **kwargs
+        **kwargs,
     ):
         last_modified = datetime.now(pytz.utc)
 
@@ -85,6 +89,7 @@ def find_datasets(
                 data_feed_key="file1",
                 data_spec_version="v1",
                 file_loader=file_loader,
+                loader_kwargs={"some_extract_config": "test123"},
             )
             .add_file(
                 last_modified=last_modified,
@@ -124,7 +129,7 @@ def find_datasets(
         dataset_collection_metadata: DatasetCollectionMetadata,
         competition_id,
         season_id,
-        **kwargs
+        **kwargs,
     ):
         while not self.should_stop:
             items = []