[#105] File type schemas

api/adapters/base.py

@@ -1,15 +1,15 @@
 import abc
-
+from typing import Type
 from api.config import Settings, get_settings
-from api.models.catalog import DatasetMetadataDOC
+from api.models.catalog import T
 from api.models.user import Submission
 
 
 class AbstractRepositoryRequestHandler(abc.ABC):
     settings: Settings = get_settings()
 
     @abc.abstractmethod
-    def get_metadata(self, record_id: str):
+    def get_metadata(self, record_id: str) -> dict:
         """Returns the metadata for the specified record from a repository"""
         ...
 
@@ -19,13 +19,13 @@ class AbstractRepositoryMetadataAdapter(abc.ABC):
 
     @staticmethod
     @abc.abstractmethod
-    def to_catalog_record(metadata: dict) -> DatasetMetadataDOC:
+    def to_catalog_record(metadata: dict, meta_model_type: Type[T]) -> T:
         """Converts repository metadata to a catalog dataset record"""
         ...
 
     @staticmethod
     @abc.abstractmethod
-    def to_repository_record(catalog_record: DatasetMetadataDOC):
+    def to_repository_record(catalog_record: T):
         """Converts dataset catalog dataset record to repository metadata"""
         ...
 
@@ -35,7 +35,7 @@ def update_submission(submission: Submission, repo_record_id: str) -> Submission
         """Sets additional repository specific metadata to submission record"""
         ...
 
-    async def get_metadata(self, record_id: str):
+    async def get_metadata(self, record_id: str) -> dict:
         """Returns the metadata for the specified record from a repository"""
 
         return self.repo_api_handler.get_metadata(record_id)

api/adapters/hydroshare.py

@@ -7,8 +7,8 @@
 from api.adapters.utils import RepositoryType, register_adapter
 from api.exceptions import RepositoryException
 from api.models import schema
-from api.models.catalog import DatasetMetadataDOC
-from api.models.user import Submission, SubmissionType
+from api.models.catalog import HSResourceMetadataDOC
+from api.models.user import Submission
 
 
 class Creator(BaseModel):
@@ -163,7 +163,8 @@ def to_dataset_license(self):
 
 
 class _HydroshareRequestHandler(AbstractRepositoryRequestHandler):
-    def get_metadata(self, record_id: str):
+
+    def get_metadata(self, record_id: str) -> dict:
         hs_meta_url = self.settings.hydroshare_meta_read_url % record_id
         hs_file_url = self.settings.hydroshare_file_read_url % record_id
 
@@ -194,13 +195,13 @@ class HydroshareMetadataAdapter(AbstractRepositoryMetadataAdapter):
     repo_api_handler = _HydroshareRequestHandler()
 
     @staticmethod
-    def to_catalog_record(metadata: dict) -> DatasetMetadataDOC:
+    def to_catalog_record(metadata: dict, meta_model_type: HSResourceMetadataDOC) -> HSResourceMetadataDOC:
         """Converts hydroshare resource metadata to a catalog dataset record"""
         hs_metadata_model = _HydroshareResourceMetadata(**metadata)
         return hs_metadata_model.to_catalog_dataset()
 
     @staticmethod
-    def to_repository_record(catalog_record: DatasetMetadataDOC):
+    def to_repository_record(catalog_record: HSResourceMetadataDOC):
         """Converts dataset catalog record to hydroshare resource metadata"""
         raise NotImplementedError
 
@@ -293,7 +294,7 @@ def to_dataset_provider():
         return provider
 
     def to_catalog_dataset(self):
-        dataset = DatasetMetadataDOC.construct()
+        dataset = HSResourceMetadataDOC.construct()
         dataset.provider = self.to_dataset_provider()
         dataset.name = self.title
         dataset.description = self.abstract

api/adapters/s3.py

@@ -1,41 +1,74 @@
-import boto3
 import json
-from botocore.client import Config
+from http import HTTPStatus
+from typing import Type
+
+import boto3
 from botocore import UNSIGNED
+from botocore.client import Config
+from botocore.exceptions import ClientError as S3ClientError
 
-from api.adapters.base import AbstractRepositoryMetadataAdapter, AbstractRepositoryRequestHandler
+from api.adapters.base import (
+    AbstractRepositoryMetadataAdapter,
+    AbstractRepositoryRequestHandler,
+)
 from api.adapters.utils import RepositoryType, register_adapter
-from api.models.catalog import DatasetMetadataDOC
-from api.models.user import Submission, SubmissionType
+from api.exceptions import RepositoryException
+from api.models.catalog import T
+from api.models.user import Submission
 
 
 class _S3RequestHandler(AbstractRepositoryRequestHandler):
-    def get_metadata(self, record_id: str):
+    def get_metadata(self, record_id: str) -> dict:
         endpoint_url = record_id.split("+")[0]
         bucket_name = record_id.split("+")[1]
         file_key = record_id.split("+")[2]
 
-        s3 = boto3.client('s3', config=Config(signature_version=UNSIGNED), endpoint_url=endpoint_url)
-
-        response = s3.get_object(Bucket=bucket_name, Key=file_key)
-        json_content = response['Body'].read().decode('utf-8')
+        # TODO: Should we be expecting the path for the data file and then compute the metadata file path from that?
+        #  Or should we be expecting the metadata file path directly? May be we should get path for both
+        #  data file and metadata file. If have the path for the data file we can check that the data file
+        #  exists and then retrieve the metadata file and catalog the metadata.
+
+        # check if the endpoint URL is an AWS S3 URL
+        if endpoint_url.endswith("amazonaws.com"):
+            endpoint_url = None
+        s3 = boto3.client(
+            "s3", config=Config(signature_version=UNSIGNED), endpoint_url=endpoint_url
+        )
+        try:
+            response = s3.get_object(Bucket=bucket_name, Key=file_key)
+        except S3ClientError as ex:
+            if ex.response["Error"]["Code"] == "NoSuchKey":
+                raise RepositoryException(
+                    detail=f"Specified metadata file was not found in S3: {bucket_name}/{file_key}",
+                    status_code=HTTPStatus.NOT_FOUND
+                )
+            else:
+                err_msg = f"Error accessing S3 file({bucket_name}/{file_key}): {str(ex)}"
+                raise RepositoryException(detail=err_msg, status_code=HTTPStatus.BAD_REQUEST)
 
-        # Parse the JSON content
-        data = json.loads(json_content)
+        json_content = response["Body"].read().decode("utf-8")
+        # parse the JSON content
+        try:
+            data = json.loads(json_content)
+        except json.JSONDecodeError as ex:
+            err_msg = f"Invalid JSON content in S3 file ({file_key}). Error: {str(ex)}"
+            raise RepositoryException(detail=err_msg, status_code=HTTPStatus.BAD_REQUEST)
 
+        # remove additionalType field - this will be set by the schema model
+        data.pop("additionalType", None)
         return data
 
 
 class S3MetadataAdapter(AbstractRepositoryMetadataAdapter):
     repo_api_handler = _S3RequestHandler()
 
     @staticmethod
-    def to_catalog_record(metadata: dict) -> DatasetMetadataDOC:
-        return DatasetMetadataDOC(**metadata)
+    def to_catalog_record(metadata: dict, meta_model_type: Type[T]) -> T:
+        return meta_model_type(**metadata)
 
     @staticmethod
-    def to_repository_record(catalog_record: DatasetMetadataDOC):
-        """Converts dataset catalog record to hydroshare resource metadata"""
+    def to_repository_record(catalog_record: T):
+        """Converts dataset catalog record to repository resource/dataset metadata"""
         raise NotImplementedError
 
     @staticmethod

api/adapters/utils.py

@@ -1,5 +1,5 @@
 from enum import Enum
-from typing import Type, Union
+from typing import Type, Union, Dict
 
 from api.adapters.base import AbstractRepositoryMetadataAdapter
 
@@ -9,7 +9,7 @@ class RepositoryType(str, Enum):
     S3 = 'S3'
 
 
-_adapter_registry = {}
+_adapter_registry: Dict[RepositoryType, Type[AbstractRepositoryMetadataAdapter]] = {}
 
 
 def register_adapter(repository_type: RepositoryType, adapter_class: Type[AbstractRepositoryMetadataAdapter]) -> None:
@@ -21,3 +21,4 @@ def get_adapter_by_type(repository_type: RepositoryType) -> Union[AbstractReposi
     if adapter_cls:
         return adapter_cls()
     return None
+

api/main.py

@@ -14,7 +14,13 @@
 from starlette.responses import PlainTextResponse
 
 from api.config import get_settings
-from api.models.catalog import DatasetMetadataDOC
+from api.models.catalog import (
+    CoreMetadataDOC,
+    HSResourceMetadataDOC,
+    GenericDatasetMetadataDOC,
+    NetCDFMetadataDOC,
+    RasterMetadataDOC,
+)
 from api.models.user import Submission, User
 from api.routes.catalog import router as catalog_router
 from api.routes.discovery import router as discovery_router
@@ -48,7 +54,11 @@ async def startup_db_client():
     settings = get_settings()
     app.mongodb_client = AsyncIOMotorClient(settings.db_connection_string)
     app.mongodb = app.mongodb_client[settings.database_name]
-    await init_beanie(database=app.mongodb, document_models=[DatasetMetadataDOC, User, Submission])
+    await init_beanie(
+        database=app.mongodb,
+        document_models=[CoreMetadataDOC, HSResourceMetadataDOC, GenericDatasetMetadataDOC, NetCDFMetadataDOC,
+                         RasterMetadataDOC, User, Submission]
+    )
 
 
 @app.on_event("shutdown")

api/models/catalog.py

@@ -1,16 +1,23 @@
 import datetime
-from typing import Optional
+from typing import TypeVar, Optional
 
 from beanie import Document
 
 from api.models.user import Submission, S3Path
-from .schema import CoreMetadata, DatasetMetadata
+from .schema import (
+    CoreMetadata,
+    GenericDatasetMetadata,
+    HSResourceMetadata,
+    HSNetCDFMetadata,
+    HSRasterMetadata,
+)
 
 
 class CoreMetadataDOC(Document, CoreMetadata):
-    # this field is not stored in the database, but is populated from the corresponding submission record
-    # using the type field in the submission record
+    # these fields are not stored in the database, but are populated from the corresponding submission record
     submission_type: str = None
+    repository_identifier: str = None
+    s3_path: Optional[S3Path] = None
 
     class Settings:
         # name is the collection name in database (iguide) where the Metadata Record documents will be stored
@@ -36,6 +43,27 @@ def as_submission(self) -> Submission:
         )
 
 
-class DatasetMetadataDOC(CoreMetadataDOC, DatasetMetadata):
-    repository_identifier: str = None
-    s3_path: Optional[S3Path] = None
+class HSResourceMetadataDOC(CoreMetadataDOC, HSResourceMetadata):
+
+    def as_submission(self) -> Submission:
+        submission = super().as_submission()
+        submission.repository = "HydroShare"
+        submission.repository_identifier = self.repository_identifier
+        return submission
+
+
+class GenericDatasetMetadataDOC(CoreMetadataDOC, GenericDatasetMetadata):
+    pass
+
+
+class NetCDFMetadataDOC(CoreMetadataDOC, HSNetCDFMetadata):
+    pass
+
+
+class RasterMetadataDOC(CoreMetadataDOC, HSRasterMetadata):
+    pass
+
+# T is a type variable that can be used for type hinting for any schema model that inherits from CoreMetadataDOC
+
+
+T = TypeVar("T", bound=CoreMetadataDOC)

api/models/management/generate_schema.py

@@ -3,30 +3,65 @@
 
 import typer
 
-from api.models.schema import DatasetMetadata
-
-
-def main(output_name: str = "api/models/schemas/schema.json"):
-    schema = DatasetMetadata.schema()
-    json_schema = DatasetMetadata.schema_json()#indent=2)
-    # Have to run it a few times for the definitions to get updated before inserted into another model
-    while "#/definitions/" in json_schema:
-        for definition in schema["definitions"]:
-            class_definition = schema["definitions"][definition]
-            # replace allOf with a single definition
-            json_schema = json_schema.replace(
-                f'"allOf": [{{"$ref": "#/definitions/{definition}"}}]',
-                json.dumps(class_definition)[1:-1]
-            )
-            #replace definition directly
-            json_schema = json_schema.replace(
-                f'"$ref": "#/definitions/{definition}"',
-                json.dumps(class_definition)[1:-1]
-            )
-    embedded_schema = json.loads(json_schema)
-    current_directory = absolute_directory(output_name)
-    with open(current_directory, "w") as f:
-        f.write(json.dumps(embedded_schema, indent=2))
+from api.models.schema import (
+    GenericDatasetMetadata,
+    HSNetCDFMetadata,
+    HSRasterMetadata,
+    HSResourceMetadata,
+)
+
+
+def main():
+    def generate_schema_json(schema_model, folder_name):
+        base_directory = "api/models/schemas"
+        schema_file_path = os.path.join(base_directory, folder_name, "schema.json")
+        schema = schema_model.schema()
+        json_schema = schema_model.schema_json()
+
+        # Have to run it a few times for the definitions to get updated before inserted into another model
+        while "#/definitions/" in json_schema:
+            for definition in schema["definitions"]:
+                class_definition = schema["definitions"][definition]
+                # replace allOf with a single definition
+                json_schema = json_schema.replace(
+                    f'"allOf": [{{"$ref": "#/definitions/{definition}"}}]',
+                    json.dumps(class_definition)[1:-1]
+                )
+                #replace definition directly
+                json_schema = json_schema.replace(
+                    f'"$ref": "#/definitions/{definition}"',
+                    json.dumps(class_definition)[1:-1]
+                )
+        embedded_schema = json.loads(json_schema)
+        current_directory = absolute_directory(schema_file_path)
+        with open(current_directory, "w") as f:
+            f.write(json.dumps(embedded_schema, indent=2))
+
+    schemas = get_schemas()
+    for schema_item in schemas:
+        generate_schema_json(schema_model=schema_item["model"], folder_name=schema_item["folder_name"])
+
+
+def get_schemas():
+    schemas = [
+        {
+            "model": GenericDatasetMetadata,
+            "folder_name": "generic",
+        },
+        {
+            "model": HSResourceMetadata,
+            "folder_name": "hs_resource",
+        },
+        {
+            "model": HSNetCDFMetadata,
+            "folder_name": "netcdf",
+        },
+        {
+            "model": HSRasterMetadata,
+            "folder_name": "raster",
+        },
+    ]
+    return schemas
 
 
 def absolute_directory(output_name):