Merge pull request #365 from uc-cdis/fix/content-dates-for-new-version

Fix/content dates for new version

.github/workflows/ci.yaml

@@ -0,0 +1,23 @@
+name: CI
+on:
+  push:
+  pull_request:
+    types: [opened, reopened]
+concurrency:
+  group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}'
+  cancel-in-progress: true
+jobs:
+  Security:
+    name: Security Pipeline
+    uses: uc-cdis/.github/.github/workflows/securitypipeline.yaml@master
+    with:
+       python-poetry: 'false'
+    secrets: inherit
+
+  UnitTest:
+    name: Python Unit Test with Postgres
+    uses: uc-cdis/.github/.github/workflows/python_unit_test.yaml@master
+    with:
+       python-version: '3.9'
+       test-script: 'tests/ci_commands_script.sh'
+       run-coveralls: true

.secrets.baseline

@@ -94,13 +94,13 @@
     }
   ],
   "results": {
-    ".travis.yml": [
+    ".github/workflows/ci.yaml": [
       {
-        "type": "Base64 High Entropy String",
-        "filename": ".travis.yml",
-        "hashed_secret": "8c0fdc72bf4daf2a2338287334d0cbd221fa9ef4",
+        "type": "Secret Keyword",
+        "filename": ".github/workflows/ci.yaml",
+        "hashed_secret": "3e26d6750975d678acb8fa35a0f69237881576b0",
         "is_verified": false,
-        "line_number": 27
+        "line_number": 15
       }
     ],
     "README.md": [
@@ -349,49 +349,49 @@
         "filename": "tests/test_client.py",
         "hashed_secret": "ff9c79b737b3ea7386618cc9437d3fb0a772182b",
         "is_verified": false,
-        "line_number": 2406
+        "line_number": 2408
       },
       {
         "type": "Hex High Entropy String",
         "filename": "tests/test_client.py",
         "hashed_secret": "c8176f1e75e62e15dabaa4087fb7194451c8f6d2",
         "is_verified": false,
-        "line_number": 2409
+        "line_number": 2411
       },
       {
         "type": "Hex High Entropy String",
         "filename": "tests/test_client.py",
         "hashed_secret": "d5198f8eddb1cbeb437899cd99e5ee97ab8531b4",
         "is_verified": false,
-        "line_number": 2409
+        "line_number": 2411
       },
       {
         "type": "Hex High Entropy String",
         "filename": "tests/test_client.py",
         "hashed_secret": "02dc196562514eaa3e2feac1f441ccf6ad81e09d",
         "is_verified": false,
-        "line_number": 2413
+        "line_number": 2415
       },
       {
         "type": "Hex High Entropy String",
         "filename": "tests/test_client.py",
         "hashed_secret": "f1cb2d91a95165a2ab909eadd9f7b65f312c7e2d",
         "is_verified": false,
-        "line_number": 2414
+        "line_number": 2416
       },
       {
         "type": "Hex High Entropy String",
         "filename": "tests/test_client.py",
         "hashed_secret": "58db546de03270b55a4c889a5c5e6296b29fef25",
         "is_verified": false,
-        "line_number": 2415
+        "line_number": 2417
       },
       {
         "type": "Hex High Entropy String",
         "filename": "tests/test_client.py",
         "hashed_secret": "b6c0bd08fde409c18760f32bef8705191840c402",
         "is_verified": false,
-        "line_number": 2416
+        "line_number": 2418
       }
     ],
     "tests/test_deprecated_aliases_endpoints.py": [
@@ -413,5 +413,5 @@
       }
     ]
   },
-  "generated_at": "2023-04-20T22:58:41Z"
+  "generated_at": "2023-09-27T23:03:38Z"
 }

indexd/app.py

@@ -17,17 +17,18 @@
 from .blueprint import blueprint as cross_blueprint
 from indexd.urls.blueprint import blueprint as index_urls_blueprint
 
+logger = cdislogging.get_logger(__name__)
+
 
 def app_init(app, settings=None):
     app.url_map.strict_slashes = False
-    app.logger.addHandler(cdislogging.get_stream_handler())
     if not settings:
         from .default_settings import settings
     app.config.update(settings["config"])
 
     if settings.get("AUTO_MIGRATE", True):
         engine_name = settings["config"]["INDEX"]["driver"].engine.dialect.name
-        app.logger.info(f"Auto migrating. Engine name: {engine_name}")
+        logger.info(f"Auto migrating. Engine name: {engine_name}")
         if engine_name == "sqlite":
             IndexBase.metadata.create_all()
             AliasBase.metadata.create_all()
@@ -37,7 +38,7 @@ def app_init(app, settings=None):
         else:
             alembic_main(["--raiseerr", "upgrade", "head"])
     else:
-        app.logger.info("Auto migrations are disabled")
+        logger.info("Auto migrations are disabled")
 
     app.auth = settings["auth"]
     app.hostname = os.environ.get("HOSTNAME") or "http://example.io"

indexd/index/drivers/alchemy.py

@@ -676,6 +676,20 @@ def get_urls(self, size=None, hashes=None, ids=None, start=0, limit=100):
                 for r in query
             ]
 
+    def _validate_and_format_content_dates(
+        self, record, content_created_date, content_updated_date
+    ):
+        if content_created_date is not None:
+            record.content_created_date = datetime.datetime.fromisoformat(
+                content_created_date
+            )
+            # Users cannot set content_updated_date without a content_created_date
+            record.content_updated_date = (
+                datetime.datetime.fromisoformat(content_updated_date)
+                if content_updated_date is not None
+                else record.content_created_date  # Set updated to created if no updated is provided
+            )
+
     def add(
         self,
         form,
@@ -755,16 +769,11 @@ def add(
 
             record.description = description
 
-            if content_created_date is not None:
-                record.content_created_date = datetime.datetime.fromisoformat(
-                    content_created_date
-                )
-                # Users cannot set content_updated_date without a content_created_date
-                record.content_updated_date = (
-                    datetime.datetime.fromisoformat(content_updated_date)
-                    if content_updated_date is not None
-                    else record.content_created_date  # Set updated to created if no updated is provided
-                )
+            self._validate_and_format_content_dates(
+                record=record,
+                content_created_date=content_created_date,
+                content_updated_date=content_updated_date,
+            )
 
             session.merge(base_version)
 
@@ -1384,6 +1393,12 @@ def add_version(
                 for m_key, m_value in metadata.items()
             ]
 
+            self._validate_and_format_content_dates(
+                record=record,
+                content_created_date=content_created_date,
+                content_updated_date=content_updated_date,
+            )
+
             try:
                 session.add(record)
                 create_urls_metadata(urls_metadata, record, session)

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "indexd"
-version = "5.0.0"
+version = "5.0.2"
 description = "Gen3 Indexing Service"
 authors = ["CTDS UChicago <[email protected]>"]
 license = "Apache-2.0"

tests/ci_commands_script.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+poetry run pytest -vv --cov=indexd --cov-report xml tests

tests/test_client.py

@@ -2115,6 +2115,8 @@ def test_create_index_version(client, user):
         "urls": ["s3://endpointurl/bucket2/key"],
         "hashes": {"md5": "8b9942cf415384b27cadf1f4d2d981f5"},
         "acl": ["a"],
+        "content_updated_date": "2023-03-14T17:02:54",
+        "content_created_date": "2023-03-13T17:02:54",
     }
 
     res_2 = client.post("/index/" + rec["did"], json=dataNew, headers=user)