add lookback_window_days config

archdotdev · Dec 18, 2024 · 089b65c · 089b65c
1 parent 900e986
commit 089b65c
Show file tree

Hide file tree

Showing 3 changed files with 33 additions and 0 deletions.
diff --git a/README.md b/README.md
@@ -93,6 +93,26 @@ The backfill_date_parameter value must be in the parameters list.
   ]
 ```
 
+Optionally add `lookback_window_days` to define the amount of days to lookback when running incrementally.
+This is used to handled retroactively updated data in previously synced reports.
+
+```json
+  "custom_reports": [
+      {
+          "report_category": "accounting",
+          "report_name": "my_custom_accounting_report",
+          "report_id": "123",
+          "backfill_date_parameter": "AsOfDate",
+          "lookback_window_days": 30,
+          "parameters": [
+              {
+                  "name": "AsOfDate",
+                  "value": "2024-09-01"
+              }
+          ]
+      }
+  ]
+```
 ### Configure using environment variables
 
 This Singer tap will automatically import any environment variables within the working directory's

diff --git a/tap_service_titan/streams/reporting.py b/tap_service_titan/streams/reporting.py
@@ -70,6 +70,10 @@ def _get_initial_date_param(self) -> date | None:
         ).date()
         bookmark = self.stream_state.get("replication_key_value")
         if bookmark:
+            if self._report.get("lookback_window_days"):
+                bookmark = bookmark - timedelta(
+                    days=self._report["lookback_window_days"]
+                )
             return max(
                 configured_date_param,
                 datetime.strptime(bookmark, "%Y-%m-%dT%H:%M:%S%z").date(),

diff --git a/tap_service_titan/tap.py b/tap_service_titan/tap.py
@@ -93,6 +93,15 @@ class TapServiceTitan(Tap):
                         th.StringType,
                         description="The date parameter to use for backfilling. The report will be retrieved for each date until the current date.",  # noqa: E501
                     ),
+                    th.Property(
+                        "lookback_window_days",
+                        th.StringType,
+                        description=(
+                            "The amount of days to lookback when running incrementally."
+                            "This is used to handled retroactively updated data in "
+                            "previously synced reports.",
+                        )
+                    ),
                     th.Property(
                         "parameters",
                         th.ArrayType(