Skip to content

Commit

Permalink
improve report performance using larger page size and safer backoff h…
Browse files Browse the repository at this point in the history
…andling
  • Loading branch information
pnadolny13 committed Dec 11, 2024
1 parent c536a64 commit 15b3ea1
Showing 1 changed file with 3 additions and 1 deletion.
4 changes: 3 additions & 1 deletion tap_service_titan/streams/reporting.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

from __future__ import annotations

import math
import typing as t
from datetime import datetime, timedelta, timezone
from functools import cached_property
Expand Down Expand Up @@ -149,6 +150,7 @@ def get_url_params(
"""
params = super().get_url_params(context, next_page_token)
params.pop("modifiedOnOrAfter", "")
params["pageSize"] = 25000
return params

def prepare_request_payload(
Expand Down Expand Up @@ -236,6 +238,6 @@ def backoff_wait_generator(self) -> t.Callable[..., t.Generator[int, t.Any, None

def _backoff_from_headers(retriable_api_error) -> int: # noqa: ANN001
response_headers = retriable_api_error.response.headers
return int(response_headers.get("Retry-After", 0))
return int(math.ceil(response_headers.get("Retry-After", 0)))

return self.backoff_runtime(value=_backoff_from_headers)

0 comments on commit 15b3ea1

Please sign in to comment.