diff --git a/metadata-ingestion/src/datahub/ingestion/source/sac/sac.py b/metadata-ingestion/src/datahub/ingestion/source/sac/sac.py index e2fbc75ef0851b..7d21eaab37044d 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sac/sac.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sac/sac.py @@ -9,6 +9,8 @@ import pyodata.v2.service from authlib.integrations.requests_client import OAuth2Session from pydantic import Field, SecretStr, validator +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry from datahub.configuration.common import AllowDenyPattern from datahub.configuration.source_common import ( @@ -131,6 +133,11 @@ class SACSourceConfig( description="Controls whether Analytic Applications should be ingested", ) + ingest_import_data_model_schema_metadata: bool = Field( + default=True, + description="Controls whether schema metadata of Import Data Models should be ingested (ingesting schema metadata of Import Data Models significantly increases overall ingestion time)" + ) + resource_id_pattern: AllowDenyPattern = Field( AllowDenyPattern.allow_all(), description="Patterns for selecting resource ids that are to be included", @@ -385,7 +392,7 @@ def get_model_workunits( yield mcp.as_workunit() - if model.is_import: + if model.is_import and self.config.ingest_import_data_model_schema_metadata: primary_fields: List[str] = [] schema_fields: List[SchemaFieldClass] = [] @@ -546,6 +553,22 @@ def get_sac_connection( grant_type="client_credentials", ) + retries = 3 + backoff_factor = 10 + status_forcelist = (500,) + + retry = Retry( + total=retries, + read=retries, + connect=retries, + backoff_factor=backoff_factor, + status_forcelist=status_forcelist, + ) + + adapter = HTTPAdapter(max_retries=retry) + session.mount("http://", adapter) + session.mount("https://", adapter) + session.register_compliance_hook( "protected_request", _add_sap_sac_custom_auth_header )