Skip to content

Commit

Permalink
feat(great-expectations): add SqlAlchemyDataset data asset support
Browse files Browse the repository at this point in the history
  • Loading branch information
seuf committed Nov 10, 2023
1 parent 9c0f4de commit f23b158
Showing 1 changed file with 28 additions and 0 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
ExpectationSuiteIdentifier,
ValidationResultIdentifier,
)
from great_expectations.dataset.sqlalchemy_dataset import SqlAlchemyDataset
from great_expectations.execution_engine.sqlalchemy_execution_engine import (
SqlAlchemyExecutionEngine,
)
Expand Down Expand Up @@ -686,6 +687,33 @@ def get_dataset_partitions(self, batch_identifier, data_asset):
batch_spec_type=type(ge_batch_spec)
)
)
elif isinstance(data_asset, SqlAlchemyDataset):
if "." in data_asset._table.name:
schema_name, table_name = data_asset._table.name.split(".")
else:
schema_name = data_asset._table.schema
table_name = data_asset._table.name

if data_asset.engine.url.startswith("bigquery://"):
sqlalchemy_uri = f"{data_asset.engine.url}/{schema_name}"
else:
sqlalchemy_uri = data_asset.engine.url

dataset_urn = make_dataset_urn_from_sqlalchemy_uri(
sqlalchemy_uri=sqlalchemy_uri,
schema_name=schema_name,
table_name=table_name,
env=self.env
)
partitionSpec = None
batchSpec = None
dataset_partitions.append(
{
"dataset_urn": dataset_urn,
"partitionSpec": partitionSpec,
"batchSpec": batchSpec,
}
)
else:
# TODO - v2-spec - SqlAlchemyDataset support
warn(
Expand Down

0 comments on commit f23b158

Please sign in to comment.