Skip to content

Commit

Permalink
fix(entityVersioning): fixes failure for entity versioning
Browse files Browse the repository at this point in the history
  • Loading branch information
RyanHolstien committed Jan 14, 2025
1 parent 41a4a6c commit 453c4da
Show file tree
Hide file tree
Showing 4 changed files with 82 additions and 0 deletions.
8 changes: 8 additions & 0 deletions docker/profiles/docker-compose.gms.yml
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,7 @@ x-datahub-system-update-service: &datahub-system-update-service
SCHEMA_REGISTRY_SYSTEM_UPDATE: ${SCHEMA_REGISTRY_SYSTEM_UPDATE:-true}
SPRING_KAFKA_PROPERTIES_AUTO_REGISTER_SCHEMAS: ${SPRING_KAFKA_PROPERTIES_AUTO_REGISTER_SCHEMAS:-true}
SPRING_KAFKA_PROPERTIES_USE_LATEST_VERSION: ${SPRING_KAFKA_PROPERTIES_USE_LATEST_VERSION:-true}
ENTITY_VERSIONING_ENABLED: ${ENTITY_VERSIONING_ENABLED:-true}
volumes:
- ${HOME}/.datahub/plugins:/etc/datahub/plugins

Expand All @@ -80,6 +81,7 @@ x-datahub-system-update-service-dev: &datahub-system-update-service-dev
SKIP_ELASTICSEARCH_CHECK: false
REPROCESS_DEFAULT_BROWSE_PATHS_V2: ${REPROCESS_DEFAULT_BROWSE_PATHS_V2:-false}
JAVA_TOOL_OPTIONS: '-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5003'
ENTITY_VERSIONING_ENABLED: ${ENTITY_VERSIONING_ENABLED:-true}
volumes:
- ../../datahub-upgrade/build/libs/:/datahub/datahub-upgrade/bin/
- ../../metadata-models/src/main/resources/:/datahub/datahub-gms/resources
Expand All @@ -101,6 +103,7 @@ x-datahub-gms-service: &datahub-gms-service
<<: [*primary-datastore-mysql-env, *graph-datastore-search-env, *search-datastore-env, *datahub-quickstart-telemetry-env, *kafka-env]
ALTERNATE_MCP_VALIDATION: ${ALTERNATE_MCP_VALIDATION:-true}
STRICT_URN_VALIDATION_ENABLED: ${STRICT_URN_VALIDATION_ENABLED:-true}
ENTITY_VERSIONING_ENABLED: ${ENTITY_VERSIONING_ENABLED:-true}
healthcheck:
test: curl -sS --fail http://datahub-gms:${DATAHUB_GMS_PORT:-8080}/health
start_period: 90s
Expand Down Expand Up @@ -131,6 +134,7 @@ x-datahub-gms-service-dev: &datahub-gms-service-dev
SEARCH_SERVICE_ENABLE_CACHE: false
LINEAGE_SEARCH_CACHE_ENABLED: false
SHOW_BROWSE_V2: true
ENTITY_VERSIONING_ENABLED: ${ENTITY_VERSIONING_ENABLED:-true}
volumes:
- ./datahub-gms/start.sh:/datahub/datahub-gms/scripts/start.sh
- ./datahub-gms/jetty.xml:/datahub/datahub-gms/scripts/jetty.xml
Expand All @@ -155,12 +159,14 @@ x-datahub-mae-consumer-service: &datahub-mae-consumer-service
- ${DATAHUB_LOCAL_MAE_ENV:-empty2.env}
environment: &datahub-mae-consumer-env
<<: [*primary-datastore-mysql-env, *graph-datastore-search-env, *search-datastore-env, *kafka-env]
ENTITY_VERSIONING_ENABLED: ${ENTITY_VERSIONING_ENABLED:-true}

x-datahub-mae-consumer-service-dev: &datahub-mae-consumer-service-dev
<<: *datahub-mae-consumer-service
image: ${DATAHUB_MAE_CONSUMER_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-mae-consumer}:${DATAHUB_VERSION:-debug}
environment:
<<: [*datahub-dev-telemetry-env, *datahub-mae-consumer-env]
ENTITY_VERSIONING_ENABLED: ${ENTITY_VERSIONING_ENABLED:-true}
volumes:
- ./datahub-mae-consumer/start.sh:/datahub/datahub-mae-consumer/scripts/start.sh
- ../../metadata-models/src/main/resources/:/datahub/datahub-mae-consumer/resources
Expand All @@ -183,12 +189,14 @@ x-datahub-mce-consumer-service: &datahub-mce-consumer-service
<<: [*primary-datastore-mysql-env, *graph-datastore-search-env, *search-datastore-env, *datahub-quickstart-telemetry-env, *kafka-env]
ALTERNATE_MCP_VALIDATION: ${ALTERNATE_MCP_VALIDATION:-true}
STRICT_URN_VALIDATION_ENABLED: ${STRICT_URN_VALIDATION_ENABLED:-true}
ENTITY_VERSIONING_ENABLED: ${ENTITY_VERSIONING_ENABLED:-true}

x-datahub-mce-consumer-service-dev: &datahub-mce-consumer-service-dev
<<: *datahub-mce-consumer-service
image: ${DATAHUB_MCE_CONSUMER_IMAGE:-${DATAHUB_REPO:-acryldata}/datahub-mce-consumer}:${DATAHUB_VERSION:-debug}
environment:
<<: [*datahub-dev-telemetry-env, *datahub-mce-consumer-env]
ENTITY_VERSIONING_ENABLED: ${ENTITY_VERSIONING_ENABLED:-true}
volumes:
- ./datahub-mce-consumer/start.sh:/datahub/datahub-mce-consumer/scripts/start.sh
- ../../metadata-jobs/mce-consumer-job/build/libs/:/datahub/datahub-mce-consumer/bin
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import static com.linkedin.metadata.Constants.INITIAL_VERSION_SORT_ID;
import static com.linkedin.metadata.Constants.VERSION_PROPERTIES_ASPECT_NAME;
import static com.linkedin.metadata.Constants.VERSION_SET_ENTITY_NAME;
import static com.linkedin.metadata.Constants.VERSION_SET_KEY_ASPECT_NAME;
import static com.linkedin.metadata.Constants.VERSION_SET_PROPERTIES_ASPECT_NAME;
import static com.linkedin.metadata.Constants.VERSION_SORT_ID_FIELD_NAME;
import static com.linkedin.metadata.aspect.validation.ConditionalWriteValidator.HTTP_HEADER_IF_VERSION_MATCH;
Expand Down Expand Up @@ -90,6 +91,15 @@ public List<IngestResult> linkLatestVersion(
+ newLatestVersion.getEntityType());
}
if (!aspectRetriever.entityExists(ImmutableSet.of(versionSet)).get(versionSet)) {
MetadataChangeProposal versionSetKeyProposal = new MetadataChangeProposal();
versionSetKeyProposal.setEntityUrn(versionSet);
versionSetKeyProposal.setEntityType(VERSION_SET_ENTITY_NAME);
versionSetKeyProposal.setAspectName(VERSION_SET_KEY_ASPECT_NAME);
versionSetKeyProposal.setAspect(GenericRecordUtils.serializeAspect(versionSetKey));
versionSetKeyProposal.setChangeType(ChangeType.CREATE_ENTITY);
entityService.ingestProposal(
opContext, versionSetKeyProposal, opContext.getAuditStamp(), false);

sortId = INITIAL_VERSION_SORT_ID;
versionSetConstraint = -1L;
versionPropertiesConstraint = -1L;
Expand Down
Empty file.
64 changes: 64 additions & 0 deletions smoke-test/tests/entity_versioning/test_versioning.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
import pytest


@pytest.fixture(scope="module", autouse=True)
def test_link_unlink_version(auth_session):
"""Fixture to execute setup before and tear down after all tests are run"""
res_data = link_version(auth_session)

assert res_data
assert res_data["data"]
assert res_data["data"]["linkAssetVersion"]
assert (
res_data["data"]["linkAssetVersion"]
== "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)"
)

res_data = unlink_version(auth_session)

assert res_data
assert res_data["data"]
assert res_data["data"]["unlinkAssetVersion"]


def link_version(auth_session):
json = {
"mutation": """mutation linkAssetVersion($input: LinkVersionInput!) {\n
linkAssetVersion(input: $input)
}\n
}""",
"variables": {
"input": {
"version": "1233456",
"versionSet": "urn:li:versionSet:(12345678910,dataset)",
"linkedEntity": "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)",
}
},
}
response = auth_session.post(
f"{auth_session.frontend_url()}/api/v2/graphql", json=json
)
response.raise_for_status()

return response.json()


def unlink_version(auth_session):
json = {
"mutation": """mutation unlinkAssetVersion($input: UnlinkVersionInput!) {\n
unlinkAssetVersion(input: $input)
}\n
}""",
"variables": {
"input": {
"versionSet": "urn:li:versionSet:(12345678910,dataset)",
"unlinkedEntity": "urn:li:dataset:(urn:li:dataPlatform:kafka,SampleKafkaDataset,PROD)",
}
},
}
response = auth_session.post(
f"{auth_session.frontend_url()}/api/v2/graphql", json=json
)
response.raise_for_status()

return response.json()

0 comments on commit 453c4da

Please sign in to comment.