-
Notifications
You must be signed in to change notification settings - Fork 3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
262 changed files
with
37,603 additions
and
5,724 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,87 @@ | ||
| name: GX Plugin | ||
| on: | ||
| push: | ||
| branches: | ||
| - master | ||
| paths: | ||
| - ".github/workflows/gx-plugin.yml" | ||
| - "metadata-ingestion-modules/gx-plugin/**" | ||
| - "metadata-ingestion/**" | ||
| - "metadata-models/**" | ||
| pull_request: | ||
| branches: | ||
| - master | ||
| paths: | ||
| - ".github/**" | ||
| - "metadata-ingestion-modules/gx-plugin/**" | ||
| - "metadata-ingestion/**" | ||
| - "metadata-models/**" | ||
| release: | ||
| types: [published] | ||
|
|
||
| concurrency: | ||
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | ||
| cancel-in-progress: true | ||
|
|
||
| jobs: | ||
| gx-plugin: | ||
| runs-on: ubuntu-latest | ||
| env: | ||
| SPARK_VERSION: 3.0.3 | ||
| DATAHUB_TELEMETRY_ENABLED: false | ||
| strategy: | ||
| matrix: | ||
| python-version: ["3.8", "3.10"] | ||
| include: | ||
| - python-version: "3.8" | ||
| extraPythonRequirement: "great-expectations~=0.15.12" | ||
| - python-version: "3.10" | ||
| extraPythonRequirement: "great-expectations~=0.16.0 numpy~=1.26.0" | ||
| - python-version: "3.11" | ||
| extraPythonRequirement: "great-expectations~=0.17.0" | ||
| fail-fast: false | ||
| steps: | ||
| - name: Set up JDK 17 | ||
| uses: actions/setup-java@v3 | ||
| with: | ||
| distribution: "zulu" | ||
| java-version: 17 | ||
| - uses: actions/checkout@v3 | ||
| - uses: actions/setup-python@v4 | ||
| with: | ||
| python-version: ${{ matrix.python-version }} | ||
| cache: "pip" | ||
| - name: Install dependencies | ||
| run: ./metadata-ingestion/scripts/install_deps.sh | ||
| - name: Install GX package and test (extras ${{ matrix.extraPythonRequirement }}) | ||
| run: ./gradlew -Pextra_pip_requirements='${{ matrix.extraPythonRequirement }}' :metadata-ingestion-modules:gx-plugin:lint :metadata-ingestion-modules:gx-plugin:testQuick | ||
| - name: pip freeze show list installed | ||
| if: always() | ||
| run: source metadata-ingestion-modules/gx-plugin/venv/bin/activate && pip freeze | ||
| - uses: actions/upload-artifact@v3 | ||
| if: ${{ always() && matrix.python-version == '3.11' && matrix.extraPythonRequirement == 'great-expectations~=0.17.0' }} | ||
| with: | ||
| name: Test Results (GX Plugin ${{ matrix.python-version}}) | ||
| path: | | ||
| **/build/reports/tests/test/** | ||
| **/build/test-results/test/** | ||
| **/junit.*.xml | ||
| - name: Upload coverage to Codecov | ||
| if: always() | ||
| uses: codecov/codecov-action@v3 | ||
| with: | ||
| token: ${{ secrets.CODECOV_TOKEN }} | ||
| directory: . | ||
| fail_ci_if_error: false | ||
| flags: gx-${{ matrix.python-version }}-${{ matrix.extraPythonRequirement }} | ||
| name: pytest-gx | ||
| verbose: true | ||
|
|
||
| event-file: | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - name: Upload | ||
| uses: actions/upload-artifact@v3 | ||
| with: | ||
| name: Event File | ||
| path: ${{ github.event_path }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
29 changes: 29 additions & 0 deletions
29
...ade/src/main/java/com/linkedin/datahub/upgrade/config/ReindexDomainDescriptionConfig.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,29 @@ | ||
| package com.linkedin.datahub.upgrade.config; | ||
|
|
||
| import com.linkedin.datahub.upgrade.system.NonBlockingSystemUpgrade; | ||
| import com.linkedin.datahub.upgrade.system.domaindescription.ReindexDomainDescription; | ||
| import com.linkedin.metadata.entity.AspectDao; | ||
| import com.linkedin.metadata.entity.EntityService; | ||
| import io.datahubproject.metadata.context.OperationContext; | ||
| import org.springframework.beans.factory.annotation.Value; | ||
| import org.springframework.context.annotation.Bean; | ||
| import org.springframework.context.annotation.Conditional; | ||
| import org.springframework.context.annotation.Configuration; | ||
|
|
||
| @Configuration | ||
| @Conditional(SystemUpdateCondition.NonBlockingSystemUpdateCondition.class) | ||
| public class ReindexDomainDescriptionConfig { | ||
|
|
||
| @Bean | ||
| public NonBlockingSystemUpgrade reindexDomainDescription( | ||
| final OperationContext opContext, | ||
| final EntityService<?> entityService, | ||
| final AspectDao aspectDao, | ||
| @Value("${systemUpdate.domainDescription.enabled}") final boolean enabled, | ||
| @Value("${systemUpdate.domainDescription.batchSize}") final Integer batchSize, | ||
| @Value("${systemUpdate.domainDescription.delayMs}") final Integer delayMs, | ||
| @Value("${systemUpdate.domainDescription.limit}") final Integer limit) { | ||
| return new ReindexDomainDescription( | ||
| opContext, entityService, aspectDao, enabled, batchSize, delayMs, limit); | ||
| } | ||
| } |
49 changes: 49 additions & 0 deletions
49
.../java/com/linkedin/datahub/upgrade/system/domaindescription/ReindexDomainDescription.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,49 @@ | ||
| package com.linkedin.datahub.upgrade.system.domaindescription; | ||
|
|
||
| import com.google.common.collect.ImmutableList; | ||
| import com.linkedin.datahub.upgrade.UpgradeStep; | ||
| import com.linkedin.datahub.upgrade.system.NonBlockingSystemUpgrade; | ||
| import com.linkedin.metadata.entity.AspectDao; | ||
| import com.linkedin.metadata.entity.EntityService; | ||
| import io.datahubproject.metadata.context.OperationContext; | ||
| import java.util.List; | ||
| import javax.annotation.Nonnull; | ||
| import lombok.extern.slf4j.Slf4j; | ||
|
|
||
| /** | ||
| * A job that reindexes all domain aspects as part of reindexing descriptions This is required to | ||
| * fix the analytics for domains | ||
| */ | ||
| @Slf4j | ||
| public class ReindexDomainDescription implements NonBlockingSystemUpgrade { | ||
|
|
||
| private final List<UpgradeStep> _steps; | ||
|
|
||
| public ReindexDomainDescription( | ||
| @Nonnull OperationContext opContext, | ||
| EntityService<?> entityService, | ||
| AspectDao aspectDao, | ||
| boolean enabled, | ||
| Integer batchSize, | ||
| Integer batchDelayMs, | ||
| Integer limit) { | ||
| if (enabled) { | ||
| _steps = | ||
| ImmutableList.of( | ||
| new ReindexDomainDescriptionStep( | ||
| opContext, entityService, aspectDao, batchSize, batchDelayMs, limit)); | ||
| } else { | ||
| _steps = ImmutableList.of(); | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public String id() { | ||
| return this.getClass().getName(); | ||
| } | ||
|
|
||
| @Override | ||
| public List<UpgradeStep> steps() { | ||
| return _steps; | ||
| } | ||
| } |
Oops, something went wrong.