diff --git a/.github/workflows/gx-plugin.yml b/.github/workflows/gx-plugin.yml index 88aca0322c4625..84ba2e0559be1b 100644 --- a/.github/workflows/gx-plugin.yml +++ b/.github/workflows/gx-plugin.yml @@ -35,15 +35,9 @@ jobs: include: - python-version: "3.8" extraPythonRequirement: "great-expectations~=0.15.12" - - python-version: "3.8" - extraPythonRequirement: "great-expectations~=0.16.0 numpy~=1.26.0" - - python-version: "3.8" - extraPythonRequirement: "great-expectations~=0.17.0" - - python-version: "3.10" - extraPythonRequirement: "great-expectations~=0.15.12" - python-version: "3.10" extraPythonRequirement: "great-expectations~=0.16.0 numpy~=1.26.0" - - python-version: "3.10" + - python-version: "3.11" extraPythonRequirement: "great-expectations~=0.17.0" fail-fast: false steps: @@ -65,7 +59,7 @@ jobs: if: always() run: source metadata-ingestion-modules/gx-plugin/venv/bin/activate && pip freeze - uses: actions/upload-artifact@v3 - if: ${{ always() && matrix.python-version == '3.10' && matrix.extraPythonRequirement == 'great-expectations~=0.15.12' }} + if: ${{ always() && matrix.python-version == '3.11' && matrix.extraPythonRequirement == 'great-expectations~=0.17.0' }} with: name: Test Results (GX Plugin ${{ matrix.python-version}}) path: | diff --git a/.github/workflows/test-results.yml b/.github/workflows/test-results.yml index c94a5fc340f473..df68752b37961d 100644 --- a/.github/workflows/test-results.yml +++ b/.github/workflows/test-results.yml @@ -2,7 +2,7 @@ name: Test Results on: workflow_run: - workflows: ["build & test", "metadata ingestion", "Airflow Plugin", "Dagster Plugin"] + workflows: ["build & test", "metadata ingestion", "Airflow Plugin", "Dagster Plugin", "GX Plugin"] types: - completed diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js index bd80a939c074f6..00406c257adbdf 100644 --- a/docs-website/sidebars.js +++ b/docs-website/sidebars.js @@ -896,6 +896,7 @@ module.exports = { //"metadata-ingestion-modules/airflow-plugin/README" //"metadata-ingestion-modules/dagster-plugin/README" //"metadata-ingestion-modules/gx-plugin/README" + // "smoke-test/tests/openapi/README.md" // "metadata-ingestion/schedule_docs/datahub", // we can delete this // TODO: change the titles of these, removing the "What is..." portion from the sidebar" // "docs/what/entity", diff --git a/metadata-ingestion-modules/dagster-plugin/build.gradle b/metadata-ingestion-modules/dagster-plugin/build.gradle index f8692298bc7d15..74ca7cedea3a52 100644 --- a/metadata-ingestion-modules/dagster-plugin/build.gradle +++ b/metadata-ingestion-modules/dagster-plugin/build.gradle @@ -50,10 +50,6 @@ task installDev(type: Exec, dependsOn: [install]) { } task lint(type: Exec, dependsOn: installDev) { - /* - The find/sed combo below is a temporary work-around for the following mypy issue with airflow 2.2.0: - "venv/lib/python3.8/site-packages/airflow/_vendor/connexion/spec.py:169: error: invalid syntax". - */ commandLine 'bash', '-c', "source ${venv_name}/bin/activate && set -x && " + "black --check --diff src/ tests/ examples/ && " + @@ -105,10 +101,6 @@ task testQuick(type: Exec, dependsOn: installDevTest) { } -task testFull(type: Exec, dependsOn: [testQuick, installDevTest]) { - commandLine 'bash', '-x', '-c', - "source ${venv_name}/bin/activate && pytest -m 'not slow_integration' -vv --continue-on-collection-errors --junit-xml=junit.full.xml" -} task buildWheel(type: Exec, dependsOn: [environmentSetup]) { commandLine 'bash', '-c', "source ${venv_name}/bin/activate && " + 'uv pip install build && RELEASE_VERSION="\${RELEASE_VERSION:-0.0.0.dev1}" RELEASE_SKIP_INSTALL=1 RELEASE_SKIP_UPLOAD=1 ./scripts/release.sh' diff --git a/metadata-ingestion-modules/gx-plugin/build.gradle b/metadata-ingestion-modules/gx-plugin/build.gradle index 480e1d63e789a2..f1adbc6676e5bc 100644 --- a/metadata-ingestion-modules/gx-plugin/build.gradle +++ b/metadata-ingestion-modules/gx-plugin/build.gradle @@ -50,10 +50,6 @@ task installDev(type: Exec, dependsOn: [install]) { } task lint(type: Exec, dependsOn: installDev) { - /* - The find/sed combo below is a temporary work-around for the following mypy issue with airflow 2.2.0: - "venv/lib/python3.8/site-packages/airflow/_vendor/connexion/spec.py:169: error: invalid syntax". - */ commandLine 'bash', '-c', "source ${venv_name}/bin/activate && set -x && " + "black --check --diff src/ tests/ && " + @@ -105,10 +101,6 @@ task testQuick(type: Exec, dependsOn: installDevTest) { } -task testFull(type: Exec, dependsOn: [testQuick, installDevTest]) { - commandLine 'bash', '-x', '-c', - "source ${venv_name}/bin/activate && pytest -m 'not slow_integration' -vv --continue-on-collection-errors --junit-xml=junit.full.xml" -} task buildWheel(type: Exec, dependsOn: [environmentSetup]) { commandLine 'bash', '-c', "source ${venv_name}/bin/activate && " + 'uv pip install build && RELEASE_VERSION="\${RELEASE_VERSION:-0.0.0.dev1}" RELEASE_SKIP_INSTALL=1 RELEASE_SKIP_UPLOAD=1 ./scripts/release.sh' diff --git a/metadata-ingestion-modules/gx-plugin/setup.py b/metadata-ingestion-modules/gx-plugin/setup.py index 7583f682ce48bc..dde1350b3cf2df 100644 --- a/metadata-ingestion-modules/gx-plugin/setup.py +++ b/metadata-ingestion-modules/gx-plugin/setup.py @@ -15,12 +15,6 @@ def get_long_description(): rest_common = {"requests", "requests_file"} -# TODO: Is this required here?? -pydantic_no_v2 = { - # pydantic 2 makes major, backwards-incompatible changes - https://github.com/pydantic/pydantic/issues/4887 - # Tags sources that require the pydantic v2 API. - "pydantic<2", -} # TODO: Can we move away from sqllineage and use sqlglot ?? sqllineage_lib = { "sqllineage==1.3.8", @@ -48,9 +42,7 @@ def get_long_description(): "traitlets<5.2.2", *rest_common, *sqllineage_lib, - # Ignoring the dependency below because it causes issues with the vercel built wheel install - # f"acryl-datahub[datahub-rest]{_self_pin}", - "acryl-datahub[datahub-rest]", + f"acryl-datahub[datahub-rest]{_self_pin}", } mypy_stubs = { @@ -105,7 +97,7 @@ def get_long_description(): *dev_requirements, "psycopg2-binary", "pyspark", - "acryl-datahub[testing-utils]", + f"acryl-datahub[testing-utils]{_self_pin}", "pytest-docker>=1.1.0", }