Merge branch 'master' into jetty-version

ryota-cloud · Jan 10, 2025 · 254a9bf · 254a9bf
2 parents 2548f87 + 9897804
commit 254a9bf
Show file tree

Hide file tree

Showing 100 changed files with 2,758 additions and 1,302 deletions.
diff --git a/.github/.codecov.yml b/.github/.codecov.yml
@@ -0,0 +1,65 @@
+comment:
+  layout: "header, files, footer" # remove "new" from "header" and "footer"
+  hide_project_coverage: true # set to false
+  require_changes: false  # if true: only post the comment if coverage changes
+
+codecov:
+  #due to ci-optimization, reports for modules that have not changed may be quite old
+  max_report_age: off
+
+flag_management:
+  default_rules: # the rules that will be followed for any flag added, generally
+    carryforward: true
+    statuses:
+      - type: project
+        target: auto
+        threshold: 0%  #Not enforcing project coverage yet.
+      - type: patch
+        target: 90%
+  individual_flags: # exceptions to the default rules above, stated flag by flag
+    - name: frontend
+      paths:
+        - "datahub-frontend/**"
+        - "datahub-web-react/**"
+    - name: backend
+      paths:
+        - "metadata-models/**"
+        - "datahub-upgrade/**"
+        - "entity-registry/**"
+        - "li-utils/**"
+        - "metadata-auth/**"
+        - "metadata-dao-impl/**"
+        - "metadata-events/**"
+        - "metadata-jobs/**"
+        - "metadata-service/**"
+        - "metadata-utils/**"
+        - "metadata-operation-context/**"
+        - "datahub-graphql-core/**"
+    - name: metadata-io
+      paths:
+        - "metadata-io/**"
+    - name: ingestion
+      paths:
+        - "metadata-ingestion/**"
+    - name: ingestion-airflow
+      paths:
+        - "metadata-ingestion-modules/airflow-plugin/**"
+    - name: ingestion-dagster
+      paths:
+        - "metadata-ingestion-modules/dagster-plugin/**"
+    - name: ingestion-gx-plugin
+      paths:
+        - "metadata-ingestion-modules/gx-plugin/**"
+    - name: ingestion-prefect
+      paths:
+        - "metadata-ingestion-modules/prefect-plugin/**"
+coverage:
+  status:
+    project:
+      default:
+        target: 0% # no threshold enforcement yet
+        only_pulls: true
+    patch:
+      default:
+        target: 90% # for new code added in the patch
+        only_pulls: true
diff --git a/.github/actions/ci-optimization/action.yml b/.github/actions/ci-optimization/action.yml
@@ -13,16 +13,16 @@ outputs:
     value: ${{ steps.filter.outputs.frontend == 'false' && steps.filter.outputs.ingestion == 'false' && steps.filter.outputs.backend == 'true' }}
   backend-change:
     description: "Backend code has changed"
-    value: ${{ steps.filter.outputs.backend == 'true' }}
+    value: ${{ steps.filter.outputs.backend == 'true' ||  steps.trigger.outputs.trigger == 'manual' }}
   ingestion-change:
     description: "Ingestion code has changed"
-    value: ${{ steps.filter.outputs.ingestion == 'true' }}
+    value: ${{ steps.filter.outputs.ingestion == 'true' || steps.trigger.outputs.trigger == 'manual' }}
   ingestion-base-change:
     description: "Ingestion base image docker image has changed"
     value: ${{ steps.filter.outputs.ingestion-base == 'true' }}
   frontend-change:
     description: "Frontend code has changed"
-    value: ${{ steps.filter.outputs.frontend == 'true' }}
+    value: ${{ steps.filter.outputs.frontend == 'true' || steps.trigger.outputs.trigger == 'manual' }}
   docker-change:
     description: "Docker code has changed"
     value: ${{ steps.filter.outputs.docker == 'true' }}
@@ -44,6 +44,15 @@ outputs:
 runs:
   using: "composite"
   steps:
+    - name: Check trigger type
+      id: trigger  # Add an ID to reference this step
+      shell: bash
+      run: |
+        if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
+          echo "trigger=manual" >> $GITHUB_OUTPUT
+        else
+          echo "trigger=pr" >> $GITHUB_OUTPUT
+        fi
     - uses: dorny/paths-filter@v3
       id: filter
       with:

diff --git a/.github/scripts/generate_pre_commit.py b/.github/scripts/generate_pre_commit.py
@@ -9,6 +9,7 @@
 from dataclasses import dataclass
 from enum import Enum, auto
 from pathlib import Path
+import datetime
 
 import yaml
 
@@ -188,6 +189,7 @@ def _generate_lint_fix_hook(self, project: Project) -> dict:
             "entry": f"./gradlew {project.gradle_path}:lintFix",
             "language": "system",
             "files": f"^{project.path}/.*\\.py$",
+            "pass_filenames": False,
         }
 
     def _generate_spotless_hook(self, project: Project) -> dict:
@@ -198,6 +200,7 @@ def _generate_spotless_hook(self, project: Project) -> dict:
             "entry": f"./gradlew {project.gradle_path}:spotlessApply",
             "language": "system",
             "files": f"^{project.path}/.*\\.java$",
+            "pass_filenames": False,
         }
 
 
@@ -209,8 +212,19 @@ def increase_indent(self, flow=False, *args, **kwargs):
 
 
 def write_yaml_with_spaces(file_path: str, data: dict):
-    """Write YAML file with extra spacing between hooks."""
+    """Write YAML file with extra spacing between hooks and a timestamp header."""
     with open(file_path, "w") as f:
+        # Add timestamp header
+        current_time = datetime.datetime.now(datetime.timezone.utc)
+        formatted_time = current_time.strftime("%Y-%m-%d %H:%M:%S %Z")
+        header = f"# Auto-generated by .github/scripts/generate_pre_commit.py at {formatted_time}\n"
+        f.write(header)
+        header = f"# Do not edit this file directly. Run the script to regenerate.\n"
+        f.write(header)
+        header = f"# Add additional hooks in .github/scripts/pre-commit-override.yaml\n"
+        f.write(header)
+
+        # Write the YAML content
         yaml_str = yaml.dump(
             data, Dumper=PrecommitDumper, sort_keys=False, default_flow_style=False
         )

diff --git a/.github/scripts/pre-commit-override.yaml b/.github/scripts/pre-commit-override.yaml
@@ -5,4 +5,5 @@ repos:
         name: smoke-test cypress Lint Fix
         entry: ./gradlew :smoke-test:cypressLintFix
         language: system
-        files: ^smoke-test/tests/cypress/.*$
+        files: ^smoke-test/tests/cypress/.*$
+        pass_filenames: false
diff --git a/.github/workflows/airflow-plugin.yml b/.github/workflows/airflow-plugin.yml
@@ -18,6 +18,7 @@ on:
       - "metadata-models/**"
   release:
     types: [published]
+  workflow_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -77,15 +78,15 @@ jobs:
             **/build/test-results/test/**
             **/junit.*.xml
             !**/binary/**
-      - name: Upload coverage to Codecov
+      - name: Upload coverage to Codecov with ingestion flag
         if: always()
         uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          directory: ./build/coverage-reports/
+          directory: ./build/coverage-reports/metadata-ingestion-modules/airflow-plugin/
           fail_ci_if_error: false
-          flags: airflow-${{ matrix.python-version }}-${{ matrix.extra_pip_extras }}
-          name: pytest-airflow
+          flags: ingestion-airflow
+          name: pytest-airflow-${{ matrix.python-version }}-${{ matrix.extra_pip_requirements }}
           verbose: true
       - name: Upload test results to Codecov
         if: ${{ !cancelled() }}

diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
@@ -12,6 +12,7 @@ on:
     paths-ignore:
       - "docs/**"
       - "**.md"
+  workflow_dispatch:
   release:
     types: [published]
 
@@ -113,25 +114,45 @@ jobs:
         if: ${{  matrix.command == 'except_metadata_ingestion' && needs.setup.outputs.backend_change == 'true' }}
         run: |
           ./gradlew -PjavaClassVersionDefault=8 :metadata-integration:java:spark-lineage:compileJava
+      - name: Gather coverage files
+        run: |
+          echo "BACKEND_FILES=`find ./build/coverage-reports/ -type f | grep -E '(metadata-models|entity-registry|datahuyb-graphql-core|metadata-io|metadata-jobs|metadata-utils|metadata-service|medata-dao-impl|metadata-operation|li-utils|metadata-integration|metadata-events|metadata-auth|ingestion-scheduler|notifications|datahub-upgrade)' | xargs | sed 's/ /,/g'`" >> $GITHUB_ENV
+          echo "FRONTEND_FILES=`find ./build/coverage-reports/ -type f | grep -E '(datahub-frontend|datahub-web-react).*\.(xml|json)$' | xargs | sed 's/ /,/g'`" >> $GITHUB_ENV
+      - name: Generate tz artifact name
+        run: echo "NAME_TZ=$(echo ${{ matrix.timezone }} | tr '/' '-')" >> $GITHUB_ENV
       - uses: actions/upload-artifact@v4
         if: always()
         with:
-          name: Test Results (build)
+          name: Test Results (build) - ${{ matrix.command}}-${{ env.NAME_TZ }}
           path: |
             **/build/reports/tests/test/**
             **/build/test-results/test/**
             **/junit.*.xml
             !**/binary/**
       - name: Ensure codegen is updated
         uses: ./.github/actions/ensure-codegen-updated
-      - name: Upload coverage to Codecov
-        if: always()
+      - name: Upload backend coverage to Codecov
+        if: ${{  matrix.command == 'except_metadata_ingestion' && needs.setup.outputs.backend_change == 'true' }}
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: ${{ env.BACKEND_FILES }}
+          disable_search: true
+          #handle_no_reports_found: true
+          fail_ci_if_error: false
+          flags: backend
+          name: ${{ matrix.command }}
+          verbose: true
+      - name: Upload frontend coverage to Codecov
+        if: ${{  matrix.command == 'frontend' && needs.setup.outputs.frontend_change == 'true' }}
         uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          directory: ./build/coverage-reports/
+          files: ${{ env.FRONTEND_FILES }}
+          disable_search: true
+          #handle_no_reports_found: true
           fail_ci_if_error: false
-          flags: ${{ matrix.timezone }}
+          flags: frontend
           name: ${{ matrix.command }}
           verbose: true
       - name: Upload test results to Codecov

diff --git a/.github/workflows/dagster-plugin.yml b/.github/workflows/dagster-plugin.yml
@@ -18,6 +18,7 @@ on:
       - "metadata-models/**"
   release:
     types: [published]
+  workflow_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -64,14 +65,14 @@ jobs:
             **/build/reports/tests/test/**
             **/build/test-results/test/**
             **/junit.*.xml
-      - name: Upload coverage to Codecov
+      - name: Upload coverage to Codecov with ingestion flag
         if: always()
         uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          directory: ./build/coverage-reports/
+          directory: ./build/coverage-reports/metadata-ingestion-modules/dagster-plugin/
           fail_ci_if_error: false
-          flags: dagster-${{ matrix.python-version }}-${{ matrix.extraPythonRequirement }}
+          flags: ingestion-dagster-plugin
           name: pytest-dagster
           verbose: true
       - name: Upload test results to Codecov

diff --git a/.github/workflows/gx-plugin.yml b/.github/workflows/gx-plugin.yml
@@ -18,6 +18,7 @@ on:
       - "metadata-models/**"
   release:
     types: [published]
+  workflow_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -68,14 +69,14 @@ jobs:
             **/build/reports/tests/test/**
             **/build/test-results/test/**
             **/junit.*.xml
-      - name: Upload coverage to Codecov
+      - name: Upload coverage to Codecov with ingestion flag
         if: always()
         uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          directory: ./build/coverage-reports/
+          directory: ./build/coverage-reports/metadata-ingestion-modules/gx-plugin/
           fail_ci_if_error: false
-          flags: gx-${{ matrix.python-version }}-${{ matrix.extraPythonRequirement }}
+          flags: ingestion-gx-plugin
           name: pytest-gx
           verbose: true
       - name: Upload test results to Codecov

diff --git a/.github/workflows/metadata-ingestion.yml b/.github/workflows/metadata-ingestion.yml
@@ -18,6 +18,7 @@ on:
       - "metadata-models/**"
   release:
     types: [published]
+  workflow_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -88,15 +89,15 @@ jobs:
             **/build/test-results/test/**
             **/junit.*.xml
             !**/binary/**
-      - name: Upload coverage to Codecov
-        if: ${{ always() }}
+      - name: Upload coverage to Codecov with ingestion flag
+        if: ${{ always() && matrix.python-version == '3.11' }}
         uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          directory: ./build/coverage-reports/
+          directory: ./build/coverage-reports/metadata-ingestion/
           fail_ci_if_error: false
-          flags: ingestion-${{ matrix.python-version }}-${{ matrix.command }}
-          name: pytest-ingestion
+          flags: ingestion
+          name: pytest-${{ matrix.python-version }}-${{ matrix.command }}
           verbose: true
       - name: Upload test results to Codecov
         if: ${{ !cancelled() }}

diff --git a/.github/workflows/metadata-io.yml b/.github/workflows/metadata-io.yml
@@ -20,6 +20,7 @@ on:
       - ".github/workflows/metadata-io.yml"
   release:
     types: [published]
+  workflow_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -86,8 +87,9 @@ jobs:
         uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          directory: ./build/coverage-reports/
+          directory: ./build/coverage-reports/metadata-io/
           fail_ci_if_error: false
+          flags: metadata-io
           name: metadata-io-test
           verbose: true
       - name: Upload test results to Codecov

diff --git a/.github/workflows/prefect-plugin.yml b/.github/workflows/prefect-plugin.yml
@@ -18,6 +18,7 @@ on:
       - "metadata-models/**"
   release:
     types: [published]
+  workflow_dispatch:
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -60,15 +61,15 @@ jobs:
             **/build/test-results/test/**
             **/junit.*.xml
             !**/binary/**
-      - name: Upload coverage to Codecov
+      - name: Upload coverage to Codecov with ingestion flag
         if: always()
         uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          directory: ./build/coverage-reports/
+          directory: ./build/coverage-reports/metadata-ingestion-modules/prefect-plugin/
           fail_ci_if_error: false
-          flags: prefect-${{ matrix.python-version }}
-          name: pytest-prefect
+          flags: ingestion-prefect-plugin
+          name: pytest-prefect-${{ matrix.python-version }}
           verbose: true
       - name: Upload test results to Codecov
         if: ${{ !cancelled() }}