Merge branch 'datahub-project:master' into master

.github/.codecov.yml

@@ -0,0 +1,65 @@
+comment:
+  layout: "header, files, footer" # remove "new" from "header" and "footer"
+  hide_project_coverage: true # set to false
+  require_changes: false  # if true: only post the comment if coverage changes
+
+codecov:
+  #due to ci-optimization, reports for modules that have not changed may be quite old
+  max_report_age: off
+
+flag_management:
+  default_rules: # the rules that will be followed for any flag added, generally
+    carryforward: true
+    statuses:
+      - type: project
+        target: auto
+        threshold: 0%  #Not enforcing project coverage yet.
+      - type: patch
+        target: 90%
+  individual_flags: # exceptions to the default rules above, stated flag by flag
+    - name: frontend
+      paths:
+        - "datahub-frontend/**"
+        - "datahub-web-react/**"
+    - name: backend
+      paths:
+        - "metadata-models/**"
+        - "datahub-upgrade/**"
+        - "entity-registry/**"
+        - "li-utils/**"
+        - "metadata-auth/**"
+        - "metadata-dao-impl/**"
+        - "metadata-events/**"
+        - "metadata-jobs/**"
+        - "metadata-service/**"
+        - "metadata-utils/**"
+        - "metadata-operation-context/**"
+        - "datahub-graphql-core/**"
+    - name: metadata-io
+      paths:
+        - "metadata-io/**"
+    - name: ingestion
+      paths:
+        - "metadata-ingestion/**"
+    - name: ingestion-airflow
+      paths:
+        - "metadata-ingestion-modules/airflow-plugin/**"
+    - name: ingestion-dagster
+      paths:
+        - "metadata-ingestion-modules/dagster-plugin/**"
+    - name: ingestion-gx-plugin
+      paths:
+        - "metadata-ingestion-modules/gx-plugin/**"
+    - name: ingestion-prefect
+      paths:
+        - "metadata-ingestion-modules/prefect-plugin/**"
+coverage:
+  status:
+    project:
+      default:
+        target: 0% # no threshold enforcement yet
+        only_pulls: true
+    patch:
+      default:
+        target: 90% # for new code added in the patch
+        only_pulls: true

.github/scripts/generate_pre_commit.py

@@ -9,6 +9,7 @@
 from dataclasses import dataclass
 from enum import Enum, auto
 from pathlib import Path
+import datetime
 
 import yaml
 
@@ -188,6 +189,7 @@ def _generate_lint_fix_hook(self, project: Project) -> dict:
             "entry": f"./gradlew {project.gradle_path}:lintFix",
             "language": "system",
             "files": f"^{project.path}/.*\\.py$",
+            "pass_filenames": False,
         }
 
     def _generate_spotless_hook(self, project: Project) -> dict:
@@ -198,6 +200,7 @@ def _generate_spotless_hook(self, project: Project) -> dict:
             "entry": f"./gradlew {project.gradle_path}:spotlessApply",
             "language": "system",
             "files": f"^{project.path}/.*\\.java$",
+            "pass_filenames": False,
         }
 
 
@@ -209,8 +212,19 @@ def increase_indent(self, flow=False, *args, **kwargs):
 
 
 def write_yaml_with_spaces(file_path: str, data: dict):
-    """Write YAML file with extra spacing between hooks."""
+    """Write YAML file with extra spacing between hooks and a timestamp header."""
     with open(file_path, "w") as f:
+        # Add timestamp header
+        current_time = datetime.datetime.now(datetime.timezone.utc)
+        formatted_time = current_time.strftime("%Y-%m-%d %H:%M:%S %Z")
+        header = f"# Auto-generated by .github/scripts/generate_pre_commit.py at {formatted_time}\n"
+        f.write(header)
+        header = f"# Do not edit this file directly. Run the script to regenerate.\n"
+        f.write(header)
+        header = f"# Add additional hooks in .github/scripts/pre-commit-override.yaml\n"
+        f.write(header)
+
+        # Write the YAML content
         yaml_str = yaml.dump(
             data, Dumper=PrecommitDumper, sort_keys=False, default_flow_style=False
         )

.github/scripts/pre-commit-override.yaml

@@ -5,4 +5,5 @@ repos:
         name: smoke-test cypress Lint Fix
         entry: ./gradlew :smoke-test:cypressLintFix
         language: system
-        files: ^smoke-test/tests/cypress/.*$
+        files: ^smoke-test/tests/cypress/.*$
+        pass_filenames: false

.github/workflows/airflow-plugin.yml

@@ -77,15 +77,15 @@ jobs:
             **/build/test-results/test/**
             **/junit.*.xml
             !**/binary/**
-      - name: Upload coverage to Codecov
+      - name: Upload coverage to Codecov with ingestion flag
         if: always()
         uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          directory: ./build/coverage-reports/
+          directory: ./build/coverage-reports/metadata-ingestion-modules/airflow-plugin/
           fail_ci_if_error: false
-          flags: airflow-${{ matrix.python-version }}-${{ matrix.extra_pip_extras }}
-          name: pytest-airflow
+          flags: ingestion-airflow
+          name: pytest-airflow-${{ matrix.python-version }}-${{ matrix.extra_pip_requirements }}
           verbose: true
       - name: Upload test results to Codecov
         if: ${{ !cancelled() }}

.github/workflows/build-and-test.yml

@@ -113,6 +113,10 @@ jobs:
         if: ${{  matrix.command == 'except_metadata_ingestion' && needs.setup.outputs.backend_change == 'true' }}
         run: |
           ./gradlew -PjavaClassVersionDefault=8 :metadata-integration:java:spark-lineage:compileJava
+      - name: Gather coverage files
+        run: |
+          echo "BACKEND_FILES=`find ./build/coverage-reports/ -type f | grep -E '(metadata-models|entity-registry|datahuyb-graphql-core|metadata-io|metadata-jobs|metadata-utils|metadata-service|medata-dao-impl|metadata-operation|li-utils|metadata-integration|metadata-events|metadata-auth|ingestion-scheduler|notifications|datahub-upgrade)' | xargs | sed 's/ /,/g'`" >> $GITHUB_ENV
+          echo "FRONTEND_FILES=`find ./build/coverage-reports/ -type f | grep -E '(datahub-frontend|datahub-web-react).*\.(xml|json)$' | xargs | sed 's/ /,/g'`" >> $GITHUB_ENV
       - uses: actions/upload-artifact@v4
         if: always()
         with:
@@ -124,14 +128,28 @@ jobs:
             !**/binary/**
       - name: Ensure codegen is updated
         uses: ./.github/actions/ensure-codegen-updated
-      - name: Upload coverage to Codecov
-        if: always()
+      - name: Upload backend coverage to Codecov
+        if: ${{  matrix.command == 'except_metadata_ingestion' && needs.setup.outputs.backend_change == 'true' }}
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: ${{ env.BACKEND_FILES }}
+          disable_search: true
+          #handle_no_reports_found: true
+          fail_ci_if_error: false
+          flags: backend
+          name: ${{ matrix.command }}
+          verbose: true
+      - name: Upload frontend coverage to Codecov
+        if: ${{  matrix.command == 'frontend' && needs.setup.outputs.frontend_change == 'true' }}
         uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          directory: ./build/coverage-reports/
+          files: ${{ env.FRONTEND_FILES }}
+          disable_search: true
+          #handle_no_reports_found: true
           fail_ci_if_error: false
-          flags: ${{ matrix.timezone }}
+          flags: frontend
           name: ${{ matrix.command }}
           verbose: true
       - name: Upload test results to Codecov

.github/workflows/dagster-plugin.yml

@@ -64,14 +64,14 @@ jobs:
             **/build/reports/tests/test/**
             **/build/test-results/test/**
             **/junit.*.xml
-      - name: Upload coverage to Codecov
+      - name: Upload coverage to Codecov with ingestion flag
         if: always()
         uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          directory: ./build/coverage-reports/
+          directory: ./build/coverage-reports/metadata-ingestion-modules/dagster-plugin/
           fail_ci_if_error: false
-          flags: dagster-${{ matrix.python-version }}-${{ matrix.extraPythonRequirement }}
+          flags: ingestion-dagster-plugin
           name: pytest-dagster
           verbose: true
       - name: Upload test results to Codecov

.github/workflows/gx-plugin.yml

@@ -68,14 +68,14 @@ jobs:
             **/build/reports/tests/test/**
             **/build/test-results/test/**
             **/junit.*.xml
-      - name: Upload coverage to Codecov
+      - name: Upload coverage to Codecov with ingestion flag
         if: always()
         uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          directory: ./build/coverage-reports/
+          directory: ./build/coverage-reports/metadata-ingestion-modules/gx-plugin/
           fail_ci_if_error: false
-          flags: gx-${{ matrix.python-version }}-${{ matrix.extraPythonRequirement }}
+          flags: ingestion-gx-plugin
           name: pytest-gx
           verbose: true
       - name: Upload test results to Codecov

.github/workflows/metadata-ingestion.yml

@@ -88,15 +88,15 @@ jobs:
             **/build/test-results/test/**
             **/junit.*.xml
             !**/binary/**
-      - name: Upload coverage to Codecov
-        if: ${{ always() }}
+      - name: Upload coverage to Codecov with ingestion flag
+        if: ${{ always() && matrix.python-version == '3.11' }}
         uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          directory: ./build/coverage-reports/
+          directory: ./build/coverage-reports/metadata-ingestion/
           fail_ci_if_error: false
-          flags: ingestion-${{ matrix.python-version }}-${{ matrix.command }}
-          name: pytest-ingestion
+          flags: ingestion
+          name: pytest-${{ matrix.python-version }}-${{ matrix.command }}
           verbose: true
       - name: Upload test results to Codecov
         if: ${{ !cancelled() }}

.github/workflows/metadata-io.yml

@@ -86,8 +86,9 @@ jobs:
         uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          directory: ./build/coverage-reports/
+          directory: ./build/coverage-reports/metadata-io/
           fail_ci_if_error: false
+          flags: metadata-io
           name: metadata-io-test
           verbose: true
       - name: Upload test results to Codecov

.github/workflows/prefect-plugin.yml

@@ -60,15 +60,15 @@ jobs:
             **/build/test-results/test/**
             **/junit.*.xml
             !**/binary/**
-      - name: Upload coverage to Codecov
+      - name: Upload coverage to Codecov with ingestion flag
         if: always()
         uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          directory: ./build/coverage-reports/
+          directory: ./build/coverage-reports/metadata-ingestion-modules/prefect-plugin/
           fail_ci_if_error: false
-          flags: prefect-${{ matrix.python-version }}
-          name: pytest-prefect
+          flags: ingestion-prefect-plugin
+          name: pytest-prefect-${{ matrix.python-version }}
           verbose: true
       - name: Upload test results to Codecov
         if: ${{ !cancelled() }}