Merge branch 'develop' into helm-docs-github-action

.github/scripts/validate_charts.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+
+import glob
+import os
+import re
+import sys
+from difflib import unified_diff
+from pathlib import Path
+
+import yaml
+
+
+def load_yaml_file(file_path):
+    if not os.path.exists(file_path):
+        print(f"Error: File not found: {file_path}")
+        return None, None
+
+    try:
+        with open(file_path, "r", encoding="utf-8") as f:
+            raw_content = f.read()
+            yaml_content = yaml.safe_load(raw_content)
+            return yaml_content, raw_content
+    except Exception as e:
+        print(f"Error loading {file_path}: {e}")
+        return None, None
+
+
+def compare_yaml_files(file1, file2):
+    result1 = load_yaml_file(file1)
+    result2 = load_yaml_file(file2)
+
+    if result1[0] is None or result2[0] is None:
+        return False
+
+    content1, raw1 = result1
+    content2, raw2 = result2
+
+    if content1 != content2:
+        print("❌ Files are not in sync!")
+        print(f"Source file: {file1}")
+        print(f"Target file: {file2}")
+        print("-" * 80)
+
+        # Generate and print git-style diff
+        diff = unified_diff(
+            raw1.splitlines(keepends=True),
+            raw2.splitlines(keepends=True),
+            fromfile="Source",
+            tofile="Target",
+            lineterm="",
+        )
+        print("".join(list(diff)))
+        return False
+    return True
+
+
+def validate_production_files():
+    success = True
+
+    # Check values files sync
+    chart_values = glob.glob("charts/**/values/production*.yaml", recursive=True)
+    for chart_value_file in chart_values:
+        service_name = Path(chart_value_file).parts[-3]
+        production_file = Path(chart_value_file).parts[-1]  # e.g. "production-0.yaml"
+
+        # Construct the corresponding example file path
+        example_file = f"examples/values/{service_name}-{production_file}"
+
+        if os.path.exists(example_file):
+            if not compare_yaml_files(chart_value_file, example_file):
+                success = False
+        else:
+            print(f"❌ Missing example values file: {example_file}")
+            success = False
+
+    return success
+
+
+def validate_example_makefile():
+    makefile_path = "examples/Makefile.example"
+    if not os.path.exists(makefile_path):
+        return True
+
+    with open(makefile_path, "r") as f:
+        # Replace line continuations (\newline) with spaces
+        makefile_content = re.sub(r"\\\n\s*", " ", f.read())
+
+    # Extract version patterns from helm upgrade commands
+    version_patterns = re.findall(
+        r"helm upgrade -i ([a-zA-Z0-9-]+)\s+oci://[^\s]+\s+.*?--version=(\d+\.\d+\.\d+)",
+        makefile_content,
+    )
+
+    success = True
+    for service, version in version_patterns:
+        # Handle special cases like l2-sequencer-0 -> l2-sequencer
+        base_service = re.sub(r"-\d+$", "", service)
+
+        # Find corresponding Chart.yaml
+        chart_file = f"charts/{base_service}/Chart.yaml"
+        if not os.path.exists(chart_file):
+            print(f"❌ Chart file not found for service: {base_service}")
+            success = False
+            continue
+
+        chart_data = load_yaml_file(chart_file)
+        if not chart_data[0]:
+            continue
+
+        chart_version = chart_data[0].get("version")
+        if version != chart_version:
+            print(f"❌ Version mismatch in Makefile.example for {base_service}")
+            print(f"  Makefile version: {version}")
+            print(f"  Chart version: {chart_version}")
+            success = False
+
+    return success
+
+
+def main():
+    success = True
+
+    # Check production files sync
+    if not validate_production_files():
+        success = False
+
+    # Check example Makefile versions
+    if not validate_example_makefile():
+        success = False
+
+    if not success:
+        sys.exit(1)
+
+    print("✅ All checks passed!")
+
+
+if __name__ == "__main__":
+    main()

.github/workflows/chart-sync-check.yaml

@@ -0,0 +1,32 @@
+name: Chart Sync Validation
+
+on:
+  push:
+    paths:
+      - "charts/**/values/production.yaml"
+      - "examples/values/*-production.yaml"
+      - "charts/**/Chart.yaml"
+      - "examples/Makefile.example"
+  pull_request:
+    paths:
+      - "charts/**/values/production.yaml"
+      - "examples/values/*-production.yaml"
+      - "charts/**/Chart.yaml"
+      - "examples/Makefile.example"
+
+jobs:
+  validate-chart-sync:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.x"
+
+      - name: Install dependencies
+        run: pip install pyyaml
+
+      - name: Validate Chart Files
+        run: python .github/scripts/validate_charts.py

.github/workflows/lint-test-charts.yml

@@ -29,7 +29,7 @@ jobs:
         uses: helm/[email protected]
 
       - name: Setup scroll-sdk values
-        run: cd charts/scroll-sdk && docker run --rm -t -v .:/contracts/volume scrolltech/scroll-stack-contracts:gen-configs-v0.0.9
+        run: cd charts/scroll-sdk && docker run --rm -t -v .:/contracts/volume scrolltech/scroll-stack-contracts:gen-configs-50c2661f6d3a40c34cc5b4fffbd0d7ba59d3dbae
 
       - name: Helm registry login
         run: |
@@ -47,4 +47,4 @@ jobs:
 
       - name: Run chart-testing (lint)
         if: steps.list-changed.outputs.changed == 'true'
-        run: ct lint --config ct.yaml --target-branch ${{ github.event.repository.default_branch }}
+        run: ct lint --config ct.yaml --excluded-charts scroll-sdk --target-branch ${{ github.event.repository.default_branch }}

.github/workflows/publish-chart.yaml

@@ -7,9 +7,108 @@ on:
       - 'charts/**'
 
 jobs:
-  helm-chart-testing:
+  helm-chart-testing-not-scroll-sdk:
+    name: chart-testing-without-scroll-sdk
     runs-on: ubuntu-latest
     environment: test
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Set up Helm
+        uses: azure/setup-helm@v3
+        with:
+          version: v3.12.1
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.x'
+          check-latest: true
+
+      - name: Set up chart-testing
+        uses: helm/[email protected]
+
+      - name: Helm registry login
+        run: |
+          helm registry login ghcr.io/scroll-tech/helm/scroll-sdk --username ${{ github.actor }} --password ${{ secrets.GITHUB_TOKEN }}
+        env:
+          HELM_GITHUB_PASSWORD: "${{ secrets.HELM_GITHUB_PASSWORD }}"
+
+      #  List chart change except scroll-sdk
+      - name: Run chart-testing (list-changed)
+        id: list-changed
+        run: |
+          changed=$(ct list-changed --excluded-charts scroll-sdk --target-branch ${{ github.event.repository.default_branch }} )
+          if [[ -n "$changed" ]]; then
+            echo "changed=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Run chart-testing (lint)
+        if: steps.list-changed.outputs.changed == 'true'
+        run: ct lint --config ct.yaml --excluded-charts scroll-sdk --target-branch ${{ github.event.repository.default_branch }}
+
+  publish-ghcr-not-scroll-sdk:
+    name: publish-to-github-without-scroll-sdk
+    runs-on: ubuntu-latest
+    needs:
+      - helm-chart-testing-not-scroll-sdk
+    outputs:
+      charts: ${{ steps.list-changed.outputs.changed }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - uses: dorny/paths-filter@v2
+        id: filter
+        with:
+          base: ${{ github.event.repository.default_branch }}
+          list-files: shell
+          filters: |
+            addedOrModified:
+              - added|modified: 'charts/**'
+
+      - name: Helm registry login
+        run: |
+          helm registry login ghcr.io/scroll-tech/helm/scroll-sdk --username ${{ github.actor }} --password ${{ secrets.GITHUB_TOKEN }}
+        env:
+          HELM_GITHUB_PASSWORD: "${{ secrets.HELM_GITHUB_PASSWORD }}"
+
+        # We filter here the scroll-sdk chart
+      - name: Push chart to ghcr
+        if: steps.filter.outputs.addedOrModified == 'true'
+        env:
+          HELM_EXPERIMENTAL_OCI: 1
+        run: |
+          set -x
+          CHARTS=()
+          PATHS=(${{ steps.filter.outputs.addedOrModified_files }})
+          echo ${PATHS}
+          # Get only the chart paths
+          for i in "${PATHS[@]}"
+          do
+            chart=$(echo $i | awk -F "/" '{print $2}')
+            if [ "$chart" != "scroll-sdk" ]; then
+            CHARTS+=($chart)
+            fi
+          done
+          # Remove duplicates
+          CHARTS=( `for i in ${CHARTS[@]}; do echo $i; done | sort -u` )
+          echo "CHARTS: ${CHARTS[@]}"
+          for chart in ${CHARTS[@]}; do
+            helm dependencies build charts/$chart
+            helm package charts/$chart
+            export CHART_VERSION=$(grep 'version:' charts/$chart/Chart.yaml | head -n1 | awk '{ print $2 }')
+            helm push $chart-${CHART_VERSION}.tgz oci://ghcr.io/scroll-tech/scroll-sdk/helm
+          done
+
+  helm-chart-testing-scroll-sdk:
+    name: chart-testing-scroll-sdk
+    runs-on: ubuntu-latest
+    needs:
+      - publish-ghcr-not-scroll-sdk
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -30,31 +129,32 @@ jobs:
         uses: helm/[email protected]
 
       - name: Setup scroll-sdk values
-        run: cd charts/scroll-sdk && docker run --rm -t -v .:/contracts/volume scrolltech/scroll-stack-contracts:gen-configs-v0.0.9
+        run: cd charts/scroll-sdk && docker run --rm -t -v .:/contracts/volume scrolltech/scroll-stack-contracts:gen-configs-50c2661f6d3a40c34cc5b4fffbd0d7ba59d3dbae
 
       - name: Helm registry login
         run: |
           helm registry login ghcr.io/scroll-tech/helm/scroll-sdk --username ${{ github.actor }} --password ${{ secrets.GITHUB_TOKEN }}
         env:
           HELM_GITHUB_PASSWORD: "${{ secrets.HELM_GITHUB_PASSWORD }}"
 
+      #  List scroll-sdk chart if it has changed
       - name: Run chart-testing (list-changed)
         id: list-changed
         run: |
-          changed=$(ct list-changed --target-branch ${{ github.event.repository.default_branch }})
+          changed=$(ct list-changed --chart-dirs charts/scroll-sdk --target-branch ${{ github.event.repository.default_branch }} )
           if [[ -n "$changed" ]]; then
             echo "changed=true" >> "$GITHUB_OUTPUT"
           fi
 
       - name: Run chart-testing (lint)
         if: steps.list-changed.outputs.changed == 'true'
-        run: ct lint --config ct.yaml --target-branch ${{ github.event.repository.default_branch }}
+        run: ct lint --config ct.yaml --charts scroll-sdk --target-branch ${{ github.event.repository.default_branch }}
 
-  publish-ghcr:
-    name: publish-to-github
+  publish-ghcr-scroll-sdk:
+    name: publish-scroll-sdk-to-github
     runs-on: ubuntu-latest
     needs:
-      - helm-chart-testing
+      - helm-chart-testing-scroll-sdk
     outputs:
       charts: ${{ steps.list-changed.outputs.changed }}
     steps:
@@ -65,11 +165,11 @@ jobs:
       - uses: dorny/paths-filter@v2
         id: filter
         with:
-          base: ${{ github.ref }}
+          base: ${{ github.event.repository.default_branch }}
           list-files: shell
           filters: |
             addedOrModified:
-              - added|modified: 'charts/**'
+              - added|modified: 'charts/scroll-sdk/**'
 
       - name: Helm registry login
         run: |