Dev updates 2024-12-03

.circleci/config.yml

@@ -41,94 +41,94 @@ workflows:
             - build-and-release-internal
 
       - airflow-test:
-          name: test-1-28-13-CeleryExecutor
-          kube_version: 1.28.13
+          name: test-1-28-15-CeleryExecutor
+          kube_version: 1.28.15
           executor: CeleryExecutor
           requires:
             - build-and-release-internal
 
             - approve-test-all
       - airflow-test:
-          name: test-1-28-13-LocalExecutor
-          kube_version: 1.28.13
+          name: test-1-28-15-LocalExecutor
+          kube_version: 1.28.15
           executor: LocalExecutor
           requires:
             - build-and-release-internal
 
             - approve-test-all
       - airflow-test:
-          name: test-1-28-13-KubernetesExecutor
-          kube_version: 1.28.13
+          name: test-1-28-15-KubernetesExecutor
+          kube_version: 1.28.15
           executor: KubernetesExecutor
           requires:
             - build-and-release-internal
 
             - approve-test-all
       - airflow-test:
-          name: test-1-29-8-CeleryExecutor
-          kube_version: 1.29.8
+          name: test-1-29-10-CeleryExecutor
+          kube_version: 1.29.10
           executor: CeleryExecutor
           requires:
             - build-and-release-internal
 
             - approve-test-all
       - airflow-test:
-          name: test-1-29-8-LocalExecutor
-          kube_version: 1.29.8
+          name: test-1-29-10-LocalExecutor
+          kube_version: 1.29.10
           executor: LocalExecutor
           requires:
             - build-and-release-internal
 
             - approve-test-all
       - airflow-test:
-          name: test-1-29-8-KubernetesExecutor
-          kube_version: 1.29.8
+          name: test-1-29-10-KubernetesExecutor
+          kube_version: 1.29.10
           executor: KubernetesExecutor
           requires:
             - build-and-release-internal
 
             - approve-test-all
       - airflow-test:
-          name: test-1-30-4-CeleryExecutor
-          kube_version: 1.30.4
+          name: test-1-30-6-CeleryExecutor
+          kube_version: 1.30.6
           executor: CeleryExecutor
           requires:
             - build-and-release-internal
 
             - approve-test-all
       - airflow-test:
-          name: test-1-30-4-LocalExecutor
-          kube_version: 1.30.4
+          name: test-1-30-6-LocalExecutor
+          kube_version: 1.30.6
           executor: LocalExecutor
           requires:
             - build-and-release-internal
 
             - approve-test-all
       - airflow-test:
-          name: test-1-30-4-KubernetesExecutor
-          kube_version: 1.30.4
+          name: test-1-30-6-KubernetesExecutor
+          kube_version: 1.30.6
           executor: KubernetesExecutor
           requires:
             - build-and-release-internal
 
             - approve-test-all
       - airflow-test:
-          name: test-1-31-0-CeleryExecutor
-          kube_version: 1.31.0
+          name: test-1-31-2-CeleryExecutor
+          kube_version: 1.31.2
           executor: CeleryExecutor
           requires:
             - build-and-release-internal
 
       - airflow-test:
-          name: test-1-31-0-LocalExecutor
-          kube_version: 1.31.0
+          name: test-1-31-2-LocalExecutor
+          kube_version: 1.31.2
           executor: LocalExecutor
           requires:
             - build-and-release-internal
 
       - airflow-test:
-          name: test-1-31-0-KubernetesExecutor
-          kube_version: 1.31.0
+          name: test-1-31-2-KubernetesExecutor
+          kube_version: 1.31.2
           executor: KubernetesExecutor
           requires:
             - build-and-release-internal
@@ -139,18 +139,18 @@ workflows:
             - "test-1-27-16-CeleryExecutor"
             - "test-1-27-16-LocalExecutor"
             - "test-1-27-16-KubernetesExecutor"
-            - "test-1-28-13-CeleryExecutor"
-            - "test-1-28-13-LocalExecutor"
-            - "test-1-28-13-KubernetesExecutor"
-            - "test-1-29-8-CeleryExecutor"
-            - "test-1-29-8-LocalExecutor"
-            - "test-1-29-8-KubernetesExecutor"
-            - "test-1-30-4-CeleryExecutor"
-            - "test-1-30-4-LocalExecutor"
-            - "test-1-30-4-KubernetesExecutor"
-            - "test-1-31-0-CeleryExecutor"
-            - "test-1-31-0-LocalExecutor"
-            - "test-1-31-0-KubernetesExecutor"
+            - "test-1-28-15-CeleryExecutor"
+            - "test-1-28-15-LocalExecutor"
+            - "test-1-28-15-KubernetesExecutor"
+            - "test-1-29-10-CeleryExecutor"
+            - "test-1-29-10-LocalExecutor"
+            - "test-1-29-10-KubernetesExecutor"
+            - "test-1-30-6-CeleryExecutor"
+            - "test-1-30-6-LocalExecutor"
+            - "test-1-30-6-KubernetesExecutor"
+            - "test-1-31-2-CeleryExecutor"
+            - "test-1-31-2-LocalExecutor"
+            - "test-1-31-2-KubernetesExecutor"
           filters:
             branches:
               only:
@@ -171,7 +171,7 @@ workflows:
 jobs:
   run_pre_commit:
     docker:
-      - image: quay.io/astronomer/ci-pre-commit:2024-09
+      - image: quay.io/astronomer/ci-pre-commit:2024-12
     steps:
       - checkout
       - run:
@@ -206,7 +206,7 @@ jobs:
 
   unittest-charts:
     docker:
-      - image: quay.io/astronomer/ci-helm-release:2024-09
+      - image: quay.io/astronomer/ci-helm-release:2024-12
     parallelism: 8
     steps:
       - setup_remote_docker:
@@ -225,7 +225,7 @@ jobs:
 
   build-and-release-internal:
     docker:
-      - image: quay.io/astronomer/ci-helm-release:2024-09
+      - image: quay.io/astronomer/ci-helm-release:2024-12
     steps:
       - checkout
       - run:
@@ -239,7 +239,7 @@ jobs:
   airflow-test:
     machine:
       # https://circleci.com/developer/machine/image/ubuntu-2204
-      image: ubuntu-2204:2024.05.1
+      image: ubuntu-2204:2024.11.1
       resource_class: large
     parameters:
       executor:
@@ -265,7 +265,7 @@ jobs:
           path: test-results
   release-internal:
     docker:
-      - image: quay.io/astronomer/ci-helm-release:2024-09
+      - image: quay.io/astronomer/ci-helm-release:2024-12
     steps:
       - checkout
       - run:
@@ -274,7 +274,7 @@ jobs:
 
   release-public:
     docker:
-      - image: quay.io/astronomer/ci-helm-release:2024-09
+      - image: quay.io/astronomer/ci-helm-release:2024-12
     steps:
       - checkout
       - publish-github-release

.github/workflows/pr_checks.yml

@@ -0,0 +1,43 @@
+name: Check PR description
+
+on:
+  # https://docs.github.com/en/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows#pull_request
+  pull_request:
+    types:
+      - edited
+      - labeled
+      - opened
+      - reopened
+      - synchronize
+      - unlabeled
+    branches:
+      - master
+
+jobs:
+  trigger-build:
+    # https://docs.github.com/en/actions/writing-workflows/choosing-where-your-workflow-runs/choosing-the-runner-for-a-job
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check PR description
+        env:
+          PR_BODY: ${{ github.event.pull_request.body }}
+          KTLO: ${{ contains(github.event.pull_request.labels.*.name, 'ktlo') }}
+        run: |
+          errors=0
+
+          if [[ "${PR_BODY}" == *"Do not merge"* ]] ; then
+            echo "PR description contains 'Do not merge'"
+            ((errors += 1))
+          fi
+
+          if [[ "${KTLO}" != "true" ]] && [[ "${PR_BODY}" != *"astronomer/issues"* ]] ; then
+            echo "PR description does not contain an issue link"
+            ((errors += 1))
+          fi
+
+          if [[ "${#PR_BODY}" -lt 20 ]] ; then
+            echo "PR description is too short"
+            ((errors += 1))
+          fi
+
+          exit "$errors"

.pre-commit-config.yaml

@@ -2,19 +2,15 @@
 exclude: '^(venv|\.vscode|tests/k8s_schema|tests/chart/test_data)' # regex
 repos:
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: "v0.6.4"
+    rev: "v0.8.1"
     hooks:
       - id: ruff
         args:
           - "--fix"
           - "--exit-non-zero-on-fix"
           # - "--unsafe-fixes"
-  - repo: https://github.com/psf/black
-    rev: 24.8.0
-    hooks:
-      - id: black
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v5.0.0
     hooks:
       - id: check-case-conflict
       - id: check-executables-have-shebangs

Makefile

@@ -2,10 +2,10 @@
 
 .PHONY: help
 help: ## Print Makefile help.
-	@grep -Eh '^[a-z.A-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-25s\033[0m %s\n", $$1, $$2}'
+	@grep -Eh '^[a-z.A-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[1;36m%-20s\033[0m %s\n", $$1, $$2}'
 
 venv: ## Setup venv required for unit testing the helm chart
-	virtualenv venv -p python3
+	uv venv venv -p 3.11 --seed || virtualenv venv -p python3
 	venv/bin/pip install -r requirements/chart.txt
 
 charts: ## Update dependent charts

bin/generate_circleci_config.py

@@ -2,17 +2,17 @@
 """This script is used to create the circle config file so that we can stay
 DRY."""
 from pathlib import Path
-import yaml
 
+import yaml
 from jinja2 import Template
 
 git_root_dir = next(iter([x for x in Path(__file__).resolve().parents if (x / ".git").is_dir()]), None)
 metadata = yaml.safe_load((git_root_dir / "metadata.yaml").read_text())
 kube_versions = metadata["test_k8s_versions"]
 
 executors = ["CeleryExecutor", "LocalExecutor", "KubernetesExecutor"]
-machine_image_version = "ubuntu-2204:2024.05.1"
-ci_runner_version = "2024-09"
+machine_image_version = "ubuntu-2204:2024.11.1" # https://circleci.com/developer/machine/image/ubuntu-2204
+ci_runner_version = "2024-12" # This should be the current YYYY-MM
 
 
 def main():

bin/get-k8s-versions.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import argparse
+
 import requests
 import yaml
 

bin/install-ci-tools

@@ -1,8 +1,8 @@
 #!/bin/bash
 set -e
 
-export KIND_VERSION="${KIND_VERSION:-0.24.0}"
-export HELM_VERSION="${HELM_VERSION:-3.16.1}"
+export KIND_VERSION="${KIND_VERSION:-0.25.0}"
+export HELM_VERSION="${HELM_VERSION:-3.16.2}"
 
 # Determine the platform we are running on
 OS=$(uname | tr '[:upper:]' '[:lower:]')

bin/start-kind-cluster

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 # shellcheck disable=SC2181
 
-KUBE_VERSION="${KUBE_VERSION:-1.30.2}"
+KUBE_VERSION="${KUBE_VERSION:-1.30.6}"
 
 # Create a kind cluster
 kind create cluster --image "kindest/node:v${KUBE_VERSION}"

metadata.yaml

@@ -2,7 +2,7 @@
 # https://hub.docker.com/r/kindest/node/tags
 test_k8s_versions:
   - 1.27.16
-  - 1.28.13
-  - 1.29.8
-  - 1.30.4
-  - 1.31.0
+  - 1.28.15
+  - 1.29.10
+  - 1.30.6
+  - 1.31.2

pyproject.toml

@@ -1,22 +1,37 @@
-[tool.black]
+# https://www.python.org/dev/peps/pep-0518/#file-format
+
+# https://docs.astral.sh/ruff/settings/
+[tool.ruff]
 line-length = 132
+target-version = "py311"
 
 [tool.ruff.lint]
 fixable = ["ALL"]
 
 # https://docs.astral.sh/ruff/rules/
 select = [
   "ASYNC",
+  "ARG",
+  "BLE",
   "C",
   "E",
   "F",
+  # "FBT",
+  "FLY",
   "FURB",
+  "I",
+  # "N",
   "PERF",
+  "PGH",
+  # "PL",
   "Q",
+  "R",
   "RUF",
   "S",
+  "TID",
   "UP",
   "W",
+  "YTT",
 ]
 ignore = [
   "E501",