more aggressive caching w/ gha mode=max

.github/actions/docker-custom-build-and-push/action.yml

@@ -31,7 +31,7 @@ inputs:
     description: "Main tag to use for the Docker image"
     required: true
   flavor:
-    description: 'Image flavor (e.g., slim, full)'
+    description: "Image flavor (e.g., slim, full)"
     required: false
   target:
     description: "Sets the target stage to build"
@@ -61,7 +61,7 @@ runs:
 
     # Code for testing the build when not pushing to Docker Hub.
     - name: Build and Load image for testing (if not publishing)
-      uses: docker/build-push-action@v5
+      uses: docker/build-push-action@v6
       if: ${{ inputs.publish != 'true' }}
       with:
         context: ${{ inputs.context }}
@@ -74,8 +74,12 @@ runs:
         target: ${{ inputs.target }}
         load: true
         push: false
-        cache-from: type=registry,ref=${{ steps.docker_meta.outputs.tags }}
-        cache-to: type=inline
+        cache-from: |
+          type=gha
+          type=registry,ref=${{ steps.docker_meta.outputs.tags }}
+        cache-to: |
+          type=gha
+          type=inline
     - name: Single Tag
       if: ${{ inputs.publish != 'true' }}
       shell: bash
@@ -108,7 +112,7 @@ runs:
         username: ${{ inputs.username }}
         password: ${{ inputs.password }}
     - name: Build and Push Multi-Platform image
-      uses: docker/build-push-action@v5
+      uses: docker/build-push-action@v6
       if: ${{ inputs.publish == 'true' }}
       with:
         context: ${{ inputs.context }}
@@ -118,7 +122,11 @@ runs:
         tags: ${{ steps.docker_meta.outputs.tags }}
         target: ${{ inputs.target }}
         push: true
-        cache-from: type=registry,ref=${{ steps.docker_meta.outputs.tags }}
-        cache-to: type=inline
+        cache-from: |
+          type=gha
+          type=registry,ref=${{ steps.docker_meta.outputs.tags }}
+        cache-to: |
+          type=gha
+          type=inline
 
     # TODO add code for vuln scanning?

docker/datahub-ingestion-base/Dockerfile

@@ -54,6 +54,13 @@ ENV PATH="${VIRTUAL_ENV}/bin:$PATH"
 RUN python3 -m venv $VIRTUAL_ENV && \
     uv pip install --no-cache --upgrade pip setuptools wheel
 
+# Note: Normally uv will create hardlinks from the cache directory to the venv.
+# In our docker files, we normally use `RUN --mount=type=cache,... uv pip install ...`,
+# which means the cache directory is on a separate filesystem. uv will emit a warning:
+#   Failed to hardlink files; falling back to full copy. This may lead to degraded performance.
+#   If the cache and target directories are on different filesystems, hardlinking may not be supported.
+#   If this is intentional, set `export UV_LINK_MODE=copy` or use `--link-mode=copy` to suppress this warning.
+
 ENTRYPOINT [ "/entrypoint.sh" ]
 
 FROM ${BASE_IMAGE} AS full-install

docker/datahub-ingestion/Dockerfile

@@ -31,7 +31,7 @@ RUN sed -i.bak "s/__version__ = \"1\!0.0.0.dev0\"/__version__ = \"$(echo $RELEAS
 FROM base AS slim-install
 
 RUN --mount=type=cache,target=/datahub-ingestion/.cache/uv,uid=1000,gid=1000 \
-    uv pip install -e ".[base,datahub-rest,datahub-kafka,snowflake,bigquery,redshift,mysql,postgres,hive,clickhouse,glue,dbt,looker,lookml,tableau,powerbi,superset,datahub-business-glossary]"
+    UV_LINK_MODE=copy uv pip install -e ".[base,datahub-rest,datahub-kafka,snowflake,bigquery,redshift,mysql,postgres,hive,clickhouse,glue,dbt,looker,lookml,tableau,powerbi,superset,datahub-business-glossary]"
 
 FROM base AS full-install-build
 
@@ -42,7 +42,7 @@ USER datahub
 COPY ./docker/datahub-ingestion/pyspark_jars.sh .
 
 RUN --mount=type=cache,target=/datahub-ingestion/.cache/uv,uid=1000,gid=1000 \
-    uv pip install -e ".[base,all]" "./airflow-plugin[plugin-v2]" && \
+    UV_LINK_MODE=copy uv pip install -e ".[base,all]" "./airflow-plugin[plugin-v2]" && \
     ./pyspark_jars.sh && \
     datahub --version
 

docker/datahub-ingestion/Dockerfile-slim-only

@@ -21,7 +21,7 @@ RUN sed -i.bak "s/__version__ = \"1\!0.0.0.dev0\"/__version__ = \"$(echo $RELEAS
 FROM base as slim-install
 
 RUN --mount=type=cache,target=/datahub-ingestion/.cache/uv,uid=1000,gid=1000 \
-    uv pip install -e ".[base,datahub-rest,datahub-kafka,snowflake,bigquery,redshift,mysql,postgres,hive,clickhouse,glue,dbt,looker,lookml,tableau,powerbi,superset,datahub-business-glossary]" && \
+    UV_LINK_MODE=copy uv pip install -e ".[base,datahub-rest,datahub-kafka,snowflake,bigquery,redshift,mysql,postgres,hive,clickhouse,glue,dbt,looker,lookml,tableau,powerbi,superset,datahub-business-glossary]" && \
     datahub --version
 
 FROM slim-install as final

docker/datahub-ingestion/build.gradle

@@ -12,7 +12,7 @@ ext {
     docker_target = project.getProperties().getOrDefault("dockerTarget", "slim")
     docker_version = "${version}${docker_target == 'slim' ? '-slim' : ''}"
 
-    revision = 6 // increment to trigger rebuild
+    revision = 7 // increment to trigger rebuild
 }
 
 dependencies {