From ee55956e1514d2ed0e2da9ca9c547278c86019ad Mon Sep 17 00:00:00 2001 From: Parker Timmerman Date: Sun, 12 Jan 2025 15:51:43 -0500 Subject: [PATCH] start, introduce a 'min' version of the CI builder image * min version is much smaller than the the full image, allowing for faster CI startup * updates ci-builder/Dockerfile with the first layer being the 'min' image * update CI jobs to use 'min' image where possible --- bin/ci-builder | 24 ++- ci/builder/Dockerfile | 164 ++++++++++++------ ci/mkpipeline.sh | 10 +- ci/nightly/pipeline.template.yml | 4 +- .../pipeline.template.yml | 4 +- ci/test/pipeline.template.yml | 5 +- 6 files changed, 137 insertions(+), 74 deletions(-) diff --git a/bin/ci-builder b/bin/ci-builder index 34d20064c5070..1aaf8357f61f8 100755 --- a/bin/ci-builder +++ b/bin/ci-builder @@ -24,7 +24,7 @@ cd "$(dirname "$0")/.." if [[ $# -lt 2 ]] then - echo "usage: $0 [...] + echo "usage: $0 [...] Manages the ci-builder Docker image, which contains the dependencies required to build, test, and deploy the code in this repository. @@ -40,17 +40,25 @@ For details, consult ci/builder/README.md." fi cmd=$1 && shift -channel=$1 && shift +flavor=$1 && shift rust_date= -case "$channel" in - stable) rust_version=$(sed -n 's/rust-version = "\(.*\)"/\1/p' Cargo.toml) ;; +case "$flavor" in + min) + docker_target=ci-builder-min + rust_version=$(sed -n 's/rust-version = "\(.*\)"/\1/p' Cargo.toml) + ;; + stable) + docker_target=ci-builder-full + rust_version=$(sed -n 's/rust-version = "\(.*\)"/\1/p' Cargo.toml) + ;; nightly) + docker_target=ci-builder-full rust_version=nightly rust_date=/$NIGHTLY_RUST_DATE ;; *) - printf "unknown rust channel %q\n" "$channel" + printf "unknown CI builder flavor %q\n" "$flavor" exit 1 ;; esac @@ -58,7 +66,7 @@ esac arch_gcc=${MZ_DEV_CI_BUILDER_ARCH:-$(arch_gcc)} arch_go=$(arch_go "$arch_gcc") -cid_file=ci/builder/.${channel%%-*}.cidfile +cid_file=ci/builder/.${flavor%%-*}.cidfile rust_components=rustc,cargo,rust-std-$arch_gcc-unknown-linux-gnu,llvm-tools-preview if [[ $rust_version = nightly ]]; then @@ -86,6 +94,7 @@ build() { --build-arg "BAZEL_VERSION=$bazel_version" \ --tag materialize/ci-builder:"$tag" \ --tag materialize/ci-builder:"$cache_tag" \ + --target $docker_target \ "$@" ci/builder } @@ -111,6 +120,7 @@ files+=" rust-version:$rust_version rust-date:$rust_date arch:$arch_gcc +flavor:$flavor " tag=$(echo "$files" | python3 -c ' import base64 @@ -121,7 +131,7 @@ input = sys.stdin.buffer.read() hash = base64.b32encode(hashlib.sha1(input).digest()) print(hash.decode()) ') -cache_tag=cache-$rust_version-$arch_go +cache_tag=cache-$flavor-$rust_version-$arch_go case "$cmd" in diff --git a/ci/builder/Dockerfile b/ci/builder/Dockerfile index 016a79018757c..671d2389c6d03 100644 --- a/ci/builder/Dockerfile +++ b/ci/builder/Dockerfile @@ -7,10 +7,112 @@ # the Business Source License, use of this software will be governed # by the Apache License, Version 2.0. -# Build a cross-compiling toolchain that targets the oldest version of Linux -# that we support. +# Stage 1: Build a minimum CI Builder image that we can use for the initial +# steps like `mkpipeline` and `Build`, as well as any tests that are self +# contained and use other Docker images. +FROM ubuntu:noble-20241015 AS ci-builder-min +WORKDIR /workdir + +ARG ARCH_GCC +ARG ARCH_GO + +# Environment variables that should be set for the entire build container. + +# Ensure any Rust binaries that crash print a backtrace. +ENV RUST_BACKTRACE=1 +# Ensure that all python output is unbuffered, otherwise it is not +# logged properly in Buildkite. +ENV PYTHONUNBUFFERED=1 +# Set a environment variable that tools can check to see if they're in the +# builder or not. +ENV MZ_DEV_CI_BUILDER=1 + +# Faster uncompression +ARG XZ_OPT=-T0 + +# Absolute minimum set of dependencies needed for a CI job. +# +# Please take care with what gets added here. The goal of this initial layer is to be as small as +# possible since it's used for the `mkpipeline` and `Build` CI jobs, which block __all other__ +# jobs. +RUN apt-get update --fix-missing && TZ=UTC DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + docker.io \ + gdb \ + git \ + gnupg2 \ + libxml2 \ + python3 \ + && rm -rf /var/lib/apt/lists/* + +# Install Python dependencies. These are necessary to run some of our base tooling. +COPY requirements.txt /workdir/ +RUN curl -LsSf https://astral.sh/uv/0.4.25/install.sh | UV_INSTALL_DIR=/usr/local UV_UNMANAGED_INSTALL=1 sh \ + && uv pip install --system --break-system-packages -r /workdir/requirements.txt && rm /workdir/requirements*.txt + +# Install extra tools not available in apt repositories. + +COPY rust.asc . +RUN gpg --import rust.asc \ + && rm rust.asc \ + && echo "trusted-key 85AB96E6FA1BE5FE" >> ~/.gnupg/gpg.conf + +ARG BAZEL_VERSION +ARG RUST_DATE +ARG RUST_VERSION + +RUN \ + # 1. autouseradd + # + # Ensures that the UID used when running the container has a proper entry in + # `/etc/passwd`, and writable home directory. + curl -fsSL https://github.com/benesch/autouseradd/releases/download/1.3.0/autouseradd-1.3.0-$ARCH_GO.tar.gz \ + | tar xz -C / --strip-components 1 \ + # 2. Bazel + # + # We primarily build Materialize via Bazel in CI, and Bazel pulls in its own dependencies. + && arch_bazel=$(echo "$ARCH_GCC" | sed -e "s/aarch64/arm64/" -e "s/amd64/x86_64/") bazel_version=$(echo "$BAZEL_VERSION") \ + && curl -fsSL -o /usr/local/bin/bazel https://github.com/bazelbuild/bazel/releases/download/$bazel_version/bazel-$bazel_version-linux-$arch_bazel \ + && if [ "$arch_bazel" = arm64 ]; then echo 'fac4b954e0501c2be8b9653a550b443eb85284e568d08b102977e2bf587b09d7 /usr/local/bin/bazel' | sha256sum --check; fi \ + && if [ "$arch_bazel" = x86_64 ]; then echo '48ea0ff9d397a48add6369c261c5a4431fe6d5d5348cfb81411782fb80c388d3 /usr/local/bin/bazel' | sha256sum --check; fi \ + && chmod +x /usr/local/bin/bazel \ + # 3. Docker + # + # If you upgrade Docker (Compose) version here, also update it in misc/python/cli/mzcompose.py. + && mkdir -p /usr/local/lib/docker/cli-plugins \ + && curl -fsSL https://github.com/docker/compose/releases/download/v2.15.1/docker-compose-linux-$ARCH_GCC > /usr/local/lib/docker/cli-plugins/docker-compose \ + && chmod +x /usr/local/lib/docker/cli-plugins/docker-compose \ + && curl -fsSL https://github.com/christian-korneck/docker-pushrm/releases/download/v1.9.0/docker-pushrm_linux_$ARCH_GO > /usr/local/lib/docker/cli-plugins/docker-pushrm \ + && chmod +x /usr/local/lib/docker/cli-plugins/docker-pushrm \ + # 4. Cargo + # + # Some parts of our stack use 'cargo' to read metadata, so we install just that. Importantly we + # do not install 'rustc' or any of the other tools, this keeps the Docker image small. + && mkdir rust \ + && curl -fsSL https://static.rust-lang.org/dist$RUST_DATE/rust-$RUST_VERSION-$ARCH_GCC-unknown-linux-gnu.tar.gz > rust.tar.gz \ + && curl -fsSL https://static.rust-lang.org/dist$RUST_DATE/rust-$RUST_VERSION-$ARCH_GCC-unknown-linux-gnu.tar.gz.asc > rust.asc \ + && gpg --verify rust.asc rust.tar.gz \ + && tar -xzf rust.tar.gz -C rust --strip-components=1 \ + && rust/install.sh --components=cargo \ + && rm -rf rust.asc rust.tar.gz rust + +# Make the image as small as possible. +RUN find /workdir /root -mindepth 1 -maxdepth 1 -exec rm -rf {} + + +# Remove Ubuntu user causing UID collisions. +# https://bugs.launchpad.net/cloud-images/+bug/2005129 +RUN userdel -r ubuntu + +ENTRYPOINT ["autouseradd", "--user", "materialize"] + +# Stage 2. Build a cross-compiling toolchain that targets the oldest version of +# Linux that we support. +# +# TODO(parkmycar): This shouldn't be necessary anymore with Bazel. FROM ubuntu:noble-20241015 as crosstool + ARG ARCH_GCC ARG ARCH_GO @@ -68,10 +170,10 @@ RUN DEFCONFIG=crosstool-$ARCH_GCC.defconfig ct-ng defconfig \ && rm crosstool-$ARCH_GCC.defconfig \ && ct-ng build -# Import the cross-compiling toolchain into a fresh image, omitting the -# dependencies that we needed to actually build the toolchain. +# Stage 3: Build a full CI Builder image that imports the cross-compiling +# toolchain and can be used for any CI job. +FROM ci-builder-min as ci-builder-full -FROM ubuntu:noble-20241015 ARG ARCH_GCC ARG ARCH_GO @@ -135,9 +237,7 @@ RUN gpg --dearmor < nodesource.asc > /etc/apt/keyrings/nodesource.gpg \ && apt-get update \ && apt-get install -y --no-install-recommends nodejs -RUN curl -fsSL https://github.com/benesch/autouseradd/releases/download/1.3.0/autouseradd-1.3.0-$ARCH_GO.tar.gz \ - | tar xz -C / --strip-components 1 \ - && curl -fsSL https://github.com/koalaman/shellcheck/releases/download/v0.8.0/shellcheck-v0.8.0.linux.$ARCH_GCC.tar.xz > shellcheck.tar.xz \ +RUN curl -fsSL https://github.com/koalaman/shellcheck/releases/download/v0.8.0/shellcheck-v0.8.0.linux.$ARCH_GCC.tar.xz > shellcheck.tar.xz \ && tar -xJf shellcheck.tar.xz -C /usr/local/bin --strip-components 1 shellcheck-v0.8.0/shellcheck \ && rm shellcheck.tar.xz \ && curl -fsSL https://github.com/bufbuild/buf/releases/download/v1.18.0/buf-Linux-$ARCH_GCC.tar.gz > buf.tar.gz \ @@ -148,12 +248,6 @@ RUN curl -fsSL https://github.com/benesch/autouseradd/releases/download/1.3.0/au && tar -xf kail.tar.gz -C /usr/local/bin kail \ && rm kail.tar.gz \ && chmod +x /usr/local/bin/kail \ - && mkdir -p /usr/local/lib/docker/cli-plugins \ - # If you upgrade Docker (Compose) version here, also update it in misc/python/cli/mzcompose.py \ - && curl -fsSL https://github.com/docker/compose/releases/download/v2.15.1/docker-compose-linux-$ARCH_GCC > /usr/local/lib/docker/cli-plugins/docker-compose \ - && chmod +x /usr/local/lib/docker/cli-plugins/docker-compose \ - && curl -fsSL https://github.com/christian-korneck/docker-pushrm/releases/download/v1.9.0/docker-pushrm_linux_$ARCH_GO > /usr/local/lib/docker/cli-plugins/docker-pushrm \ - && chmod +x /usr/local/lib/docker/cli-plugins/docker-pushrm \ && curl -fsSL https://github.com/parca-dev/parca-debuginfo/releases/download/v0.11.0/parca-debuginfo_0.11.0_Linux_$(echo "$ARCH_GCC" | sed "s/aarch64/arm64/").tar.gz \ | tar xz -C /usr/local/bin parca-debuginfo @@ -212,8 +306,6 @@ RUN mkdir rust \ RUN ln -s /usr/bin/lld /opt/x-tools/$ARCH_GCC-unknown-linux-gnu/bin/$ARCH_GCC-unknown-linux-gnu-ld.lld \ && ln -s /usr/bin/lld /opt/x-tools/$ARCH_GCC-unknown-linux-gnu/bin/$ARCH_GCC-unknown-linux-gnu-lld -RUN curl -LsSf https://astral.sh/uv/0.4.25/install.sh | UV_INSTALL_DIR=/usr/local UV_UNMANAGED_INSTALL=1 sh - # Shims for sanitizers COPY sanshim/$ARCH_GCC /sanshim @@ -223,13 +315,6 @@ COPY sanshim/$ARCH_GCC /sanshim COPY pyright-version.sh /workdir/ RUN npx pyright@$(sh /workdir/pyright-version.sh) --help -# Install Python dependencies. These are so quick to install and change -# frequently enough that it makes sense to install them last. - -COPY requirements.txt /workdir/ - -RUN uv pip install --system --break-system-packages -r /workdir/requirements.txt && rm /workdir/requirements*.txt - # Install APT repo generator. RUN curl -fsSL https://github.com/deb-s3/deb-s3/releases/download/0.11.3/deb-s3-0.11.3.gem > deb-s3.gem \ @@ -258,21 +343,7 @@ RUN if [ $ARCH_GCC = x86_64 ]; then \ && rm hugo.tar.gz; \ fi -# Install Bazel. -# -# TODO(parkmycar): Run Bazel in a Docker image that does not have access to clang/gcc or any other tools. - -ARG BAZEL_VERSION - -# Download the bazel binary from the official GitHub releases since the apt repositories do not -# contain arm64 releases. -RUN arch_bazel=$(echo "$ARCH_GCC" | sed -e "s/aarch64/arm64/" -e "s/amd64/x86_64/") bazel_version=$(echo "$BAZEL_VERSION") \ - && curl -fsSL -o /usr/local/bin/bazel https://github.com/bazelbuild/bazel/releases/download/$bazel_version/bazel-$bazel_version-linux-$arch_bazel \ - && if [ "$arch_bazel" = arm64 ]; then echo 'fac4b954e0501c2be8b9653a550b443eb85284e568d08b102977e2bf587b09d7 /usr/local/bin/bazel' | sha256sum --check; fi \ - && if [ "$arch_bazel" = x86_64 ]; then echo '48ea0ff9d397a48add6369c261c5a4431fe6d5d5348cfb81411782fb80c388d3 /usr/local/bin/bazel' | sha256sum --check; fi \ - && chmod +x /usr/local/bin/bazel - -# Install KinD, kubectl, helm, helm-docs & terraform +# Install KinD, kubectl, helm & helm-docs RUN curl -fsSL https://kind.sigs.k8s.io/dl/v0.14.0/kind-linux-$ARCH_GO > /usr/local/bin/kind \ && chmod +x /usr/local/bin/kind \ @@ -340,28 +411,11 @@ ENV CARGO_TARGET_DIR=/mnt/build ENV CARGO_INCREMENTAL=0 ENV HELM_PLUGINS=/usr/local/share/helm/plugins -# Set a environment variable that tools can check to see if they're in the -# builder or not. - -ENV MZ_DEV_CI_BUILDER=1 - # Set up for a persistent volume to hold Cargo metadata, so that crate metadata # does not need to be refetched on every compile. - ENV CARGO_HOME=/cargo RUN mkdir /cargo && chmod 777 /cargo VOLUME /cargo -# Ensure any Rust binaries that crash print a backtrace. -ENV RUST_BACKTRACE=1 - # Make the image as small as possible. RUN find /workdir /root -mindepth 1 -maxdepth 1 -exec rm -rf {} + - -# remove Ubuntu user causing UID collisions -# https://bugs.launchpad.net/cloud-images/+bug/2005129 -RUN userdel -r ubuntu - -# Ensure that all python output is unbuffered, otherwise it is not -# logged properly in Buildkite -ENV PYTHONUNBUFFERED=1 diff --git a/ci/mkpipeline.sh b/ci/mkpipeline.sh index 388cbc65843db..81e16ade79965 100755 --- a/ci/mkpipeline.sh +++ b/ci/mkpipeline.sh @@ -23,15 +23,15 @@ pipeline=${1:-test} bootstrap_steps= for arch in x86_64 aarch64; do - for toolchain in stable nightly; do - if ! MZ_DEV_CI_BUILDER_ARCH=$arch bin/ci-builder exists $toolchain; then + for flavor in stable nightly min; do + if ! MZ_DEV_CI_BUILDER_ARCH=$arch bin/ci-builder exists $flavor; then queue=builder-linux-x86_64 if [[ $arch = aarch64 ]]; then queue=builder-linux-aarch64-mem fi bootstrap_steps+=" - - label: bootstrap $toolchain $arch - command: bin/ci-builder push $toolchain + - label: bootstrap $flavor $arch + command: bin/ci-builder push $flavor agents: queue: $queue " @@ -47,7 +47,7 @@ steps: env: CI_BAZEL_BUILD: 1 CI_BAZEL_REMOTE_CACHE: "https://bazel-remote.dev.materialize.com" - command: bin/ci-builder run stable bin/pyactivate -m ci.mkpipeline $pipeline $@ + command: bin/ci-builder run min bin/pyactivate -m ci.mkpipeline $pipeline $@ priority: 200 agents: queue: hetzner-aarch64-4cpu-8gb diff --git a/ci/nightly/pipeline.template.yml b/ci/nightly/pipeline.template.yml index 6066ab1f3099c..ec1ae3df88653 100644 --- a/ci/nightly/pipeline.template.yml +++ b/ci/nightly/pipeline.template.yml @@ -20,7 +20,7 @@ steps: steps: - id: build-x86_64 label: ":bazel: Build x86_64" - command: bin/ci-builder run stable bin/pyactivate -m ci.test.build + command: bin/ci-builder run min bin/pyactivate -m ci.test.build inputs: - "*" artifact_paths: bazel-explain.log @@ -31,7 +31,7 @@ steps: - id: build-aarch64 label: ":bazel: Build aarch64" - command: bin/ci-builder run stable bin/pyactivate -m ci.test.build + command: bin/ci-builder run min bin/pyactivate -m ci.test.build inputs: - "*" artifact_paths: bazel-explain.log diff --git a/ci/release-qualification/pipeline.template.yml b/ci/release-qualification/pipeline.template.yml index 1c137e9a6d4eb..1b0cdba9ff507 100644 --- a/ci/release-qualification/pipeline.template.yml +++ b/ci/release-qualification/pipeline.template.yml @@ -20,7 +20,7 @@ steps: steps: - id: build-aarch64 label: ":bazel: Build aarch64" - command: bin/ci-builder run stable bin/pyactivate -m ci.test.build + command: bin/ci-builder run min bin/pyactivate -m ci.test.build inputs: - "*" artifact_paths: bazel-explain.log @@ -39,7 +39,7 @@ steps: label: ":bazel: Build x86_64" env: CI_BAZEL_BUILD: "1" - command: bin/ci-builder run stable bin/pyactivate -m ci.test.build + command: bin/ci-builder run min bin/pyactivate -m ci.test.build inputs: - "*" artifact_paths: bazel-explain.log diff --git a/ci/test/pipeline.template.yml b/ci/test/pipeline.template.yml index 52b33cdaf5fc4..10b67472bcdb0 100644 --- a/ci/test/pipeline.template.yml +++ b/ci/test/pipeline.template.yml @@ -15,7 +15,6 @@ dag: true env: - CI_BUILDER_SCCACHE: 1 CI_BAZEL_BUILD: 1 CI_BAZEL_REMOTE_CACHE: $BAZEL_REMOTE_CACHE # Note: In .cargo/config we set the default build jobs to -1 so on developer machines we keep @@ -90,7 +89,7 @@ steps: - id: build-x86_64 label: ":bazel: Build x86_64" env: - command: bin/ci-builder run stable bin/pyactivate -m ci.test.build + command: bin/ci-builder run min bin/pyactivate -m ci.test.build inputs: - "*" artifact_paths: bazel-explain.log @@ -102,7 +101,7 @@ steps: - id: build-aarch64 label: ":bazel: Build aarch64" env: - command: bin/ci-builder run stable bin/pyactivate -m ci.test.build + command: bin/ci-builder run min bin/pyactivate -m ci.test.build inputs: - "*" artifact_paths: bazel-explain.log