From 9ce91eb1e8f6392ed1031b2b72e22257be54e800 Mon Sep 17 00:00:00 2001 From: Andy Pfister Date: Mon, 22 Apr 2024 19:33:03 +0200 Subject: [PATCH] Build, test and deploy from GitHub Actions --- .github/workflows/ci.yml | 122 +++++++++++++++++++++++ Dockerfile | 204 +++++++++++++++++++++++---------------- README.md | 2 +- test.sh | 85 +++++----------- 4 files changed, 268 insertions(+), 145 deletions(-) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..726245d --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,122 @@ +name: Build and push + +on: + push: + pull_request: + +jobs: + base-images: + # we cannot offer this caching optimizations to forks as it would require access to our Dockerhub + if: github.repository == 'pgautoupgrade/docker-pgautoupgrade' + runs-on: ubuntu-latest + strategy: + matrix: + pg_version: + - "9.5" + - "9.6" + - "10" + - "11" + - "12" + - "13" + - "14" + - "15" + - "16" + + steps: + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Build and push image + uses: docker/build-push-action@v5 + with: + push: true + platforms: linux/amd64,linux/arm64 + build-args: | + "PGTARGET=16" + target: "build-${{ matrix.pg_version }}" + tags: "pgautoupgrade/pgautoupgrade:build-${{ matrix.pg_version }}" + cache-to: type=inline + cache-from: type=registry,ref=pgautoupgrade/pgautoupgrade:build-${{ matrix.pg_version }} + + target-images: + runs-on: ubuntu-latest + needs: base-images + # otherwise, on forks it would skip the build entirely (because the base step does not run) + if: always() + env: + # but they can still use our public caches, but they might be outdated and it still triggers a full rebuild + TARGET_TAG: ${{ github.ref == 'refs/heads/main' && 'alpine3.19' || 'dev-alpine3.19' }} + CACHE_FROM: | + type=registry,ref=pgautoupgrade/pgautoupgrade:build-9.5 + type=registry,ref=pgautoupgrade/pgautoupgrade:build-9.6 + type=registry,ref=pgautoupgrade/pgautoupgrade:build-10 + type=registry,ref=pgautoupgrade/pgautoupgrade:build-11 + type=registry,ref=pgautoupgrade/pgautoupgrade:build-12 + type=registry,ref=pgautoupgrade/pgautoupgrade:build-13 + type=registry,ref=pgautoupgrade/pgautoupgrade:build-14 + type=registry,ref=pgautoupgrade/pgautoupgrade:build-15 + type=registry,ref=pgautoupgrade/pgautoupgrade:build-16 + type=registry,ref=pgautoupgrade/pgautoupgrade:${{ matrix.pg_target }}-alpine3.19 + # we cannot access TARGET_TAG from env + # https://docs.github.com/en/actions/learn-github-actions/contexts#context-availability + + strategy: + matrix: + pg_target: + - "12" + - "13" + - "14" + - "15" + - "16" + + steps: + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + if: github.repository == 'pgautoupgrade/docker-pgautoupgrade' + uses: docker/login-action@v3 + with: + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Build image + uses: docker/build-push-action@v5 + with: + load: true + tags: "pgautoupgrade/pgautoupgrade:${{ matrix.pg_target }}-alpine3.19" + build-args: | + "PGTARGET=${{ matrix.pg_target }}" + cache-to: type=inline + cache-from: "${{ env.CACHE_FROM }}" + + - name: Checkout code + uses: actions/checkout@v4 + + - name: Test image + run: | + make test + env: + PGTARGET: ${{ matrix.pg_target }} + + - name: Push image + if: github.repository == 'pgautoupgrade/docker-pgautoupgrade' && github.ref == 'refs/heads/main' + uses: docker/build-push-action@v5 + with: + platforms: linux/amd64,linux/arm64 + tags: "pgautoupgrade/pgautoupgrade:${{ matrix.pg_target }}-${{ env.TARGET_TAG }}" + push: true + cache-to: type=inline + cache-from: "${{ env.CACHE_FROM }}" diff --git a/Dockerfile b/Dockerfile index 9e55d40..2a768df 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,7 @@ -# The version of PostgreSQL this container migrates data to ARG PGTARGET=16 -# We use Alpine as a base image to compile older -# PostgreSQL versions in, then copy the binaries -# into the official PG Alpine image -FROM alpine:3.19 AS build - -# We need to define this here, to make the above PGTARGET available after the FROM -ARG PGTARGET +### Things we need in all build containers +FROM alpine:3.19 as base-build # Where we'll do all our compiling and similar ENV BUILD_ROOT /buildroot @@ -16,76 +10,118 @@ ENV BUILD_ROOT /buildroot RUN mkdir ${BUILD_ROOT} WORKDIR ${BUILD_ROOT} -# Download the source code for previous PG releases -RUN wget https://ftp.postgresql.org/pub/source/v9.5.25/postgresql-9.5.25.tar.bz2 && \ - wget https://ftp.postgresql.org/pub/source/v9.6.24/postgresql-9.6.24.tar.bz2 && \ - wget https://ftp.postgresql.org/pub/source/v10.23/postgresql-10.23.tar.bz2 && \ - wget https://ftp.postgresql.org/pub/source/v11.22/postgresql-11.22.tar.bz2 -RUN if [ "${PGTARGET}" -gt 12 ]; then wget https://ftp.postgresql.org/pub/source/v12.18/postgresql-12.18.tar.bz2; fi -RUN if [ "${PGTARGET}" -gt 13 ]; then wget https://ftp.postgresql.org/pub/source/v13.14/postgresql-13.14.tar.bz2; fi -RUN if [ "${PGTARGET}" -gt 14 ]; then wget https://ftp.postgresql.org/pub/source/v14.11/postgresql-14.11.tar.bz2; fi -RUN if [ "${PGTARGET}" -gt 15 ]; then wget https://ftp.postgresql.org/pub/source/v15.6/postgresql-15.6.tar.bz2; fi - -# Extract the source code -RUN tar -xf postgresql-9.5*.tar.bz2 && \ - tar -xf postgresql-9.6*.tar.bz2 && \ - tar -xf postgresql-10*.tar.bz2 && \ - tar -xf postgresql-11*.tar.bz2 -RUN if [ "${PGTARGET}" -gt 12 ]; then tar -xf postgresql-12*.tar.bz2; fi -RUN if [ "${PGTARGET}" -gt 13 ]; then tar -xf postgresql-13*.tar.bz2; fi -RUN if [ "${PGTARGET}" -gt 14 ]; then tar -xf postgresql-14*.tar.bz2; fi -RUN if [ "${PGTARGET}" -gt 15 ]; then tar -xf postgresql-15*.tar.bz2; fi - # Install things needed for development # We might want to install "alpine-sdk" instead of "build-base", if build-base # doesn't have everything we need RUN apk update && \ - apk upgrade && \ - apk add --update build-base icu-data-full icu-dev linux-headers lz4-dev musl musl-locales musl-utils tzdata zlib-dev zstd-dev && \ - apk cache clean + apk upgrade && \ + apk add --update build-base icu-data-full icu-dev linux-headers lz4-dev musl musl-locales musl-utils tzdata zlib-dev zstd-dev && \ + apk cache clean + +### PostgreSQL 9.5 +FROM base-build as build-9.5 + +RUN wget https://ftp.postgresql.org/pub/source/v9.5.25/postgresql-9.5.25.tar.bz2 && \ + tar -xf postgresql-9.5*.tar.bz2 -# Compile PG releases with fairly minimal options -# Note that given some time, we could likely remove the pieces of the older PG installs which aren't needed by pg_upgrade RUN cd postgresql-9.5.* && \ - ./configure --prefix=/usr/local-pg9.5 --with-openssl=no --without-readline --with-system-tzdata=/usr/share/zoneinfo --enable-debug=no CFLAGS="-Os" && \ - make -j $(nproc) && \ - make install-world && \ - rm -rf /usr/local-pg9.5/include + ./configure --prefix=/usr/local-pg9.5 --with-openssl=no --without-readline --with-system-tzdata=/usr/share/zoneinfo --enable-debug=no CFLAGS="-Os" && \ + make -j $(nproc) && \ + make install && \ + rm -rf /usr/local-pg9.5/include + +### PostgreSQL 9.6 +FROM base-build as build-9.6 + +RUN wget https://ftp.postgresql.org/pub/source/v9.6.24/postgresql-9.6.24.tar.bz2 && \ + tar -xf postgresql-9.6*.tar.bz2 + RUN cd postgresql-9.6.* && \ - ./configure --prefix=/usr/local-pg9.6 --with-openssl=no --without-readline --with-system-tzdata=/usr/share/zoneinfo --enable-debug=no CFLAGS="-Os" && \ - make -j $(nproc) && \ - make install-world && \ - rm -rf /usr/local-pg9.6/include + ./configure --prefix=/usr/local-pg9.6 --with-openssl=no --without-readline --with-system-tzdata=/usr/share/zoneinfo --enable-debug=no CFLAGS="-Os" && \ + make -j $(nproc) && \ + make install && \ + rm -rf /usr/local-pg9.6/include + +### PostgreSQL 10 +FROM base-build as build-10 +RUN wget https://ftp.postgresql.org/pub/source/v10.23/postgresql-10.23.tar.bz2 && \ + tar -xf postgresql-10*.tar.bz2 + RUN cd postgresql-10.* && \ - ./configure --prefix=/usr/local-pg10 --with-openssl=no --without-readline --with-system-tzdata=/usr/share/zoneinfo --with-icu --enable-debug=no CFLAGS="-Os" && \ - make -j $(nproc) && \ - make install-world && \ - rm -rf /usr/local-pg10/include + ./configure --prefix=/usr/local-pg10 --with-openssl=no --without-readline --with-icu --with-system-tzdata=/usr/share/zoneinfo --enable-debug=no CFLAGS="-Os" && \ + make -j $(nproc) && \ + make install && \ + rm -rf /usr/local-pg10/include + +### PostgreSQL 11 +FROM base-build as build-11 +RUN wget https://ftp.postgresql.org/pub/source/v11.22/postgresql-11.22.tar.bz2 && \ + tar -xf postgresql-11*.tar.bz2 + RUN cd postgresql-11.* && \ - ./configure --prefix=/usr/local-pg11 --with-openssl=no --without-readline --with-system-tzdata=/usr/share/zoneinfo --with-icu --enable-debug=no CFLAGS="-Os" && \ - make -j $(nproc) && \ - make install-world && \ - rm -rf /usr/local-pg11/include -RUN if [ "${PGTARGET}" -gt 12 ]; then cd postgresql-12.* && \ - ./configure --prefix=/usr/local-pg12 --with-openssl=no --without-readline --with-system-tzdata=/usr/share/zoneinfo --with-icu --enable-debug=no CFLAGS="-Os" && \ - make -j $(nproc) && \ - make install-world && \ - rm -rf /usr/local-pg12/include; else mkdir /usr/local-pg12; fi -RUN if [ "${PGTARGET}" -gt 13 ]; then cd postgresql-13.* && \ - ./configure --prefix=/usr/local-pg13 --with-openssl=no --without-readline --with-system-tzdata=/usr/share/zoneinfo --with-icu --enable-debug=no CFLAGS="-Os" && \ - make -j $(nproc) && \ - make install-world && \ - rm -rf /usr/local-pg13/include; else mkdir /usr/local-pg13; fi -RUN if [ "${PGTARGET}" -gt 14 ]; then cd postgresql-14.* && \ - ./configure --prefix=/usr/local-pg14 --with-openssl=no --without-readline --with-system-tzdata=/usr/share/zoneinfo --with-icu --with-lz4 --enable-debug=no CFLAGS="-Os" && \ - make -j $(nproc) && \ - make install-world && \ - rm -rf /usr/local-pg14/include; else mkdir /usr/local-pg14; fi -RUN if [ "${PGTARGET}" -gt 15 ]; then cd postgresql-15.* && \ - ./configure --prefix=/usr/local-pg15 --with-openssl=no --without-readline --with-system-tzdata=/usr/share/zoneinfo --with-icu --with-lz4 --with-zstd --enable-debug=no CFLAGS="-Os" && \ - make -j $(nproc) && \ - make install-world && \ - rm -rf /usr/local-pg15/include; else mkdir /usr/local-pg15; fi + ./configure --prefix=/usr/local-pg11 --with-openssl=no --without-readline --with-icu --with-system-tzdata=/usr/share/zoneinfo --enable-debug=no CFLAGS="-Os" && \ + make -j $(nproc) && \ + make install && \ + rm -rf /usr/local-pg11/include + +### PostgreSQL 12 +FROM base-build as build-12 +RUN wget https://ftp.postgresql.org/pub/source/v12.17/postgresql-12.17.tar.bz2 && \ + tar -xf postgresql-12*.tar.bz2 + +RUN cd postgresql-12.* && \ + ./configure --prefix=/usr/local-pg12 --with-openssl=no --without-readline --with-icu --with-system-tzdata=/usr/share/zoneinfo --enable-debug=no CFLAGS="-Os" && \ + make -j $(nproc) && \ + make install && \ + rm -rf /usr/local-pg12/include + +### PostgreSQL 13 +FROM base-build as build-13 + +RUN wget https://ftp.postgresql.org/pub/source/v13.13/postgresql-13.13.tar.bz2 && \ + tar -xf postgresql-13*.tar.bz2 + +RUN cd postgresql-13.* && \ + ./configure --prefix=/usr/local-pg13 --with-openssl=no --without-readline --with-icu --with-system-tzdata=/usr/share/zoneinfo --enable-debug=no CFLAGS="-Os" && \ + make -j $(nproc) && \ + make install && \ + rm -rf /usr/local-pg13/include + +### PostgreSQL 14 +FROM base-build as build-14 + +RUN wget https://ftp.postgresql.org/pub/source/v14.10/postgresql-14.10.tar.bz2 && \ + tar -xf postgresql-14*.tar.bz2 + +RUN cd postgresql-14.* && \ + ./configure --prefix=/usr/local-pg14 --with-openssl=no --without-readline --with-icu --with-lz4 --with-system-tzdata=/usr/share/zoneinfo --enable-debug=no CFLAGS="-Os" && \ + make -j $(nproc) && \ + make install && \ + rm -rf /usr/local-pg14/include + +### PostgreSQL 15 +FROM base-build as build-15 + +RUN wget https://ftp.postgresql.org/pub/source/v15.5/postgresql-15.5.tar.bz2 && \ + tar -xf postgresql-15*.tar.bz2 + +RUN cd postgresql-15.* && \ + ./configure --prefix=/usr/local-pg15 --with-openssl=no --without-readline --with-icu --with-lz4 --with-system-tzdata=/usr/share/zoneinfo --enable-debug=no CFLAGS="-Os" && \ + make -j $(nproc) && \ + make install && \ + rm -rf /usr/local-pg15/include + +### PostgreSQL 16 +FROM base-build as build-16 + +RUN wget https://ftp.postgresql.org/pub/source/v16.2/postgresql-16.2.tar.gz && \ + tar -xf postgresql-16*.tar.gz + +RUN cd postgresql-16.* && \ + ./configure --prefix=/usr/local-pg16 --with-openssl=no --without-readline --with-icu --with-lz4 --with-system-tzdata=/usr/share/zoneinfo --enable-debug=no CFLAGS="-Os" && \ + make -j $(nproc) && \ + make install && \ + rm -rf /usr/local-pg16/include # Use the PostgreSQL Alpine image as our output image base FROM postgres:${PGTARGET}-alpine3.19 @@ -94,25 +130,27 @@ FROM postgres:${PGTARGET}-alpine3.19 ARG PGTARGET # Copy across our compiled files -COPY --from=build /usr/local-pg9.5 /usr/local-pg9.5 -COPY --from=build /usr/local-pg9.6 /usr/local-pg9.6 -COPY --from=build /usr/local-pg10 /usr/local-pg10 -COPY --from=build /usr/local-pg11 /usr/local-pg11 -COPY --from=build /usr/local-pg12 /usr/local-pg12 -COPY --from=build /usr/local-pg13 /usr/local-pg13 -COPY --from=build /usr/local-pg14 /usr/local-pg14 -COPY --from=build /usr/local-pg15 /usr/local-pg15 +COPY --from=build-9.5 /usr/local-pg9.5 /usr/local-pg9.5 +COPY --from=build-9.6 /usr/local-pg9.6 /usr/local-pg9.6 +COPY --from=build-10 /usr/local-pg10 /usr/local-pg10 +COPY --from=build-11 /usr/local-pg11 /usr/local-pg11 +COPY --from=build-12 /usr/local-pg12 /usr/local-pg12 +COPY --from=build-13 /usr/local-pg13 /usr/local-pg13 +COPY --from=build-14 /usr/local-pg14 /usr/local-pg14 +COPY --from=build-15 /usr/local-pg15 /usr/local-pg15 +COPY --from=build-16 /usr/local-pg16 /usr/local-pg16 # Remove any left over PG directory stubs. Doesn't help with image size, just with clarity on what's in the image. -RUN if [ "${PGTARGET}" -eq 12 ]; then rmdir /usr/local-pg12 /usr/local-pg13 /usr/local-pg14 /usr/local-pg15; fi -RUN if [ "${PGTARGET}" -eq 13 ]; then rmdir /usr/local-pg13 /usr/local-pg14 /usr/local-pg15; fi -RUN if [ "${PGTARGET}" -eq 14 ]; then rmdir /usr/local-pg14 /usr/local-pg15; fi -RUN if [ "${PGTARGET}" -eq 15 ]; then rmdir /usr/local-pg15; fi +RUN if [ "${PGTARGET}" -eq 12 ]; then rm -rf /usr/local-pg12 /usr/local-pg13 /usr/local-pg14 /usr/local-pg15 /usr/local-pg16; fi +RUN if [ "${PGTARGET}" -eq 13 ]; then rm -rf /usr/local-pg13 /usr/local-pg14 /usr/local-pg15 /usr/local-pg16; fi +RUN if [ "${PGTARGET}" -eq 14 ]; then rm -rf /usr/local-pg14 /usr/local-pg15 /usr/local-pg16; fi +RUN if [ "${PGTARGET}" -eq 15 ]; then rm -rf /usr/local-pg15 /usr/local-pg16; fi +RUN if [ "${PGTARGET}" -eq 16 ]; then rm -rf /usr/local-pg16; fi # Install locale RUN apk update && \ - apk add --update icu-data-full musl musl-utils musl-locales tzdata && \ - apk cache clean + apk add --update icu-data-full musl musl-utils musl-locales tzdata && \ + apk cache clean ## FIXME: Only useful while developing this Dockerfile ##RUN apk add man-db man-pages-posix diff --git a/README.md b/README.md index ab92e7d..e9f0b85 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ If you instead want to run a specific version of PostgreSQL then pick a matching tag on our Docker Hub. For example, to use PostgreSQL 15 you can use: - pgautoupgrade/pgautoupgrade:15-alpine3.8 + pgautoupgrade/pgautoupgrade:15-alpine3.19 ### "One shot" mode diff --git a/test.sh b/test.sh index 2035a92..9fc160a 100755 --- a/test.sh +++ b/test.sh @@ -1,10 +1,14 @@ #!/usr/bin/env bash +set -eux FAILURE=0 +# Array of PostgreSQL versions for testing +PG_VERSIONS=(9.5 9.6 10 11 12 13 14 15 16) + # Stop any existing containers from previous test runs test_down() { - docker-compose -f test/docker-compose-pgauto.yml down + docker compose -f test/docker-compose-pgauto.yml down } test_run() { @@ -18,20 +22,10 @@ test_run() { fi # Create the PostgreSQL database using a specific version of PostgreSQL - docker-compose -f "docker-compose-pg${VERSION}.yml" run --rm server create_db + docker compose -f "docker-compose-pg${VERSION}.yml" run --rm server create_db # Start Redash normally, using an "autoupdate" version of PostgreSQL - if [ "${TARGET}" = "16" ]; then - docker-compose -f docker-compose-pgauto.yml up -d - elif [ "${TARGET}" = "15" ]; then - TARGET_TAG=15-dev docker-compose -f docker-compose-pgauto.yml up -d - elif [ "${TARGET}" = "14" ]; then - TARGET_TAG=14-dev docker-compose -f docker-compose-pgauto.yml up -d - elif [ "${TARGET}" = "13" ]; then - TARGET_TAG=13-dev docker-compose -f docker-compose-pgauto.yml up -d - elif [ "${TARGET}" = "12" ]; then - TARGET_TAG=12-dev docker-compose -f docker-compose-pgauto.yml up -d - fi + TARGET_TAG="${TARGET}-alpine3.19" docker compose -f docker-compose-pgauto.yml up --wait -d # Verify the PostgreSQL data files are now the target version PGVER=$(sudo cat postgres-data/PG_VERSION) @@ -51,7 +45,7 @@ test_run() { fi # Shut down containers from previous test runs - docker-compose -f docker-compose-pgauto.yml down + docker compose -f docker-compose-pgauto.yml down } # Shut down containers from previous test runs @@ -59,60 +53,29 @@ test_down # If the user gives a first argument of "down", then we exit # after shutting down any running containers from previous test runs -if [ "$1" = "down" ]; then +arg="${1:-}" +if [ "$arg" = "down" ]; then exit 0 fi # Change into the test directory cd test || exit 1 -# Testing upgrading from each major PG version directly to PG 12 -test_run 9.5 12 -test_run 9.6 12 -test_run 10 12 -test_run 11 12 - -# Testing upgrading from each major PG version directly to PG 13 -test_run 9.5 13 -test_run 9.6 13 -test_run 10 13 -test_run 11 13 -test_run 12 13 - -# Testing upgrading from each major PG version directly to PG 14 -test_run 9.5 14 -test_run 9.6 14 -test_run 10 14 -test_run 11 14 -test_run 12 14 -test_run 13 14 - -# Testing upgrading from each major PG version directly to PG 15 -test_run 9.5 15 -test_run 9.6 15 -test_run 10 15 -test_run 11 15 -test_run 12 15 -test_run 13 15 -test_run 14 15 - -# Testing upgrading from each major PG version directly to PG 16 -test_run 9.5 16 -test_run 9.6 16 -test_run 10 16 -test_run 11 16 -test_run 12 16 -test_run 13 16 -test_run 14 16 -test_run 15 16 +for version in "${PG_VERSIONS[@]}"; do + # Only test if the version is less than the latest version + if [[ $(echo "$version < $PGTARGET" | bc) -eq 1 ]]; then + test_run "$version" "$PGTARGET" + fi +done +# Check for failure if [ "${FAILURE}" -ne 0 ]; then - echo - echo "FAILURE: Automatic upgrade of PostgreSQL failed in one of the tests. Please investigate." - echo - exit 1 + echo + echo "FAILURE: Automatic upgrade of PostgreSQL failed in one of the tests. Please investigate." + echo + exit 1 else - echo - echo "SUCCESS: Automatic upgrade testing of PostgreSQL to PG 12, 13, 14, 15, and 16 passed without issue." - echo + echo + echo "SUCCESS: Automatic upgrade testing of PostgreSQL to all versions up to $PGTARGET passed without issue." + echo fi