diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..f1e33b5 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,119 @@ +name: Build and push + +on: + push: + pull_request: + +jobs: + base-images: + # we cannot offer this caching optimizations to forks as it would require access to our Dockerhub + if: github.repository == 'pgautoupgrade/docker-pgautoupgrade' + runs-on: ubuntu-latest + strategy: + matrix: + pg_version: + - "9.5" + - "9.6" + - "10" + - "11" + - "12" + - "13" + - "14" + - "15" + + steps: + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Build and push image + uses: docker/build-push-action@v5 + with: + push: true + platforms: linux/amd64,linux/arm64 + build-args: | + "PGTARGET=16" + target: "build-${{ matrix.pg_version }}" + tags: "pgautoupgrade/pgautoupgrade:build-${{ matrix.pg_version }}" + cache-to: type=inline + cache-from: type=registry,ref=pgautoupgrade/pgautoupgrade:build-${{ matrix.pg_version }} + + target-images: + runs-on: ubuntu-latest + needs: base-images + # otherwise, on forks it would skip the build entirely (because the base step does not run) + if: always() + env: + # but they can still use our public caches, but they might be outdated and it still triggers a full rebuild + TARGET_TAG: ${{ github.ref == 'refs/heads/main' && 'alpine3.18' || 'dev-alpine3.18' }} + CACHE_FROM: | + type=registry,ref=pgautoupgrade/pgautoupgrade:build-9.5 + type=registry,ref=pgautoupgrade/pgautoupgrade:build-9.6 + type=registry,ref=pgautoupgrade/pgautoupgrade:build-10 + type=registry,ref=pgautoupgrade/pgautoupgrade:build-11 + type=registry,ref=pgautoupgrade/pgautoupgrade:build-12 + type=registry,ref=pgautoupgrade/pgautoupgrade:build-13 + type=registry,ref=pgautoupgrade/pgautoupgrade:build-14 + type=registry,ref=pgautoupgrade/pgautoupgrade:build-15 + type=registry,ref=pgautoupgrade/pgautoupgrade:${{ matrix.pg_target }}-alpine3.18 + # we cannot access TARGET_TAG from env + # https://docs.github.com/en/actions/learn-github-actions/contexts#context-availability + + strategy: + matrix: + pg_target: + - "13" + - "14" + - "15" + - "16" + + steps: + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to Docker Hub + if: github.repository == 'pgautoupgrade/docker-pgautoupgrade' + uses: docker/login-action@v3 + with: + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_PASSWORD }} + + - name: Build image + uses: docker/build-push-action@v5 + with: + load: true + tags: "pgautoupgrade/pgautoupgrade:${{ matrix.pg_target }}-alpine3.18" + build-args: | + "PGTARGET=${{ matrix.pg_target }}" + cache-to: type=inline + cache-from: "${{ env.CACHE_FROM }}" + + - name: Checkout code + uses: actions/checkout@v4 + + - name: Test image + run: | + make test + env: + PGTARGET: ${{ matrix.pg_target }} + + - name: Push image + if: github.repository == 'pgautoupgrade/docker-pgautoupgrade' && github.ref == 'refs/heads/main' + uses: docker/build-push-action@v5 + with: + platforms: linux/amd64,linux/arm64 + tags: "pgautoupgrade/pgautoupgrade:${{ matrix.pg_target }}-${{ env.TARGET_TAG }}" + push: true + cache-to: type=inline + cache-from: "${{ env.CACHE_FROM }}" diff --git a/Dockerfile b/Dockerfile index 324166b..1e2c903 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,7 @@ -# The version of PostgreSQL this container migrates data to ARG PGTARGET=16 -# We use Alpine as a base image to compile older -# PostgreSQL versions in, then copy the binaries -# into the PG 15 Alpine image -FROM alpine:3.18 AS build - -# We need to define this here, to make the above PGTARGET available after the FROM -ARG PGTARGET +### Things we need in all build containers +FROM alpine:3.18 as base-build # Where we'll do all our compiling and similar ENV BUILD_ROOT /buildroot @@ -16,76 +10,106 @@ ENV BUILD_ROOT /buildroot RUN mkdir ${BUILD_ROOT} WORKDIR ${BUILD_ROOT} -# Download the source code for previous PG releases -RUN wget https://ftp.postgresql.org/pub/source/v9.5.25/postgresql-9.5.25.tar.bz2 && \ - wget https://ftp.postgresql.org/pub/source/v9.6.24/postgresql-9.6.24.tar.bz2 && \ - wget https://ftp.postgresql.org/pub/source/v10.23/postgresql-10.23.tar.bz2 && \ - wget https://ftp.postgresql.org/pub/source/v11.22/postgresql-11.22.tar.bz2 && \ - wget https://ftp.postgresql.org/pub/source/v12.17/postgresql-12.17.tar.bz2 -RUN if [ "${PGTARGET}" -gt 13 ]; then wget https://ftp.postgresql.org/pub/source/v13.13/postgresql-13.13.tar.bz2; fi -RUN if [ "${PGTARGET}" -gt 14 ]; then wget https://ftp.postgresql.org/pub/source/v14.10/postgresql-14.10.tar.bz2; fi -RUN if [ "${PGTARGET}" -gt 15 ]; then wget https://ftp.postgresql.org/pub/source/v15.5/postgresql-15.5.tar.bz2; fi - -# Extract the source code -RUN tar -xf postgresql-9.5*.tar.bz2 && \ - tar -xf postgresql-9.6*.tar.bz2 && \ - tar -xf postgresql-10*.tar.bz2 && \ - tar -xf postgresql-11*.tar.bz2 && \ - tar -xf postgresql-12*.tar.bz2 -RUN if [ "${PGTARGET}" -gt 13 ]; then tar -xf postgresql-13*.tar.bz2; fi -RUN if [ "${PGTARGET}" -gt 14 ]; then tar -xf postgresql-14*.tar.bz2; fi -RUN if [ "${PGTARGET}" -gt 15 ]; then tar -xf postgresql-15*.tar.bz2; fi - # Install things needed for development # We might want to install "alpine-sdk" instead of "build-base", if build-base # doesn't have everything we need RUN apk update && \ - apk upgrade && \ - apk add --update build-base icu-data-full icu-dev linux-headers lz4-dev musl musl-locales musl-utils tzdata zlib-dev && \ - apk cache clean + apk upgrade && \ + apk add --update build-base icu-data-full icu-dev linux-headers lz4-dev musl musl-locales musl-utils tzdata zlib-dev && \ + apk cache clean + +### PostgreSQL 9.5 +FROM base-build as build-9.5 + +RUN wget https://ftp.postgresql.org/pub/source/v9.5.25/postgresql-9.5.25.tar.bz2 && \ + tar -xf postgresql-9.5*.tar.bz2 -# Compile PG releases with fairly minimal options -# Note that given some time, we could likely remove the pieces of the older PG installs which aren't needed by pg_upgrade RUN cd postgresql-9.5.* && \ - ./configure --prefix=/usr/local-pg9.5 --with-openssl=no --without-readline --enable-debug=no CFLAGS="-Os" && \ - make -j12 && \ - make install && \ - rm -rf /usr/local-pg9.5/include + ./configure --prefix=/usr/local-pg9.5 --with-openssl=no --without-readline --enable-debug=no CFLAGS="-Os" && \ + make -j12 && \ + make install && \ + rm -rf /usr/local-pg9.5/include + +### PostgreSQL 9.6 +FROM base-build as build-9.6 + +RUN wget https://ftp.postgresql.org/pub/source/v9.6.24/postgresql-9.6.24.tar.bz2 && \ + tar -xf postgresql-9.6*.tar.bz2 + RUN cd postgresql-9.6.* && \ - ./configure --prefix=/usr/local-pg9.6 --with-openssl=no --without-readline --enable-debug=no CFLAGS="-Os" && \ - make -j12 && \ - make install && \ - rm -rf /usr/local-pg9.6/include + ./configure --prefix=/usr/local-pg9.6 --with-openssl=no --without-readline --enable-debug=no CFLAGS="-Os" && \ + make -j12 && \ + make install && \ + rm -rf /usr/local-pg9.6/include + +### PostgreSQL 10 +FROM base-build as build-10 +RUN wget https://ftp.postgresql.org/pub/source/v10.23/postgresql-10.23.tar.bz2 && \ + tar -xf postgresql-10*.tar.bz2 + RUN cd postgresql-10.* && \ - ./configure --prefix=/usr/local-pg10 --with-openssl=no --without-readline --with-icu --enable-debug=no CFLAGS="-Os" && \ - make -j12 && \ - make install && \ - rm -rf /usr/local-pg10/include + ./configure --prefix=/usr/local-pg10 --with-openssl=no --without-readline --with-icu --enable-debug=no CFLAGS="-Os" && \ + make -j12 && \ + make install && \ + rm -rf /usr/local-pg10/include + +### PostgreSQL 11 +FROM base-build as build-11 +RUN wget https://ftp.postgresql.org/pub/source/v11.22/postgresql-11.22.tar.bz2 && \ + tar -xf postgresql-11*.tar.bz2 + RUN cd postgresql-11.* && \ - ./configure --prefix=/usr/local-pg11 --with-openssl=no --without-readline --with-icu --enable-debug=no CFLAGS="-Os" && \ - make -j12 && \ - make install && \ - rm -rf /usr/local-pg11/include + ./configure --prefix=/usr/local-pg11 --with-openssl=no --without-readline --with-icu --enable-debug=no CFLAGS="-Os" && \ + make -j12 && \ + make install && \ + rm -rf /usr/local-pg11/include + +### PostgreSQL 12 +FROM base-build as build-12 +RUN wget https://ftp.postgresql.org/pub/source/v12.17/postgresql-12.17.tar.bz2 && \ + tar -xf postgresql-12*.tar.bz2 + RUN cd postgresql-12.* && \ - ./configure --prefix=/usr/local-pg12 --with-openssl=no --without-readline --with-icu --enable-debug=no CFLAGS="-Os" && \ - make -j12 && \ - make install && \ - rm -rf /usr/local-pg12/include -RUN if [ "${PGTARGET}" -gt 13 ]; then cd postgresql-13.* && \ - ./configure --prefix=/usr/local-pg13 --with-openssl=no --without-readline --with-icu --enable-debug=no CFLAGS="-Os" && \ - make -j12 && \ - make install && \ - rm -rf /usr/local-pg13/include; else mkdir /usr/local-pg13; fi -RUN if [ "${PGTARGET}" -gt 14 ]; then cd postgresql-14.* && \ - ./configure --prefix=/usr/local-pg14 --with-openssl=no --without-readline --with-icu --with-lz4 --enable-debug=no CFLAGS="-Os" && \ - make -j12 && \ - make install && \ - rm -rf /usr/local-pg14/include; else mkdir /usr/local-pg14; fi -RUN if [ "${PGTARGET}" -gt 15 ]; then cd postgresql-15.* && \ - ./configure --prefix=/usr/local-pg15 --with-openssl=no --without-readline --with-icu --with-lz4 --enable-debug=no CFLAGS="-Os" && \ - make -j12 && \ - make install && \ - rm -rf /usr/local-pg15/include; else mkdir /usr/local-pg15; fi + ./configure --prefix=/usr/local-pg12 --with-openssl=no --without-readline --with-icu --enable-debug=no CFLAGS="-Os" && \ + make -j12 && \ + make install && \ + rm -rf /usr/local-pg12/include + +### PostgreSQL 13 +FROM base-build as build-13 + +RUN wget https://ftp.postgresql.org/pub/source/v13.13/postgresql-13.13.tar.bz2 && \ + tar -xf postgresql-13*.tar.bz2 + +RUN cd postgresql-13.* && \ + ./configure --prefix=/usr/local-pg13 --with-openssl=no --without-readline --with-icu --enable-debug=no CFLAGS="-Os" && \ + make -j12 && \ + make install && \ + rm -rf /usr/local-pg13/include + +### PostgreSQL 14 +FROM base-build as build-14 + +RUN wget https://ftp.postgresql.org/pub/source/v14.10/postgresql-14.10.tar.bz2 && \ + tar -xf postgresql-14*.tar.bz2 + +RUN cd postgresql-14.* && \ + ./configure --prefix=/usr/local-pg14 --with-openssl=no --without-readline --with-icu --with-lz4 --enable-debug=no CFLAGS="-Os" && \ + make -j12 && \ + make install && \ + rm -rf /usr/local-pg14/include + +### PostgreSQL 15 +FROM base-build as build-15 + +RUN wget https://ftp.postgresql.org/pub/source/v15.5/postgresql-15.5.tar.bz2 && \ + tar -xf postgresql-15*.tar.bz2 + +RUN cd postgresql-15.* && \ + ./configure --prefix=/usr/local-pg15 --with-openssl=no --without-readline --with-icu --with-lz4 --enable-debug=no CFLAGS="-Os" && \ + make -j12 && \ + make install && \ + rm -rf /usr/local-pg15/include # Use the PostgreSQL Alpine image as our output image base FROM postgres:${PGTARGET}-alpine3.18 @@ -94,24 +118,24 @@ FROM postgres:${PGTARGET}-alpine3.18 ARG PGTARGET # Copy across our compiled files -COPY --from=build /usr/local-pg9.5 /usr/local-pg9.5 -COPY --from=build /usr/local-pg9.6 /usr/local-pg9.6 -COPY --from=build /usr/local-pg10 /usr/local-pg10 -COPY --from=build /usr/local-pg11 /usr/local-pg11 -COPY --from=build /usr/local-pg12 /usr/local-pg12 -COPY --from=build /usr/local-pg13 /usr/local-pg13 -COPY --from=build /usr/local-pg14 /usr/local-pg14 -COPY --from=build /usr/local-pg15 /usr/local-pg15 +COPY --from=build-9.5 /usr/local-pg9.5 /usr/local-pg9.5 +COPY --from=build-9.6 /usr/local-pg9.6 /usr/local-pg9.6 +COPY --from=build-10 /usr/local-pg10 /usr/local-pg10 +COPY --from=build-11 /usr/local-pg11 /usr/local-pg11 +COPY --from=build-12 /usr/local-pg12 /usr/local-pg12 +COPY --from=build-13 /usr/local-pg13 /usr/local-pg13 +COPY --from=build-14 /usr/local-pg14 /usr/local-pg14 +COPY --from=build-15 /usr/local-pg15 /usr/local-pg15 # Remove any left over PG directory stubs. Doesn't help with image size, just with clarity on what's in the image. -RUN if [ "${PGTARGET}" -eq 13 ]; then rmdir /usr/local-pg13 /usr/local-pg14 /usr/local-pg15; fi -RUN if [ "${PGTARGET}" -eq 14 ]; then rmdir /usr/local-pg14 /usr/local-pg15; fi -RUN if [ "${PGTARGET}" -eq 15 ]; then rmdir /usr/local-pg15; fi +RUN if [ "${PGTARGET}" -eq 13 ]; then rm -rf /usr/local-pg13 /usr/local-pg14 /usr/local-pg15; fi +RUN if [ "${PGTARGET}" -eq 14 ]; then rm -rf /usr/local-pg14 /usr/local-pg15; fi +RUN if [ "${PGTARGET}" -eq 15 ]; then rm -rf /usr/local-pg15; fi # Install locale RUN apk update && \ - apk add --update icu-data-full musl musl-utils musl-locales tzdata && \ - apk cache clean + apk add --update icu-data-full musl musl-utils musl-locales tzdata && \ + apk cache clean ## FIXME: Only useful while developing this Dockerfile ##RUN apk add man-db man-pages-posix diff --git a/Makefile b/Makefile index 731f35d..3611ef1 100644 --- a/Makefile +++ b/Makefile @@ -17,8 +17,8 @@ dev: 16dev docker build -t pgautoupgrade/pgautoupgrade:16-dev -t pgautoupgrade/pgautoupgrade:dev . prod: - docker build --build-arg PGTARGET=15 -t pgautoupgrade/pgautoupgrade:15-alpine3.8 . && \ - docker build -t pgautoupgrade/pgautoupgrade:16-alpine3.8 -t pgautoupgrade/pgautoupgrade:latest . + docker build --build-arg PGTARGET=15 -t pgautoupgrade/pgautoupgrade:15-alpine3.18 . && \ + docker build -t pgautoupgrade/pgautoupgrade:16-alpine3.18 -t pgautoupgrade/pgautoupgrade:latest . attach: docker exec -it pgauto /bin/bash @@ -29,7 +29,7 @@ before: clean: docker image rm --force pgautoupgrade/pgautoupgrade:dev pgautoupgrade/pgautoupgrade:13-dev && \ pgautoupgrade/pgautoupgrade:14-dev pgautoupgrade/pgautoupgrade:15-dev pgautoupgrade/pgautoupgrade:16-dev && \ - pgautoupgrade/pgautoupgrade:15-alpine3.8 pgautoupgrade/pgautoupgrade:16-alpine3.8 pgautoupgrade/pgautoupgrade:latest && \ + pgautoupgrade/pgautoupgrade:15-alpine3.18 pgautoupgrade/pgautoupgrade:16-alpine3.18 pgautoupgrade/pgautoupgrade:latest && \ docker image prune -f && \ docker volume prune -f @@ -53,5 +53,5 @@ pushdev: docker push pgautoupgrade/pgautoupgrade:dev pushprod: - docker push pgautoupgrade/pgautoupgrade:16-alpine3.8 && \ + docker push pgautoupgrade/pgautoupgrade:16-alpine3.18 && \ docker push pgautoupgrade/pgautoupgrade:latest \ No newline at end of file diff --git a/README.md b/README.md index f6c6802..07a550f 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ If you instead want to run a specific version of PostgreSQL then pick a matching tag on our Docker Hub. For example, to use PostgreSQL 15 you can use: - pgautoupgrade/pgautoupgrade:15-alpine3.8 + pgautoupgrade/pgautoupgrade:15-alpine3.18 # For Developers diff --git a/test.sh b/test.sh index 8e45eed..bc7c01c 100755 --- a/test.sh +++ b/test.sh @@ -2,6 +2,9 @@ FAILURE=0 +# Array of PostgreSQL versions for testing +PG_VERSIONS=(9.5 9.6 10 11 12 13 14 15) + # Stop any existing containers from previous test runs test_down() { docker-compose -f test/docker-compose-pgauto.yml down @@ -21,15 +24,7 @@ test_run() { docker-compose -f "docker-compose-pg${VERSION}.yml" run --rm server create_db # Start Redash normally, using an "autoupdate" version of PostgreSQL - if [ "${TARGET}" = "16" ]; then - docker-compose -f docker-compose-pgauto.yml up -d - elif [ "${TARGET}" = "15" ]; then - TARGET_TAG=15-dev docker-compose -f docker-compose-pgauto.yml up -d - elif [ "${TARGET}" = "14" ]; then - TARGET_TAG=14-dev docker-compose -f docker-compose-pgauto.yml up -d - elif [ "${TARGET}" = "13" ]; then - TARGET_TAG=13-dev docker-compose -f docker-compose-pgauto.yml up -d - fi + TARGET_TAG="${TARGET}-alpine3.18" docker-compose -f docker-compose-pgauto.yml up # Verify the PostgreSQL data files are now the target version PGVER=$(sudo cat postgres-data/PG_VERSION) @@ -64,47 +59,21 @@ fi # Change into the test directory cd test || exit 1 -# Testing upgrading from each major PG version directly to PG 13 -test_run 9.5 13 -test_run 9.6 13 -test_run 10 13 -test_run 11 13 -test_run 12 13 - -# Testing upgrading from each major PG version directly to PG 14 -test_run 9.5 14 -test_run 9.6 14 -test_run 10 14 -test_run 11 14 -test_run 12 14 -test_run 13 14 - -# Testing upgrading from each major PG version directly to PG 15 -test_run 9.5 15 -test_run 9.6 15 -test_run 10 15 -test_run 11 15 -test_run 12 15 -test_run 13 15 -test_run 14 15 - -# Testing upgrading from each major PG version directly to PG 16 -test_run 9.5 16 -test_run 9.6 16 -test_run 10 16 -test_run 11 16 -test_run 12 16 -test_run 13 16 -test_run 14 16 -test_run 15 16 +for version in "${PG_VERSIONS[@]}"; do + # Only test if the version is less than the latest version + if [[ $(echo "$version < $PGTARGET" | bc) -eq 1 ]]; then + test_run "$version" "$PGTARGET" + fi +done +# Check for failure if [ "${FAILURE}" -ne 0 ]; then - echo - echo "FAILURE: Automatic upgrade of PostgreSQL failed in one of the tests. Please investigate." - echo - exit 1 + echo + echo "FAILURE: Automatic upgrade of PostgreSQL failed in one of the tests. Please investigate." + echo + exit 1 else - echo - echo "SUCCESS: Automatic upgrade testing of PostgreSQL to PG 13, 14, 15, and 16 passed without issue." - echo + echo + echo "SUCCESS: Automatic upgrade testing of PostgreSQL to all versions up to $LATEST_VERSION passed without issue." + echo fi