diff --git a/CHANGELOG.md b/CHANGELOG.md index 6110a9c76..011963656 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## [Unreleased] +- Changed compression format and S3 URL for Python runtime archives. ([#1567](https://github.com/heroku/heroku-buildpack-python/pull/1567)) - Adjusted compiler options used to build Python for improved parity with the Docker Hub Python images. ([#1566](https://github.com/heroku/heroku-buildpack-python/pull/1566)) - Excluded `LD_LIBRARY_PATH` and `PYTHONHOME` app config vars when invoking subprocesses during the build. ([#1565](https://github.com/heroku/heroku-buildpack-python/pull/1565)) diff --git a/bin/steps/python b/bin/steps/python index b231e92bb..95046ac1b 100755 --- a/bin/steps/python +++ b/bin/steps/python @@ -8,8 +8,11 @@ runtime-fixer runtime.txt || true PYTHON_VERSION=$(cat runtime.txt) -# The location of the pre-compiled python binary. -PYTHON_URL="${S3_BASE_URL}/${STACK}/runtimes/${PYTHON_VERSION}.tar.gz" +# The Python runtime archive filename is of form: 'python-X.Y.Z-ubuntu-22.04-amd64.tar.zst' +# The Ubuntu version is calculated from `STACK` since it's faster than calling `lsb_release`. +# TODO: Switch to dynamically calculating the architecture when adding support for Heroku-24. +UBUNTU_VERSION="${STACK/heroku-}.04" +PYTHON_URL="${S3_BASE_URL}/${PYTHON_VERSION}-ubuntu-${UBUNTU_VERSION}-amd64.tar.zst" if ! curl --output /dev/null --silent --head --fail --retry 3 --retry-connrefused --connect-timeout 10 "${PYTHON_URL}"; then puts-warn "Requested runtime '${PYTHON_VERSION}' is not available for this stack (${STACK})." @@ -135,7 +138,7 @@ else # Prepare destination directory. mkdir -p .heroku/python - if ! curl --silent --show-error --fail --retry 3 --retry-connrefused --connect-timeout 10 "${PYTHON_URL}" | tar -zxC .heroku/python; then + if ! curl --silent --show-error --fail --retry 3 --retry-connrefused --connect-timeout 10 "${PYTHON_URL}" | tar --zstd --extract --directory .heroku/python; then # The Python version was confirmed to exist previously, so any failure here is due to # a networking issue or archive/buildpack bug rather than the runtime not existing. puts-warn "Failed to download/install ${PYTHON_VERSION}" diff --git a/builds/build_python_runtime.sh b/builds/build_python_runtime.sh index 814267ad0..dfd7bc368 100755 --- a/builds/build_python_runtime.sh +++ b/builds/build_python_runtime.sh @@ -10,7 +10,7 @@ PYTHON_MAJOR_VERSION="${PYTHON_VERSION%.*}" # we install Python into an arbitrary location that intentionally matches neither location. INSTALL_DIR="/tmp/python" SRC_DIR="/tmp/src" -UPLOAD_DIR="/tmp/upload/${STACK}/runtimes" +UPLOAD_DIR="/tmp/upload" function error() { echo "Error: ${1}" >&2 @@ -213,12 +213,12 @@ LD_LIBRARY_PATH="${SRC_DIR}" "${SRC_DIR}/python" -m compileall -f --invalidation # This symlink must be relative, to ensure that the Python install remains relocatable. ln -srvT "${INSTALL_DIR}/bin/python3" "${INSTALL_DIR}/bin/python" -# The tar file is gzipped separately, so we can set a higher gzip compression level than -# the default. In the future we'll also want to create a second archive that used zstd. -# Results in a compressed archive filename of form: 'python-X.Y.Z.tar.gz' -TAR_FILEPATH="${UPLOAD_DIR}/python-${PYTHON_VERSION}.tar" +# Results in a compressed archive filename of form: 'python-X.Y.Z-ubuntu-22.04-amd64.tar.zst' +# TODO: Switch to dynamically calculating the architecture when adding support for Heroku-24. +UBUNTU_VERSION=$(lsb_release --short --release 2>/dev/null) +TAR_FILEPATH="${UPLOAD_DIR}/python-${PYTHON_VERSION}-ubuntu-${UBUNTU_VERSION}-amd64.tar" tar --create --format=pax --sort=name --file "${TAR_FILEPATH}" --directory="${INSTALL_DIR}" . -gzip --best "${TAR_FILEPATH}" +zstd -T0 -22 --ultra --long --no-progress --rm "${TAR_FILEPATH}" du --max-depth 1 --human-readable "${INSTALL_DIR}" du --all --human-readable "${UPLOAD_DIR}"