diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c5a1fb8e..575845c60 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ ## [Unreleased] +- Fixed cache handling so that it now also discards the cache when the package manager (or its version) changes. ([#1679](https://github.com/heroku/heroku-buildpack-python/pull/1679)) +- Improved the build log output shown when restoring or discarding the cache. For example, if the cache was invalidated all reasons are now shown. ([#1679](https://github.com/heroku/heroku-buildpack-python/pull/1679)) +- Stopped performing unnecessary cache file copies when the cache is due to be invalidated. This required moving the cache restoration step to after the `bin/pre_compile` hook runs. ([#1679](https://github.com/heroku/heroku-buildpack-python/pull/1679)) +- Fixed cache restoration in the case where an app's `requirements.txt` was formerly a symlink. ([#1679](https://github.com/heroku/heroku-buildpack-python/pull/1679)) +- Added buildpack metrics for the status of the cache and duration of cache restoration/saving. ([#1679](https://github.com/heroku/heroku-buildpack-python/pull/1679)) ## [v262] - 2024-10-25 diff --git a/bin/compile b/bin/compile index 7e40754d6..4c2d271c3 100755 --- a/bin/compile +++ b/bin/compile @@ -19,6 +19,7 @@ ENV_DIR="${3}" BUILDPACK_DIR=$(cd "$(dirname "$(dirname "${BASH_SOURCE[0]}")")" && pwd) source "${BUILDPACK_DIR}/bin/utils" +source "${BUILDPACK_DIR}/lib/cache.sh" source "${BUILDPACK_DIR}/lib/hooks.sh" source "${BUILDPACK_DIR}/lib/metadata.sh" source "${BUILDPACK_DIR}/lib/output.sh" @@ -93,56 +94,14 @@ export PIP_NO_PYTHON_VERSION_WARNING=1 cd "$BUILD_DIR" -# The Cache -# --------- - -# The workflow for the Python Buildpack's cache is as follows: -# -# - `~/.heroku/{known-paths}` are copied from the cache into the slug. -# - The build is executed, modifying `~/.heroku/{known-paths}`. -# - Once the build is complete, `~/.heroku/{known-paths}` is copied back into the cache. - -mkdir -p "$CACHE_DIR/.heroku" - -# Restore old artifacts from the cache. -mkdir -p .heroku -# The Python installation. -cp -R "$CACHE_DIR/.heroku/python" .heroku/ &>/dev/null || true -# A plain text file which contains the current stack being used (used for cache busting). -cp -R "$CACHE_DIR/.heroku/python-stack" .heroku/ &>/dev/null || true -# A plain text file which contains the current python version being used (used for cache busting). -cp -R "$CACHE_DIR/.heroku/python-version" .heroku/ &>/dev/null || true -# A plain text file which contains the current sqlite3 version being used (used for cache busting). -cp -R "$CACHE_DIR/.heroku/python-sqlite3-version" .heroku/ &>/dev/null || true -# "editable" installations of code repositories, via pip or pipenv. -if [[ -d "$CACHE_DIR/.heroku/src" ]]; then - cp -R "$CACHE_DIR/.heroku/src" .heroku/ &>/dev/null || true -fi - # Runs a `bin/pre_compile` script if found in the app source, allowing build customisation. hooks::run_hook "pre_compile" -# TODO: Clear the cache if this isn't a valid version, as part of the cache refactor. -# (Currently the version is instead validated in `read_requested_python_version()`) -if [[ -f "$CACHE_DIR/.heroku/python-version" ]]; then - cached_python_version="$(cat "${CACHE_DIR}/.heroku/python-version")" - # `python-X.Y.Z` -> `X.Y` - cached_python_version="${cached_python_version#python-}" -else - cached_python_version= -fi - -# We didn't always record the stack version. -if [[ -f "$CACHE_DIR/.heroku/python-stack" ]]; then - CACHED_PYTHON_STACK=$(cat "$CACHE_DIR/.heroku/python-stack") -else - # shellcheck disable=SC2154 # TODO: Env var is referenced but not assigned. - CACHED_PYTHON_STACK=$STACK -fi - package_manager="$(package_manager::determine_package_manager "${BUILD_DIR}")" meta_set "package_manager" "${package_manager}" +cached_python_version="$(cache::cached_python_version "${CACHE_DIR}")" + # We use the Bash 4.3+ `nameref` feature to pass back multiple values from this function # without having to hardcode globals. See: https://stackoverflow.com/a/38997681 python_version::read_requested_python_version "${BUILD_DIR}" "${package_manager}" "${cached_python_version}" requested_python_version python_version_origin @@ -170,6 +129,8 @@ python_major_version="${python_full_version%.*}" meta_set "python_version" "${python_full_version}" meta_set "python_version_major" "${python_major_version}" +cache::restore "${BUILD_DIR}" "${CACHE_DIR}" "${STACK:?}" "${cached_python_version}" "${python_full_version}" "${package_manager}" + # The directory for the .profile.d scripts. mkdir -p "$(dirname "$PROFILE_PATH")" # The directory for editable VCS dependencies. @@ -300,18 +261,6 @@ cp "${BUILDPACK_DIR}/vendor/python.gunicorn.sh" "$GUNICORN_PROFILE_PATH" # Runs a `bin/post_compile` script if found in the app source, allowing build customisation. hooks::run_hook "post_compile" -# Store new artifacts in the cache. -rm -rf "$CACHE_DIR/.heroku/python" -rm -rf "$CACHE_DIR/.heroku/python-version" -rm -rf "$CACHE_DIR/.heroku/python-stack" -rm -rf "$CACHE_DIR/.heroku/src" - -mkdir -p "$CACHE_DIR/.heroku" -cp -R .heroku/python "$CACHE_DIR/.heroku/" -cp -R .heroku/python-version "$CACHE_DIR/.heroku/" -cp -R .heroku/python-stack "$CACHE_DIR/.heroku/" &>/dev/null || true -if [[ -d .heroku/src ]]; then - cp -R .heroku/src "$CACHE_DIR/.heroku/" &>/dev/null || true -fi +cache::save "${BUILD_DIR}" "${CACHE_DIR}" "${STACK}" "${python_full_version}" "${package_manager}" meta_time "total_duration" "${compile_start_time}" diff --git a/bin/report b/bin/report index 71bd20384..ecc08f0fa 100755 --- a/bin/report +++ b/bin/report @@ -60,6 +60,7 @@ kv_pair_string() { } STRING_FIELDS=( + cache_status django_collectstatic failure_reason nltk_downloader @@ -76,6 +77,8 @@ STRING_FIELDS=( # We don't want to quote numeric or boolean fields. ALL_OTHER_FIELDS=( + cache_restore_duration + cache_save_duration dependencies_install_duration django_collectstatic_duration nltk_downloader_duration diff --git a/bin/steps/python b/bin/steps/python index 37fbf6a34..316bb19ab 100755 --- a/bin/steps/python +++ b/bin/steps/python @@ -1,6 +1,5 @@ #!/usr/bin/env bash # shellcheck disable=SC2154 # TODO: Env var is referenced but not assigned. -# shellcheck disable=SC2250 # TODO: Use braces around variable references even when not strictly required. set -euo pipefail @@ -27,6 +26,28 @@ if ! curl --output /dev/null --silent --head --fail --retry 3 --retry-connrefuse exit 1 fi +if [[ -f "${BUILD_DIR}/.heroku/python/bin/python" ]]; then + output::step "Using cached install of Python ${python_full_version}" +else + output::step "Installing Python ${python_full_version}" + mkdir -p "${BUILD_DIR}/.heroku/python" + + if ! curl --silent --show-error --fail --retry 3 --retry-connrefused --connect-timeout 10 "${PYTHON_URL}" | tar --zstd --extract --directory "${BUILD_DIR}/.heroku/python"; then + # The Python version was confirmed to exist previously, so any failure here is due to + # a networking issue or archive/buildpack bug rather than the runtime not existing. + output::error <<-EOF + Error: Failed to download/install Python ${python_full_version}. + + In some cases, this happens due to an unstable network connection. + Please try again and to see if the error resolves itself. + EOF + meta_set "failure_reason" "python-download" + exit 1 + fi + + hash -r +fi + function warn_if_patch_update_available() { local requested_full_version="${1}" local requested_major_version="${2}" @@ -69,69 +90,3 @@ if [[ "${python_major_version}" == "3.8" ]]; then fi warn_if_patch_update_available "${python_full_version}" "${python_major_version}" - -if [[ "$STACK" != "$CACHED_PYTHON_STACK" ]]; then - output::step "Stack has changed from $CACHED_PYTHON_STACK to $STACK, clearing cache" - rm -rf .heroku/python-stack .heroku/python-version .heroku/python .heroku/vendor .heroku/python .heroku/python-sqlite3-version -fi - -# TODO: Clean this up as part of the cache refactor. -if [[ -f .heroku/python-version ]]; then - if [[ "${cached_python_version}" != "${python_full_version}" ]]; then - output::step "Python version has changed from ${cached_python_version} to ${python_full_version}, clearing cache" - rm -rf .heroku/python - else - SKIP_INSTALL=1 - fi -fi - -# If using pip, check if we should reinstall python dependencies given that requirements.txt -# is non-deterministic (not all packages pinned, doesn't handle uninstalls etc). We don't need -# to do this when using Pipenv, since it has a lockfile and syncs the packages for us. -if [[ -f "${BUILD_DIR}/requirements.txt" ]]; then - if [[ ! -f "$CACHE_DIR/.heroku/requirements.txt" ]]; then - # This is a the first build of an app (or the build cache was cleared). Since there - # are no cached packages, we only need to store the requirements file for next time. - cp -R "$BUILD_DIR/requirements.txt" "$CACHE_DIR/.heroku/requirements.txt" - else - # IF there IS a cached directory, check for differences with the new one - if ! diff "$BUILD_DIR/requirements.txt" "$CACHE_DIR/.heroku/requirements.txt" &>/dev/null; then - output::step "Requirements file has been changed, clearing cached dependencies" - # if there are any differences, clear the Python cache - # Installing Python over again does not take noticably more time - cp -R "$BUILD_DIR/requirements.txt" "$CACHE_DIR/.heroku/requirements.txt" - rm -rf .heroku/python - unset SKIP_INSTALL - else - output::step "No change in requirements detected, installing from cache" - fi - fi -fi - -if [[ "${SKIP_INSTALL:-0}" == "1" ]]; then - output::step "Using cached install of Python ${python_full_version}" -else - output::step "Installing Python ${python_full_version}" - - # Prepare destination directory. - mkdir -p .heroku/python - - if ! curl --silent --show-error --fail --retry 3 --retry-connrefused --connect-timeout 10 "${PYTHON_URL}" | tar --zstd --extract --directory .heroku/python; then - # The Python version was confirmed to exist previously, so any failure here is due to - # a networking issue or archive/buildpack bug rather than the runtime not existing. - output::error <<-EOF - Error: Failed to download/install Python ${python_full_version}. - - In some cases, this happens due to an unstable network connection. - Please try again and to see if the error resolves itself. - EOF - meta_set "failure_reason" "python-download" - exit 1 - fi - - # Record for future reference. - echo "python-${python_full_version}" >.heroku/python-version - echo "$STACK" >.heroku/python-stack - - hash -r -fi diff --git a/bin/steps/sqlite3 b/bin/steps/sqlite3 index 3ef119a35..1a94d5965 100755 --- a/bin/steps/sqlite3 +++ b/bin/steps/sqlite3 @@ -78,7 +78,4 @@ buildpack_sqlite3_install() { echo "Sqlite3 failed to install." # mcount "failure.python.sqlite3" fi - - # shellcheck disable=SC2154 # TODO: Env var is referenced but not assigned. - mkdir -p "$CACHE_DIR/.heroku/" } diff --git a/lib/cache.sh b/lib/cache.sh new file mode 100644 index 000000000..3e814f56e --- /dev/null +++ b/lib/cache.sh @@ -0,0 +1,192 @@ +#!/usr/bin/env bash + +# This is technically redundant, since all consumers of this lib will have enabled these, +# however, it helps Shellcheck realise the options under which these functions will run. +set -euo pipefail + +# Read the full Python version of the Python install in the cache, or the empty string +# if the cache is empty or doesn't contain a Python version metadata file. +function cache::cached_python_version() { + local cache_dir="${1}" + + if [[ -f "${cache_dir}/.heroku/python-version" ]]; then + local version + version="$(cat "${cache_dir}/.heroku/python-version")" + # For historical reasons the version has always been stored as `python-X.Y.Z`, + # so we have to remove the `python-` prefix. + echo "${version#python-}" + fi +} + +# Validates and restores the contents of the build cache if appropriate. +# The cache is discarded if any of the following have changed: +# - Stack +# - Python version +# - Package manager +# - Package manager version +# - requirements.txt contents (pip only) +function cache::restore() { + local build_dir="${1}" + local cache_dir="${2}" + local stack="${3}" + local cached_python_version="${4}" + local python_full_version="${5}" + local package_manager="${6}" + + if [[ ! -e "${cache_dir}/.heroku/python" ]]; then + meta_set "cache_status" "empty" + return 0 + fi + + local cache_restore_start_time + cache_restore_start_time=$(nowms) + local cache_invalidation_reasons=() + + local cached_stack + cached_stack="$(cat "${cache_dir}/.heroku/python-stack" || true)" + if [[ "${cached_stack}" != "${stack}" ]]; then + cache_invalidation_reasons+=("The stack has changed from ${cached_stack:-"unknown"} to ${stack}") + fi + + if [[ "${cached_python_version}" != "${python_full_version}" ]]; then + cache_invalidation_reasons+=("The Python version has changed from ${cached_python_version:-"unknown"} to ${python_full_version}") + fi + + # The metadata store only exists in caches created in v252+ of the buildpack (released 2024-06-17), + # so here and below we have to handle the case where `meta_prev_get` returns the empty string. + local cached_package_manager + cached_package_manager="$(meta_prev_get "package_manager")" + if [[ -z "${cached_package_manager}" ]]; then + # Using `compgen` since `[[ -d ... ]]` doesn't work with globs. + if compgen -G "${cache_dir}/.heroku/python/lib/python*/site-packages/pipenv" >/dev/null; then + cached_package_manager="pipenv" + elif compgen -G "${cache_dir}/.heroku/python/lib/python*/site-packages/pip" >/dev/null; then + cached_package_manager="pip" + fi + fi + + if [[ "${cached_package_manager}" != "${package_manager}" ]]; then + cache_invalidation_reasons+=("The package manager has changed from ${cached_package_manager:-"unknown"} to ${package_manager}") + else + case "${package_manager}" in + pip) + local cached_pip_version + cached_pip_version="$(meta_prev_get "pip_version")" + # Handle caches written by buildpack versions older than v252 (see above). + if [[ -z "${cached_pip_version}" ]]; then + # Whilst we don't know the old version, we know the pip version has likely + # changed since the last build, and would rather err on the side of caution. + # (The pip version was last updated in v246, but will be updated again soon.) + cache_invalidation_reasons+=("The pip version has changed") + elif [[ "${cached_pip_version}" != "${PIP_VERSION:?}" ]]; then + cache_invalidation_reasons+=("The pip version has changed from ${cached_pip_version} to ${PIP_VERSION}") + fi + + # We invalidate the cache if requirements.txt changes since pip is a package installer + # rather than a project/environment manager, and so does not deterministically manage + # installed Python packages. For example, if a package entry in a requirements file is + # later removed, pip will not uninstall the package. This check can be removed if we + # ever switch to only caching pip's HTTP/wheel cache rather than site-packages. + # TODO: Remove the `-f` check once the setup.py fallback feature is removed. + if [[ -f "${build_dir}/requirements.txt" ]] && ! cmp --silent "${cache_dir}/.heroku/requirements.txt" "${build_dir}/requirements.txt"; then + cache_invalidation_reasons+=("The contents of requirements.txt changed") + fi + ;; + pipenv) + local cached_pipenv_version + cached_pipenv_version="$(meta_prev_get "pipenv_version")" + # Handle caches written by buildpack versions older than v252 (see above). + if [[ -z "${cached_pipenv_version}" ]]; then + # Whilst we don't know the old version, we know the Pipenv version has definitely + # changed since buildpack v251. + cache_invalidation_reasons+=("The Pipenv version has changed") + elif [[ "${cached_pipenv_version}" != "${PIPENV_VERSION:?}" ]]; then + cache_invalidation_reasons+=("The Pipenv version has changed from ${cached_pipenv_version} to ${PIPENV_VERSION}") + fi + ;; + *) + utils::abort_internal_error "Unhandled package manager: ${package_manager}" + ;; + esac + fi + + if [[ -n "${cache_invalidation_reasons[*]}" ]]; then + output::step "Discarding cache since:" + local reason + for reason in "${cache_invalidation_reasons[@]}"; do + echo " - ${reason}" + done + + rm -rf \ + "${cache_dir}/.heroku/python" \ + "${cache_dir}/.heroku/python-stack" \ + "${cache_dir}/.heroku/python-version" \ + "${cache_dir}/.heroku/src" \ + "${cache_dir}/.heroku/requirements.txt" + + meta_set "cache_status" "discarded" + else + output::step "Restoring cache" + mkdir -p "${build_dir}/.heroku" + + # NB: For now this has to handle files already existing in build_dir since some apps accidentally + # run the Python buildpack twice. TODO: Add an explicit check/error for duplicate buildpacks. + # TODO: Investigate why errors are ignored and ideally stop doing so. + # TODO: Compare the performance of moving the directory vs copying files. + cp -R "${cache_dir}/.heroku/python" "${build_dir}/.heroku/" &>/dev/null || true + + # Editable VCS code repositories, used by pip/pipenv. + if [[ -d "${cache_dir}/.heroku/src" ]]; then + cp -R "${cache_dir}/.heroku/src" "${build_dir}/.heroku/" &>/dev/null || true + fi + + meta_set "cache_status" "reused" + fi + + # Remove any legacy cache contents written by older buildpack versions. + rm -rf \ + "${cache_dir}/.heroku/python-sqlite3-version" \ + "${cache_dir}/.heroku/vendor" + + meta_time "cache_restore_duration" "${cache_restore_start_time}" +} + +# Copies Python and dependencies from the build directory to the cache, for use by subsequent builds. +function cache::save() { + local build_dir="${1}" + local cache_dir="${2}" + local stack="${3}" + local python_full_version="${4}" + local package_manager="${5}" + + local cache_save_start_time + cache_save_start_time=$(nowms) + + mkdir -p "${cache_dir}/.heroku" + + rm -rf "${cache_dir}/.heroku/python" + cp -R "${build_dir}/.heroku/python" "${cache_dir}/.heroku/" + + # Editable VCS code repositories, used by pip/pipenv. + rm -rf "${cache_dir}/.heroku/src" + if [[ -d "${build_dir}/.heroku/src" ]]; then + # TODO: Investigate why errors are ignored and ideally stop doing so. + cp -R "${build_dir}/.heroku/src" "${cache_dir}/.heroku/" &>/dev/null || true + fi + + # Metadata used by subsequent builds to determine whether the cache can be reused. + # These are written/consumed via separate files and not the metadata store for compatibility + # with buildpack versions prior to the metadata store existing (which was only added in v252). + echo "${stack}" >"${cache_dir}/.heroku/python-stack" + # For historical reasons the Python version was always stored with a `python-` prefix. + # We continue to use that format so that the file can be read by older buildpack versions. + echo "python-${python_full_version}" >"${cache_dir}/.heroku/python-version" + + # TODO: Simplify this once multiple package manager files being found is turned into an + # error and the setup.py fallback feature is removed. + if [[ "${package_manager}" == "pip" && -f "${build_dir}/requirements.txt" ]]; then + cp "${build_dir}/requirements.txt" "${cache_dir}/.heroku/" + fi + + meta_time "cache_save_duration" "${cache_save_start_time}" +} diff --git a/lib/kvstore.sh b/lib/kvstore.sh index ba3809383..c4a684fea 100644 --- a/lib/kvstore.sh +++ b/lib/kvstore.sh @@ -20,6 +20,7 @@ kv_clear() { } kv_set() { + # TODO: Stop ignoring an incorrect number of passed arguments. if [[ $# -eq 3 ]]; then local f="${1}" if [[ -f "${f}" ]]; then @@ -28,11 +29,15 @@ kv_set() { fi } +# Returns a key from the key-value store file, or else the empty string +# if the file doesn't exist or the key wasn't found. kv_get() { + # TODO: Stop ignoring an incorrect number of passed arguments. if [[ $# -eq 2 ]]; then local f="${1}" if [[ -f "${f}" ]]; then - grep "^${2}=" "${f}" | sed -e "s/^${2}=//" | tail -n 1 + # shellcheck disable=SC2310 # This function is invoked in an || condition so set -e will be disabled. + grep "^${2}=" "${f}" | sed -e "s/^${2}=//" | tail -n 1 || true fi fi } diff --git a/lib/pip.sh b/lib/pip.sh index eb4535a2d..e58cea5af 100644 --- a/lib/pip.sh +++ b/lib/pip.sh @@ -4,14 +4,16 @@ # however, it helps Shellcheck realise the options under which these functions will run. set -euo pipefail +PIP_VERSION=$(get_requirement_version 'pip') +SETUPTOOLS_VERSION=$(get_requirement_version 'setuptools') +WHEEL_VERSION=$(get_requirement_version 'wheel') + function pip::install_pip_setuptools_wheel() { # We use the pip wheel bundled within Python's standard library to install our chosen # pip version, since it's faster than `ensurepip` followed by an upgrade in place. local bundled_pip_module_path="${1}" local python_major_version="${2}" - # TODO: Either make these `local` or move elsewhere as part of the cache invalidation refactoring. - PIP_VERSION=$(get_requirement_version 'pip') meta_set "pip_version" "${PIP_VERSION}" local packages_to_install=( @@ -27,11 +29,8 @@ function pip::install_pip_setuptools_wheel() { # - Most of the Python ecosystem has stopped installing them for Python 3.12+ already. # See the Python CNB's removal for more details: https://github.com/heroku/buildpacks-python/pull/243 if [[ "${python_major_version}" == +(3.8|3.9|3.10|3.11|3.12) ]]; then - SETUPTOOLS_VERSION=$(get_requirement_version 'setuptools') - WHEEL_VERSION=$(get_requirement_version 'wheel') meta_set "setuptools_version" "${SETUPTOOLS_VERSION}" meta_set "wheel_version" "${WHEEL_VERSION}" - packages_to_install+=( "setuptools==${SETUPTOOLS_VERSION}" "wheel==${WHEEL_VERSION}" @@ -39,6 +38,9 @@ function pip::install_pip_setuptools_wheel() { packages_display_text+=", setuptools ${SETUPTOOLS_VERSION} and wheel ${WHEEL_VERSION}" fi + # Note: We still perform this install step even if the cache was reused, since we have no guarantee + # that the cached package versions are correct (different versions could have been specified in the + # app's requirements.txt in the last build). The install will be a no-op if the versions match. output::step "Installing ${packages_display_text}" /app/.heroku/python/bin/python "${bundled_pip_module_path}" install --quiet --disable-pip-version-check --no-cache-dir \ diff --git a/lib/pipenv.sh b/lib/pipenv.sh index bfe0032d6..cbd524a61 100644 --- a/lib/pipenv.sh +++ b/lib/pipenv.sh @@ -4,17 +4,19 @@ # however, it helps Shellcheck realise the options under which these functions will run. set -euo pipefail +PIPENV_VERSION=$(get_requirement_version 'pipenv') + +# TODO: Either enable or remove these. # export CLINT_FORCE_COLOR=1 # export PIPENV_FORCE_COLOR=1 function pipenv::install_pipenv() { - # TODO: Either make this `local` or move elsewhere as part of the cache invalidation refactoring. - PIPENV_VERSION=$(get_requirement_version 'pipenv') meta_set "pipenv_version" "${PIPENV_VERSION}" output::step "Installing Pipenv ${PIPENV_VERSION}" # TODO: Install Pipenv into a venv so it isn't leaked into the app environment. + # TODO: Skip installing Pipenv if its version hasn't changed (once it's installed into a venv). # TODO: Explore viability of making Pipenv only be available during the build, to reduce slug size. /app/.heroku/python/bin/pip install --quiet --disable-pip-version-check --no-cache-dir "pipenv==${PIPENV_VERSION}" } diff --git a/lib/python_version.sh b/lib/python_version.sh index bd52223fa..d08fed160 100644 --- a/lib/python_version.sh +++ b/lib/python_version.sh @@ -74,8 +74,7 @@ function python_version::read_requested_python_version() { fi fi - # Protect against invalid versions somehow having been written into the cache. - # TODO: Move this validation into the cache handling as part of the cache refactor? + # Protect against unsupported (eg PyPy) or invalid versions being found in the cache metadata. if [[ "${cached_python_version}" =~ ^${PYTHON_VERSION_REGEX}$ ]]; then version="${cached_python_version}" origin="cached" diff --git a/spec/fixtures/requirements_basic/requirements.txt b/spec/fixtures/requirements_basic/requirements.txt deleted file mode 100644 index eec3a2223..000000000 --- a/spec/fixtures/requirements_basic/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -# This package has been picked since it has no dependencies and is small/fast to install. -typing-extensions==4.12.2 diff --git a/spec/fixtures/requirements_basic/requirements.txt b/spec/fixtures/requirements_basic/requirements.txt new file mode 120000 index 000000000..eb513c3cd --- /dev/null +++ b/spec/fixtures/requirements_basic/requirements.txt @@ -0,0 +1 @@ +requirements/prod.txt \ No newline at end of file diff --git a/spec/fixtures/requirements_basic/requirements/prod.txt b/spec/fixtures/requirements_basic/requirements/prod.txt new file mode 100644 index 000000000..8a7dafe83 --- /dev/null +++ b/spec/fixtures/requirements_basic/requirements/prod.txt @@ -0,0 +1,5 @@ +# This requirements file is symlinked from the repo root's requirements.txt +# in order to test that symlinked requirements files work. + +# This package has been picked since it has no dependencies and is small/fast to install. +typing-extensions==4.12.2 diff --git a/spec/hatchet/ci_spec.rb b/spec/hatchet/ci_spec.rb index 125c2d8d0..098fcdfca 100644 --- a/spec/hatchet/ci_spec.rb +++ b/spec/hatchet/ci_spec.rb @@ -70,7 +70,7 @@ expect(test_run.output).to include(<<~OUTPUT) -----> Python app detected -----> Using Python #{DEFAULT_PYTHON_MAJOR_VERSION} specified in .python-version - -----> No change in requirements detected, installing from cache + -----> Restoring cache -----> Using cached install of Python #{DEFAULT_PYTHON_FULL_VERSION} -----> Installing pip #{PIP_VERSION}, setuptools #{SETUPTOOLS_VERSION} and wheel #{WHEEL_VERSION} -----> Installing SQLite3 @@ -146,6 +146,7 @@ expect(test_run.output).to match(Regexp.new(<<~REGEX)) -----> Python app detected -----> Using Python #{DEFAULT_PYTHON_MAJOR_VERSION} specified in .python-version + -----> Restoring cache -----> Using cached install of Python #{DEFAULT_PYTHON_FULL_VERSION} -----> Installing pip #{PIP_VERSION}, setuptools #{SETUPTOOLS_VERSION} and wheel #{WHEEL_VERSION} -----> Installing Pipenv #{PIPENV_VERSION} diff --git a/spec/hatchet/pip_spec.rb b/spec/hatchet/pip_spec.rb index 199f234cb..dcf9bb762 100644 --- a/spec/hatchet/pip_spec.rb +++ b/spec/hatchet/pip_spec.rb @@ -25,7 +25,7 @@ remote: -----> Installing pip #{PIP_VERSION}, setuptools #{SETUPTOOLS_VERSION} and wheel #{WHEEL_VERSION} remote: -----> Installing SQLite3 remote: -----> Installing requirements with pip - remote: Collecting typing-extensions==4.12.2 (from -r requirements.txt (line 2)) + remote: Collecting typing-extensions==4.12.2 (from -r requirements.txt (line 5)) remote: Downloading typing_extensions-4.12.2-py3-none-any.whl.metadata (3.0 kB) remote: Downloading typing_extensions-4.12.2-py3-none-any.whl (37 kB) remote: Installing collected packages: typing-extensions @@ -61,7 +61,7 @@ expect(clean_output(app.output)).to include(<<~OUTPUT) remote: -----> Python app detected remote: -----> Using Python #{DEFAULT_PYTHON_MAJOR_VERSION} specified in .python-version - remote: -----> No change in requirements detected, installing from cache + remote: -----> Restoring cache remote: -----> Using cached install of Python #{DEFAULT_PYTHON_FULL_VERSION} remote: -----> Installing pip #{PIP_VERSION}, setuptools #{SETUPTOOLS_VERSION} and wheel #{WHEEL_VERSION} remote: -----> Installing SQLite3 @@ -77,20 +77,23 @@ it 'clears the cache before installing the packages again' do app.deploy do |app| - File.write('requirements.txt', 'six', mode: 'a') + # The test fixture's requirements.txt is a symlink to a requirements file in a subdirectory in + # order to test that symlinked requirements files work in general and with cache invalidation. + File.write('requirements/prod.txt', 'six', mode: 'a') app.commit! app.push! expect(clean_output(app.output)).to include(<<~OUTPUT) remote: -----> Python app detected remote: -----> Using Python #{DEFAULT_PYTHON_MAJOR_VERSION} specified in .python-version - remote: -----> Requirements file has been changed, clearing cached dependencies + remote: -----> Discarding cache since: + remote: - The contents of requirements.txt changed remote: -----> Installing Python #{DEFAULT_PYTHON_FULL_VERSION} remote: -----> Installing pip #{PIP_VERSION}, setuptools #{SETUPTOOLS_VERSION} and wheel #{WHEEL_VERSION} remote: -----> Installing SQLite3 remote: -----> Installing requirements with pip - remote: Collecting typing-extensions==4.12.2 (from -r requirements.txt (line 2)) + remote: Collecting typing-extensions==4.12.2 (from -r requirements.txt (line 5)) remote: Downloading typing_extensions-4.12.2-py3-none-any.whl.metadata (3.0 kB) - remote: Collecting six (from -r requirements.txt (line 3)) + remote: Collecting six (from -r requirements.txt (line 6)) remote: Downloading six-1.16.0-py2.py3-none-any.whl.metadata (1.8 kB) remote: Downloading typing_extensions-4.12.2-py3-none-any.whl (37 kB) remote: Downloading six-1.16.0-py2.py3-none-any.whl (11 kB) @@ -104,7 +107,6 @@ context 'when the package manager has changed from Pipenv to pip since the last build' do let(:app) { Hatchet::Runner.new('spec/fixtures/pipenv_basic') } - # TODO: Fix this case so the cache is actually cleared. it 'clears the cache before installing with pip' do app.deploy do |app| FileUtils.rm(['Pipfile', 'Pipfile.lock']) @@ -112,15 +114,21 @@ FileUtils.cp(FIXTURE_DIR.join('requirements_basic/requirements.txt'), '.') app.commit! app.push! - expect(clean_output(app.output)).to match(Regexp.new(<<~REGEX)) + expect(clean_output(app.output)).to include(<<~OUTPUT) remote: -----> Python app detected remote: -----> Using Python #{DEFAULT_PYTHON_MAJOR_VERSION} specified in .python-version - remote: -----> Using cached install of Python #{DEFAULT_PYTHON_FULL_VERSION} + remote: -----> Discarding cache since: + remote: - The package manager has changed from pipenv to pip + remote: -----> Installing Python #{DEFAULT_PYTHON_FULL_VERSION} remote: -----> Installing pip #{PIP_VERSION}, setuptools #{SETUPTOOLS_VERSION} and wheel #{WHEEL_VERSION} remote: -----> Installing SQLite3 remote: -----> Installing requirements with pip - remote: -----> Discovering process types - REGEX + remote: Collecting typing-extensions==4.12.2 (from -r requirements.txt (line 5)) + remote: Downloading typing_extensions-4.12.2-py3-none-any.whl.metadata (3.0 kB) + remote: Downloading typing_extensions-4.12.2-py3-none-any.whl (37 kB) + remote: Installing collected packages: typing-extensions + remote: Successfully installed typing-extensions-4.12.2 + OUTPUT end end end diff --git a/spec/hatchet/pipenv_spec.rb b/spec/hatchet/pipenv_spec.rb index 4be306e46..57c01614d 100644 --- a/spec/hatchet/pipenv_spec.rb +++ b/spec/hatchet/pipenv_spec.rb @@ -56,6 +56,7 @@ expect(clean_output(app.output)).to match(Regexp.new(<<~REGEX)) remote: -----> Python app detected remote: -----> Using Python #{DEFAULT_PYTHON_MAJOR_VERSION} specified in Pipfile.lock + remote: -----> Restoring cache remote: -----> Using cached install of Python #{DEFAULT_PYTHON_FULL_VERSION} remote: -----> Installing pip #{PIP_VERSION}, setuptools #{SETUPTOOLS_VERSION} and wheel #{WHEEL_VERSION} remote: -----> Installing Pipenv #{PIPENV_VERSION} @@ -80,13 +81,13 @@ expect(clean_output(app.output)).to match(Regexp.new(<<~REGEX)) remote: -----> Python app detected remote: -----> Using Python 3.9.0 specified in Pipfile.lock + remote: -----> Installing Python 3.9.0 remote: remote: ! Warning: A Python security update is available! remote: ! remote: ! Upgrade as soon as possible to: Python #{LATEST_PYTHON_3_9} remote: ! See: https://devcenter.heroku.com/articles/python-runtimes remote: - remote: -----> Installing Python 3.9.0 remote: -----> Installing pip #{PIP_VERSION}, setuptools #{SETUPTOOLS_VERSION} and wheel #{WHEEL_VERSION} remote: -----> Installing Pipenv #{PIPENV_VERSION} remote: -----> Installing SQLite3 @@ -283,10 +284,36 @@ end end + context 'when the Pipenv version has changed since the last build' do + let(:buildpacks) { ['https://github.com/heroku/heroku-buildpack-python#v253'] } + let(:app) { Hatchet::Runner.new('spec/fixtures/pipenv_basic', buildpacks:) } + + it 'clears the cache before installing' do + app.deploy do |app| + update_buildpacks(app, [:default]) + app.commit! + app.push! + expect(clean_output(app.output)).to match(Regexp.new(<<~REGEX)) + remote: -----> Python app detected + remote: -----> Using Python #{DEFAULT_PYTHON_MAJOR_VERSION} specified in Pipfile.lock + remote: -----> Discarding cache since: + remote: - The Python version has changed from 3.12.4 to #{DEFAULT_PYTHON_FULL_VERSION} + remote: - The Pipenv version has changed from 2023.12.1 to #{PIPENV_VERSION} + remote: -----> Installing Python #{DEFAULT_PYTHON_FULL_VERSION} + remote: -----> Installing pip #{PIP_VERSION}, setuptools #{SETUPTOOLS_VERSION} and wheel #{WHEEL_VERSION} + remote: -----> Installing Pipenv #{PIPENV_VERSION} + remote: -----> Installing SQLite3 + remote: -----> Installing dependencies with Pipenv + remote: Installing dependencies from Pipfile.lock \\(.+\\)... + remote: -----> Discovering process types + REGEX + end + end + end + context 'when the package manager has changed from pip to Pipenv since the last build' do let(:app) { Hatchet::Runner.new('spec/fixtures/requirements_basic') } - # TODO: Fix this case so the cache is actually cleared. it 'clears the cache before installing with Pipenv' do app.deploy do |app| FileUtils.rm('.python-version') @@ -298,7 +325,9 @@ expect(clean_output(app.output)).to match(Regexp.new(<<~REGEX)) remote: -----> Python app detected remote: -----> Using Python #{DEFAULT_PYTHON_MAJOR_VERSION} specified in Pipfile.lock - remote: -----> Using cached install of Python #{DEFAULT_PYTHON_FULL_VERSION} + remote: -----> Discarding cache since: + remote: - The package manager has changed from pip to pipenv + remote: -----> Installing Python #{DEFAULT_PYTHON_FULL_VERSION} remote: -----> Installing pip #{PIP_VERSION}, setuptools #{SETUPTOOLS_VERSION} and wheel #{WHEEL_VERSION} remote: -----> Installing Pipenv #{PIPENV_VERSION} remote: -----> Installing SQLite3 diff --git a/spec/hatchet/python_update_warning_spec.rb b/spec/hatchet/python_update_warning_spec.rb index c10e5c288..993a07454 100644 --- a/spec/hatchet/python_update_warning_spec.rb +++ b/spec/hatchet/python_update_warning_spec.rb @@ -16,6 +16,7 @@ expect(clean_output(app.output)).to include(<<~OUTPUT) remote: -----> Python app detected remote: -----> Using Python 3.8.0 specified in runtime.txt + remote: -----> Installing Python 3.8.0 remote: remote: ! Warning: Support for Python 3.8 is ending soon! remote: ! @@ -34,7 +35,7 @@ remote: ! Upgrade as soon as possible to: Python #{LATEST_PYTHON_3_8} remote: ! See: https://devcenter.heroku.com/articles/python-runtimes remote: - remote: -----> Installing Python 3.8.0 + remote: -----> Installing pip #{PIP_VERSION}, setuptools #{SETUPTOOLS_VERSION} and wheel #{WHEEL_VERSION} OUTPUT end end @@ -70,13 +71,14 @@ expect(clean_output(app.output)).to include(<<~OUTPUT) remote: -----> Python app detected remote: -----> Using Python 3.9.0 specified in .python-version + remote: -----> Installing Python 3.9.0 remote: remote: ! Warning: A Python security update is available! remote: ! remote: ! Upgrade as soon as possible to: Python #{LATEST_PYTHON_3_9} remote: ! See: https://devcenter.heroku.com/articles/python-runtimes remote: - remote: -----> Installing Python 3.9.0 + remote: -----> Installing pip #{PIP_VERSION}, setuptools #{SETUPTOOLS_VERSION} and wheel #{WHEEL_VERSION} OUTPUT end end diff --git a/spec/hatchet/python_version_spec.rb b/spec/hatchet/python_version_spec.rb index 215148806..26318061f 100644 --- a/spec/hatchet/python_version_spec.rb +++ b/spec/hatchet/python_version_spec.rb @@ -63,14 +63,16 @@ remote: -----> Python app detected remote: -----> No Python version was specified. Using the same version as the last build: Python 3.12.3 remote: To use a different version, see: https://devcenter.heroku.com/articles/python-runtimes + remote: -----> Discarding cache since: + remote: - The pip version has changed + remote: -----> Installing Python 3.12.3 remote: remote: ! Warning: A Python security update is available! remote: ! remote: ! Upgrade as soon as possible to: Python #{LATEST_PYTHON_3_12} remote: ! See: https://devcenter.heroku.com/articles/python-runtimes remote: - remote: -----> No change in requirements detected, installing from cache - remote: -----> Using cached install of Python 3.12.3 + remote: -----> Installing pip #{PIP_VERSION}, setuptools #{SETUPTOOLS_VERSION} and wheel #{WHEEL_VERSION} OUTPUT expect(app.run('python -V')).to include('Python 3.12.3') end @@ -88,6 +90,7 @@ expect(clean_output(app.output)).to include(<<~OUTPUT) remote: -----> Python app detected remote: -----> Using Python 3.8 specified in .python-version + remote: -----> Installing Python #{LATEST_PYTHON_3_8} remote: remote: ! Warning: Support for Python 3.8 is ending soon! remote: ! @@ -100,7 +103,6 @@ remote: ! Upgrade to a newer Python version as soon as possible to keep your app secure. remote: ! See: https://devcenter.heroku.com/articles/python-runtimes remote: - remote: -----> Installing Python #{LATEST_PYTHON_3_8} remote: -----> Installing pip #{PIP_VERSION}, setuptools #{SETUPTOOLS_VERSION} and wheel #{WHEEL_VERSION} remote: -----> Installing SQLite3 remote: -----> Installing requirements with pip @@ -415,12 +417,11 @@ File.write('.python-version', '3.13') app.commit! app.push! - # TODO: The output shouldn't say "installing from cache", since it's not. expect(clean_output(app.output)).to include(<<~OUTPUT) remote: -----> Python app detected remote: -----> Using Python 3.13 specified in .python-version - remote: -----> Python version has changed from #{LATEST_PYTHON_3_9} to #{LATEST_PYTHON_3_13}, clearing cache - remote: -----> No change in requirements detected, installing from cache + remote: -----> Discarding cache since: + remote: - The Python version has changed from #{LATEST_PYTHON_3_9} to #{LATEST_PYTHON_3_13} remote: -----> Installing Python #{LATEST_PYTHON_3_13} remote: -----> Installing pip #{PIP_VERSION} remote: -----> Installing requirements with pip diff --git a/spec/hatchet/stack_spec.rb b/spec/hatchet/stack_spec.rb index 9c84e5d16..097eb92dc 100644 --- a/spec/hatchet/stack_spec.rb +++ b/spec/hatchet/stack_spec.rb @@ -19,20 +19,20 @@ update_buildpacks(app, [:default]) app.commit! app.push! - # TODO: The requirements output shouldn't say "installing from cache", since it's not. expect(clean_output(app.output)).to include(<<~OUTPUT) remote: -----> Python app detected remote: -----> No Python version was specified. Using the same version as the last build: Python 3.12.3 remote: To use a different version, see: https://devcenter.heroku.com/articles/python-runtimes + remote: -----> Discarding cache since: + remote: - The stack has changed from heroku-22 to heroku-24 + remote: - The pip version has changed + remote: -----> Installing Python 3.12.3 remote: remote: ! Warning: A Python security update is available! remote: ! remote: ! Upgrade as soon as possible to: Python #{LATEST_PYTHON_3_12} remote: ! See: https://devcenter.heroku.com/articles/python-runtimes remote: - remote: -----> Stack has changed from heroku-22 to heroku-24, clearing cache - remote: -----> No change in requirements detected, installing from cache - remote: -----> Installing Python 3.12.3 remote: -----> Installing pip #{PIP_VERSION}, setuptools #{SETUPTOOLS_VERSION} and wheel #{WHEEL_VERSION} remote: -----> Installing SQLite3 remote: -----> Installing requirements with pip @@ -55,8 +55,8 @@ remote: -----> Python app detected remote: -----> No Python version was specified. Using the same version as the last build: Python #{DEFAULT_PYTHON_FULL_VERSION} remote: To use a different version, see: https://devcenter.heroku.com/articles/python-runtimes - remote: -----> Stack has changed from heroku-24 to heroku-22, clearing cache - remote: -----> No change in requirements detected, installing from cache + remote: -----> Discarding cache since: + remote: - The stack has changed from heroku-24 to heroku-22 remote: -----> Installing Python #{DEFAULT_PYTHON_FULL_VERSION} remote: -----> Installing pip #{PIP_VERSION}, setuptools #{SETUPTOOLS_VERSION} and wheel #{WHEEL_VERSION} remote: -----> Installing SQLite3