From e8236eedadd8e8030dac1f5685cdaa985b65d9d9 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Fri, 1 Nov 2024 13:26:51 -0600 Subject: [PATCH 001/173] Initial changes to support new packages.broadcom.com infrastructure --- bootstrap-salt.sh | 739 ++++++++++++++++++++++++++++++---------------- 1 file changed, 486 insertions(+), 253 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 4c6b289bb..89ceb7424 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -26,7 +26,7 @@ #====================================================================================================================== set -o nounset # Treat unset variables as an error -__ScriptVersion="2024.09.24" +__ScriptVersion="2024.11.01" __ScriptName="bootstrap-salt.sh" __ScriptFullName="$0" @@ -267,9 +267,10 @@ _CUSTOM_REPO_URL="null" _CUSTOM_MASTER_CONFIG="null" _CUSTOM_MINION_CONFIG="null" _QUIET_GIT_INSTALLATION=$BS_FALSE -_REPO_URL="repo.saltproject.io" -_ONEDIR_DIR="salt" -_ONEDIR_NIGHTLY_DIR="salt-dev/${_ONEDIR_DIR}" +## DGM _REPO_URL="repo.saltproject.io" +## DGM _ONEDIR_DIR="salt" +## DGM _ONEDIR_NIGHTLY_DIR="salt-dev/${_ONEDIR_DIR}" +_REPO_URL="packages.broadcom.com/artifactory" _PY_EXE="python3" _MINIMUM_PIP_VERSION="9.0.1" _MINIMUM_SETUPTOOLS_VERSION="65.6.3" @@ -296,9 +297,9 @@ __usage() { - stable Install latest stable release. This is the default install type - stable [branch] Install latest version on a branch. Only supported - for packages available at repo.saltproject.io + for packages available at packages.broadcom.com - stable [version] Install a specific version. Only supported for - packages available at repo.saltproject.io + packages available at packages.broadcom.com To pin a 3xxx minor version, specify it as 3xxx.0 - testing RHEL-family specific: configure EPEL testing repo - git Install from the head of the master branch @@ -306,11 +307,11 @@ __usage() { commit) - onedir Install latest onedir release. - onedir [version] Install a specific version. Only supported for - onedir packages available at repo.saltproject.io + onedir packages available at packages.broadcom.com - onedir_rc Install latest onedir RC release. - onedir_rc [version] Install a specific version. Only supported for - onedir RC packages available at repo.saltproject.io + onedir RC packages available at packages.broadcom.com Examples: - ${__ScriptName} @@ -393,8 +394,8 @@ __usage() { And automatically accept the minion key. -R Specify a custom repository URL. Assumes the custom repository URL points to a repository that mirrors Salt packages located at - repo.saltproject.io. The option passed with -R replaces the - "repo.saltproject.io". If -R is passed, -r is also set. Currently only + packages.broadcom.com. The option passed with -R replaces the + "packages.broadcom.com". If -R is passed, -r is also set. Currently only works on CentOS/RHEL and Debian based distributions and macOS. -s Sleep time used when waiting for daemons to start, restart and when checking for the services running. Default: ${__DEFAULT_SLEEP} @@ -613,13 +614,14 @@ elif [ "$ITYPE" = "stable" ]; then _ONEDIR_REV="latest" ITYPE="onedir" else - if [ "$(echo "$1" | grep -E '^(nightly|latest|3006|3007)$')" != "" ]; then + if [ "$(echo "$1" | grep -E '^(latest|3006|3007)$')" != "" ]; then ONEDIR_REV="$1" _ONEDIR_REV="$1" ITYPE="onedir" shift elif [ "$(echo "$1" | grep -E '^([3-9][0-5]{2}[5-9](\.[0-9]*)?)')" != "" ]; then - ONEDIR_REV="minor/$1" + ## DGM ONEDIR_REV="minor/$1" don't have minor directory anymore + ONEDIR_REV="$1" _ONEDIR_REV="$1" ITYPE="onedir" shift @@ -633,46 +635,51 @@ elif [ "$ITYPE" = "onedir" ]; then if [ "$#" -eq 0 ];then ONEDIR_REV="latest" else - if [ "$(echo "$1" | grep -E '^(nightly|latest|3006|3007)$')" != "" ]; then + if [ "$(echo "$1" | grep -E '^(latest|3006|3007)$')" != "" ]; then ONEDIR_REV="$1" shift elif [ "$(echo "$1" | grep -E '^([3-9][0-9]{3}(\.[0-9]*)?)')" != "" ]; then - ONEDIR_REV="minor/$1" + ## DGM ONEDIR_REV="minor/$1" don't have minor directory anymore + ONEDIR_REV="$1" shift else - echo "Unknown onedir version: $1 (valid: 3006, 3007, latest, nightly.)" + echo "Unknown onedir version: $1 (valid: 3006, 3007, latest.)" exit 1 fi fi elif [ "$ITYPE" = "onedir_rc" ]; then - # Change the _ONEDIR_DIR to be the location for the RC packages - _ONEDIR_DIR="salt_rc/salt" - - # Change ITYPE to onedir so we use the regular onedir functions - ITYPE="onedir" - - if [ "$#" -eq 0 ];then - ONEDIR_REV="latest" - else - if [ "$(echo "$1" | grep -E '^(latest)$')" != "" ]; then - ONEDIR_REV="$1" - shift - elif [ "$(echo "$1" | grep -E '^([3-9][0-9]{3}?rc[0-9]-[0-9]$)')" != "" ]; then - # Handle the 3xxx.0 version as 3xxx archive (pin to minor) and strip the fake ".0" suffix - #ONEDIR_REV=$(echo "$1" | sed -E 's/^([3-9][0-9]{3})\.0$/\1/') - ONEDIR_REV="minor/$1" - shift - elif [ "$(echo "$1" | grep -E '^([3-9][0-9]{3}\.[0-9]?rc[0-9]$)')" != "" ]; then - # Handle the 3xxx.0 version as 3xxx archive (pin to minor) and strip the fake ".0" suffix - #ONEDIR_REV=$(echo "$1" | sed -E 's/^([3-9][0-9]{3})\.0$/\1/') - ONEDIR_REV="minor/$1" - shift - else - echo "Unknown onedir_rc version: $1 (valid: 3006-8, 3007-1, latest)" - exit 1 - fi - fi + echoerror "RC Releases are not supported at this time" + +## # Change the _ONEDIR_DIR to be the location for the RC packages +## _ONEDIR_DIR="salt_rc/salt" +## +## # Change ITYPE to onedir so we use the regular onedir functions +## ITYPE="onedir" +## +## if [ "$#" -eq 0 ];then +## ONEDIR_REV="latest" +## else +## if [ "$(echo "$1" | grep -E '^(latest)$')" != "" ]; then +## ONEDIR_REV="$1" +## shift +## elif [ "$(echo "$1" | grep -E '^([3-9][0-9]{3}?rc[0-9]-[0-9]$)')" != "" ]; then +## # Handle the 3xxx.0 version as 3xxx archive (pin to minor) and strip the fake ".0" suffix +## #ONEDIR_REV=$(echo "$1" | sed -E 's/^([3-9][0-9]{3})\.0$/\1/') +## ## DGM ONEDIR_REV="minor/$1" don't have minor directory anymore +## ONEDIR_REV="$1" +## shift +## elif [ "$(echo "$1" | grep -E '^([3-9][0-9]{3}\.[0-9]?rc[0-9]$)')" != "" ]; then +## # Handle the 3xxx.0 version as 3xxx archive (pin to minor) and strip the fake ".0" suffix +## #ONEDIR_REV=$(echo "$1" | sed -E 's/^([3-9][0-9]{3})\.0$/\1/') +## ## DGM ONEDIR_REV="minor/$1" don't have minor directory anymore +## ONEDIR_REV="$1" +## shift +## else +## echo "Unknown onedir_rc version: $1 (valid: 3006-8, 3007-1, latest)" +## exit 1 +## fi +## fi fi # Doing a quick start, so install master @@ -760,7 +767,7 @@ fi PY_PKG_VER=3 _PY_PKG_VER="python3" _PY_MAJOR_VERSION="3" -__PY_VERSION_REPO="py3" +## DGM __PY_VERSION_REPO="py3" # Check if we're installing via a different Python executable and set major version variables if [ -n "$_PY_EXE" ]; then @@ -811,7 +818,7 @@ if [ "$ITYPE" != "git" ]; then fi fi -# Set the _REPO_URL value based on if -R was passed or not. Defaults to repo.saltproject.io. +# Set the _REPO_URL value based on if -R was passed or not. Defaults to packages.broadcom.com if [ "$_CUSTOM_REPO_URL" != "null" ]; then _REPO_URL="$_CUSTOM_REPO_URL" @@ -1340,8 +1347,10 @@ __check_dpkg_architecture() { return 1 fi - __REPO_ARCH="$DPKG_ARCHITECTURE" - __REPO_ARCH_DEB='deb [signed-by=/usr/share/keyrings/salt-archive-keyring.gpg]' + ## DGM __REPO_ARCH="$DPKG_ARCHITECTURE" + ## DGM __REPO_ARCH_DEB='deb [signed-by=/usr/share/keyrings/salt-archive-keyring.gpg]' + ## DGM TBD wonder what to do here given use of salt.sources ????, this value's use has been commented out + ## DGM for now __REPO_ARCH_DEB='deb [signed-by=/etc/apt/keyrings/salt-archive-keyring.gpg]' __return_code=0 case $DPKG_ARCHITECTURE in @@ -1356,8 +1365,10 @@ __check_dpkg_architecture() { "arm64") # Saltstack official repository has full arm64 support since 3006 error_msg="" - __REPO_ARCH="arm64" - __REPO_ARCH_DEB="deb [signed-by=/usr/share/keyrings/salt-archive-keyring.gpg arch=$__REPO_ARCH]" + ## DGM __REPO_ARCH="arm64" + ## __REPO_ARCH_DEB="deb [signed-by=/usr/share/keyrings/salt-archive-keyring.gpg arch=$__REPO_ARCH]" + ## DGM TBD wonder what to do here given use of salt.sources ???, this value's use has been commented out? + ## DGM for now __REPO_ARCH_DEB="deb [signed-by=/etc/apt/keyrings/salt-archive-keyring.gpg arch=$__REPO_ARCH]" ;; "armhf") error_msg="$_REPO_URL doesn't have packages for your system architecture: $DPKG_ARCHITECTURE." @@ -1968,9 +1979,11 @@ __apt_key_fetch() { url=$1 tempfile="$(__temp_gpg_pub)" - + ## DGM __fetch_url "$tempfile" "$url" || return 1 + ## DGM cp -f "$tempfile" /usr/share/keyrings/salt-archive-keyring.gpg && chmod 644 /usr/share/keyrings/salt-archive-keyring.gpg || return 1 __fetch_url "$tempfile" "$url" || return 1 - cp -f "$tempfile" /usr/share/keyrings/salt-archive-keyring.gpg && chmod 644 /usr/share/keyrings/salt-archive-keyring.gpg || return 1 + mkdir -p /etc/apt/keyrings + cp -f "$tempfile" /etc/apt/keyrings/salt-archive-keyring.gpg && chmod 644 /etc/apt/keyrings/salt-archive-keyring.pgp || return 1 rm -f "$tempfile" return 0 @@ -2886,13 +2899,14 @@ __install_saltstack_ubuntu_repository() { if { [ "$DISTRO_MAJOR_VERSION" -eq 20 ] && [ "$DISTRO_MINOR_VERSION" -eq 10 ]; } || \ { [ "$DISTRO_MAJOR_VERSION" -eq 22 ] && [ "$DISTRO_MINOR_VERSION" -eq 10 ]; } || \ - [ "$DISTRO_MAJOR_VERSION" -eq 21 ] || [ "$DISTRO_MAJOR_VERSION" -eq 23 ]; then + { [ "$DISTRO_MAJOR_VERSION" -eq 24 ] && [ "$DISTRO_MINOR_VERSION" -eq 10 ]; } || \ + [ "$DISTRO_MAJOR_VERSION" -eq 21 ] || [ "$DISTRO_MAJOR_VERSION" -eq 23 ] || [ "$DISTRO_MAJOR_VERSION" -eq 25 ]; then echowarn "Non-LTS Ubuntu detected, but stable packages requested. Trying packages for previous LTS release. You may experience problems." - UBUNTU_VERSION=24.04 - UBUNTU_CODENAME="noble" - else - UBUNTU_VERSION=${DISTRO_VERSION} - UBUNTU_CODENAME=${DISTRO_CODENAME} + ## DGM UBUNTU_VERSION=24.04 + ## DGM UBUNTU_CODENAME="noble" + ## DGM else + ## DGM UBUNTU_VERSION=${DISTRO_VERSION} + ## DGM UBUNTU_CODENAME=${DISTRO_CODENAME} fi # Install downloader backend for GPG keys fetching @@ -2922,24 +2936,45 @@ __install_saltstack_ubuntu_repository() { fi # SaltStack's stable Ubuntu repository: - SALTSTACK_UBUNTU_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/ubuntu/${UBUNTU_VERSION}/${__REPO_ARCH}/${STABLE_REV}" - echo "$__REPO_ARCH_DEB $SALTSTACK_UBUNTU_URL $UBUNTU_CODENAME main" > /etc/apt/sources.list.d/salt.list - __apt_key_fetch "$SALTSTACK_UBUNTU_URL/SALT-PROJECT-GPG-PUBKEY-2023.gpg" || return 1 + ## DGM SALTSTACK_UBUNTU_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/ubuntu/${UBUNTU_VERSION}/${__REPO_ARCH}/${STABLE_REV}" + ## DGM echo "$__REPO_ARCH_DEB $SALTSTACK_UBUNTU_URL $UBUNTU_CODENAME main" > /etc/apt/sources.list.d/salt.list + ## DGM __apt_key_fetch "$SALTSTACK_UBUNTU_URL/SALT-PROJECT-GPG-PUBKEY-2023.gpg" || return 1 + __fetch_url "/etc/apt/sources.list.d/salt.sources" "https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.sources" + __apt_key_fetch "${HTTP_VAL}://${_REPO_URL}//api/security/keypair/SaltProjectKey/public" || return 1 __wait_for_apt apt-get update || return 1 + + if [ "$STABLE_REV" != "latest" ]; then + # latest is default + STABLE_REV_MAJOR=$(echo "$STABLE_REV" | cut -d '.' -f 1) + if [ "$STABLE_REV_MAJOR" -eq "3006" ]; then + echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 + echo "Pin: version 3006.*" >> /etc/apt/preferences.d/salt-pin-1001 + echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 + elif [ "$STABLE_REV_MAJOR" -eq "3007" ]; then + echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 + echo "Pin: version 3007.*" >> /etc/apt/preferences.d/salt-pin-1001 + echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 + fi + fi + + ## DGM _ONEDIR_TYPE="saltproject-deb" + ## DGM SALTSTACK_UBUNTU_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_TYPE}/pool/" + } __install_saltstack_ubuntu_onedir_repository() { + echodebug "__install_saltstack_ubuntu_onedir_repository() entry" # Workaround for latest non-LTS Ubuntu if { [ "$DISTRO_MAJOR_VERSION" -eq 20 ] && [ "$DISTRO_MINOR_VERSION" -eq 10 ]; } || \ { [ "$DISTRO_MAJOR_VERSION" -eq 22 ] && [ "$DISTRO_MINOR_VERSION" -eq 10 ]; } || \ - [ "$DISTRO_MAJOR_VERSION" -eq 21 ] || [ "$DISTRO_MAJOR_VERSION" -eq 23 ]; then + [ "$DISTRO_MAJOR_VERSION" -eq 21 ] || [ "$DISTRO_MAJOR_VERSION" -eq 23 ] || [ "$DISTRO_MAJOR_VERSION" -eq 25 ]; then echowarn "Non-LTS Ubuntu detected, but stable packages requested. Trying packages for previous LTS release. You may experience problems." - UBUNTU_VERSION=24.04 - UBUNTU_CODENAME="noble" - else - UBUNTU_VERSION=${DISTRO_VERSION} - UBUNTU_CODENAME=${DISTRO_CODENAME} + ## DGM UBUNTU_VERSION=24.04 + ## DGM UBUNTU_CODENAME="noble" + ## DGM else + ## DGM UBUNTU_VERSION=${DISTRO_VERSION} + ## DGM UBUNTU_CODENAME=${DISTRO_CODENAME} fi # Install downloader backend for GPG keys fetching @@ -2964,18 +2999,34 @@ __install_saltstack_ubuntu_onedir_repository() { __apt_get_install_noinput ${__PACKAGES} || return 1 # SaltStack's stable Ubuntu repository: - SALTSTACK_UBUNTU_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/ubuntu/${UBUNTU_VERSION}/${__REPO_ARCH}/${ONEDIR_REV}/" - if [ "${ONEDIR_REV}" = "nightly" ] ; then - SALTSTACK_UBUNTU_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/ubuntu/${UBUNTU_VERSION}/${__REPO_ARCH}/" - fi - echo "$__REPO_ARCH_DEB $SALTSTACK_UBUNTU_URL $UBUNTU_CODENAME main" > /etc/apt/sources.list.d/salt.list - - __apt_key_fetch "${SALTSTACK_UBUNTU_URL}SALT-PROJECT-GPG-PUBKEY-2023.gpg" || return 1 - + ## DGM TBD need deb location + ## DGM SALTSTACK_UBUNTU_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/ubuntu/${UBUNTU_VERSION}/${__REPO_ARCH}/${ONEDIR_REV}/" + ## DGM if [ "${ONEDIR_REV}" = "nightly" ] ; then + ## DGM SALTSTACK_UBUNTU_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/ubuntu/${UBUNTU_VERSION}/${__REPO_ARCH}/" + ## DGM fi + ## DGM echo "$__REPO_ARCH_DEB $SALTSTACK_UBUNTU_URL $UBUNTU_CODENAME main" > /etc/apt/sources.list.d/salt.list + + __fetch_url "/etc/apt/sources.list.d/salt.sources" "https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.sources" + __apt_key_fetch "${HTTP_VAL}://${_REPO_URL}//api/security/keypair/SaltProjectKey/public" || return 1 __wait_for_apt apt-get update || return 1 + + if [ "$STABLE_REV" != "latest" ]; then + # latest is default + STABLE_REV_MAJOR=$(echo "$STABLE_REV" | cut -d '.' -f 1) + if [ "$STABLE_REV_MAJOR" -eq "3006" ]; then + echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 + echo "Pin: version 3006.*" >> /etc/apt/preferences.d/salt-pin-1001 + echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 + elif [ "$STABLE_REV_MAJOR" -eq "3007" ]; then + echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 + echo "Pin: version 3007.*" >> /etc/apt/preferences.d/salt-pin-1001 + echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 + fi + fi } install_ubuntu_deps() { + echodebug "install_ubuntu_deps() entry" if [ "$_DISABLE_REPOS" -eq $BS_FALSE ]; then # Install add-apt-repository if ! __check_command_exists add-apt-repository; then @@ -3046,7 +3097,8 @@ install_ubuntu_stable_deps() { if [ "${_UPGRADE_SYS}" -eq $BS_TRUE ]; then if [ "${_INSECURE_DL}" -eq $BS_TRUE ]; then ## apt-key is deprecated - if [ "$DISTRO_MAJOR_VERSION" -ge 20 ] || [ "$DISTRO_MAJOR_VERSION" -ge 21 ] || [ "$DISTRO_MAJOR_VERSION" -ge 22 ] || [ "$DISTRO_MAJOR_VERSION" -ge 23 ] || [ "$DISTRO_MAJOR_VERSION" -ge 24 ]; then + ## DGM if [ "$DISTRO_MAJOR_VERSION" -ge 20 ] || [ "$DISTRO_MAJOR_VERSION" -ge 21 ] || [ "$DISTRO_MAJOR_VERSION" -ge 22 ] || [ "$DISTRO_MAJOR_VERSION" -ge 23 ] || [ "$DISTRO_MAJOR_VERSION" -ge 24 ]; then + if [ "$DISTRO_MAJOR_VERSION" -ge 20 ]; then __apt_get_install_noinput --allow-unauthenticated debian-archive-keyring && apt-get update || return 1 else __apt_get_install_noinput --allow-unauthenticated debian-archive-keyring && @@ -3120,7 +3172,8 @@ install_ubuntu_onedir_deps() { if [ "${_UPGRADE_SYS}" -eq $BS_TRUE ]; then if [ "${_INSECURE_DL}" -eq $BS_TRUE ]; then ## apt-key is deprecated - if [ "$DISTRO_MAJOR_VERSION" -ge 20 ] || [ "$DISTRO_MAJOR_VERSION" -ge 21 ] || [ "$DISTRO_MAJOR_VERSION" -ge 22 ] || [ "$DISTRO_MAJOR_VERSION" -ge 23 ] || [ "$DISTRO_MAJOR_VERSION" -ge 24 ]; then + ## DGM if [ "$DISTRO_MAJOR_VERSION" -ge 20 ] || [ "$DISTRO_MAJOR_VERSION" -ge 21 ] || [ "$DISTRO_MAJOR_VERSION" -ge 22 ] || [ "$DISTRO_MAJOR_VERSION" -ge 23 ] || [ "$DISTRO_MAJOR_VERSION" -ge 24 ]; then + if [ "$DISTRO_MAJOR_VERSION" -ge 20 ]; then __apt_get_install_noinput --allow-unauthenticated debian-archive-keyring && apt-get update || return 1 else __apt_get_install_noinput --allow-unauthenticated debian-archive-keyring && @@ -3391,8 +3444,8 @@ install_ubuntu_check_services() { __install_saltstack_debian_repository() { echodebug "__install_saltstack_debian_repository() entry" - DEBIAN_RELEASE="$DISTRO_MAJOR_VERSION" - DEBIAN_CODENAME="$DISTRO_CODENAME" + ## DGM DEBIAN_RELEASE="$DISTRO_MAJOR_VERSION" + ## DGM DEBIAN_CODENAME="$DISTRO_CODENAME" if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -ne 3 ]; then echoerror "Python version is no longer supported, only Python 3" @@ -3415,19 +3468,35 @@ __install_saltstack_debian_repository() { # shellcheck disable=SC2086,SC2090 __apt_get_install_noinput ${__PACKAGES} || return 1 - SALTSTACK_DEBIAN_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/debian/${DEBIAN_RELEASE}/${__REPO_ARCH}/${STABLE_REV}" - echo "$__REPO_ARCH_DEB $SALTSTACK_DEBIAN_URL $DEBIAN_CODENAME main" > "/etc/apt/sources.list.d/salt.list" - - __apt_key_fetch "$SALTSTACK_DEBIAN_URL/SALT-PROJECT-GPG-PUBKEY-2023.gpg" || return 1 + ## SALTSTACK_DEBIAN_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/debian/${DEBIAN_RELEASE}/${__REPO_ARCH}/${STABLE_REV}" + ## echo "$__REPO_ARCH_DEB $SALTSTACK_DEBIAN_URL $DEBIAN_CODENAME main" > "/etc/apt/sources.list.d/salt.list" + ## __apt_key_fetch "$SALTSTACK_DEBIAN_URL/SALT-PROJECT-GPG-PUBKEY-2023.gpg" || return 1 + ## __wait_for_apt apt-get update || return 1 + __fetch_url "/etc/apt/sources.list.d/salt.sources" "https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.sources" + __apt_key_fetch "${HTTP_VAL}://${_REPO_URL}//api/security/keypair/SaltProjectKey/public" || return 1 __wait_for_apt apt-get update || return 1 + + if [ "$STABLE_REV" != "latest" ]; then + # latest is default + STABLE_REV_MAJOR=$(echo "$STABLE_REV" | cut -d '.' -f 1) + if [ "$STABLE_REV_MAJOR" -eq "3006" ]; then + echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 + echo "Pin: version 3006.*" >> /etc/apt/preferences.d/salt-pin-1001 + echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 + elif [ "$STABLE_REV_MAJOR" -eq "3007" ]; then + echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 + echo "Pin: version 3007.*" >> /etc/apt/preferences.d/salt-pin-1001 + echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 + fi + fi } __install_saltstack_debian_onedir_repository() { echodebug "__install_saltstack_debian_onedir_repository() entry" - DEBIAN_RELEASE="$DISTRO_MAJOR_VERSION" - DEBIAN_CODENAME="$DISTRO_CODENAME" + ## DGM DEBIAN_RELEASE="$DISTRO_MAJOR_VERSION" + ## DGM DEBIAN_CODENAME="$DISTRO_CODENAME" if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -ne 3 ]; then echoerror "Python version is no longer supported, only Python 3" @@ -3451,15 +3520,31 @@ __install_saltstack_debian_onedir_repository() { __apt_get_install_noinput ${__PACKAGES} || return 1 # amd64 is just a part of repository URI - SALTSTACK_DEBIAN_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/debian/${DEBIAN_RELEASE}/${__REPO_ARCH}/${ONEDIR_REV}" - if [ "${ONEDIR_REV}" = "nightly" ] ; then - SALTSTACK_DEBIAN_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/debian/${DEBIAN_RELEASE}/${__REPO_ARCH}" - fi - echo "$__REPO_ARCH_DEB $SALTSTACK_DEBIAN_URL $DEBIAN_CODENAME main" > "/etc/apt/sources.list.d/salt.list" - - __apt_key_fetch "${SALTSTACK_DEBIAN_URL}/SALT-PROJECT-GPG-PUBKEY-2023.gpg" || return 1 - + ## DGM SALTSTACK_DEBIAN_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/debian/${DEBIAN_RELEASE}/${__REPO_ARCH}/${ONEDIR_REV}" + ## DGM if [ "${ONEDIR_REV}" = "nightly" ] ; then + ## DGM SALTSTACK_DEBIAN_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/debian/${DEBIAN_RELEASE}/${__REPO_ARCH}" + ## DGM fi + ## DGM echo "$__REPO_ARCH_DEB $SALTSTACK_DEBIAN_URL $DEBIAN_CODENAME main" > "/etc/apt/sources.list.d/salt.list" + ## DGM __apt_key_fetch "${SALTSTACK_DEBIAN_URL}/SALT-PROJECT-GPG-PUBKEY-2023.gpg" || return 1 + ## DGM __wait_for_apt apt-get update || return 1 + + __fetch_url "/etc/apt/sources.list.d/salt.sources" "https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.sources" + __apt_key_fetch "${HTTP_VAL}://${_REPO_URL}//api/security/keypair/SaltProjectKey/public" || return 1 __wait_for_apt apt-get update || return 1 + + if [ "$STABLE_REV" != "latest" ]; then + # latest is default + STABLE_REV_MAJOR=$(echo "$STABLE_REV" | cut -d '.' -f 1) + if [ "$STABLE_REV_MAJOR" -eq "3006" ]; then + echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 + echo "Pin: version 3006.*" >> /etc/apt/preferences.d/salt-pin-1001 + echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 + elif [ "$STABLE_REV_MAJOR" -eq "3007" ]; then + echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 + echo "Pin: version 3007.*" >> /etc/apt/preferences.d/salt-pin-1001 + echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 + fi + fi } install_debian_onedir_deps() { @@ -3766,23 +3851,38 @@ __install_saltstack_fedora_onedir_repository() { return 1 fi - __PY_VERSION_REPO="py3" + ## DGM __PY_VERSION_REPO="py3" - GPG_KEY="SALT-PROJECT-GPG-PUBKEY-2023.pub" + ## DGM GPG_KEY="SALT-PROJECT-GPG-PUBKEY-2023.pub" REPO_FILE="/etc/yum.repos.d/salt.repo" if [ ! -s "$REPO_FILE" ] || [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]; then - FETCH_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/fedora/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/${ONEDIR_REV}" - if [ "${ONEDIR_REV}" = "nightly" ] ; then - FETCH_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/fedora/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/" - fi - + ## DGM FETCH_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/fedora/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/${ONEDIR_REV}" + ## DGM if [ "${ONEDIR_REV}" = "nightly" ] ; then + ## DGM FETCH_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/fedora/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/" + ## DGM fi + ## DGM __fetch_url "${REPO_FILE}" "${FETCH_URL}.repo" + ## DGM __rpm_import_gpg "${FETCH_URL}/${GPG_KEY}" || return 1 + ## DGM yum clean metadata || return 1 + + FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" __fetch_url "${REPO_FILE}" "${FETCH_URL}.repo" + if [ "$REPO_REV" != "latest" ]; then + # 3006.x is default + REPO_REV_MAJOR=$(echo "$REPO_REV" | cut -d '.' -f 1) + if [ "$REPO_REV_MAJOR" -eq "3007" ]; then + # Enable the Salt 3007 STS repo + dnf config-manager --set-disable salt-repo-* + dnf config-manager --set-enabled salt-repo-3007-sts + fi + else + # Enable the Salt LATEST repo + dnf config-manager --set-disable salt-repo-* + dnf config-manager --set-enabled salt-repo-latest + fi + dnf clean expire-cache || return 1 - __rpm_import_gpg "${FETCH_URL}/${GPG_KEY}" || return 1 - - yum clean metadata || return 1 elif [ "$REPO_REV" != "latest" ]; then echowarn "salt.repo already exists, ignoring salt version argument." echowarn "Use -F (forced overwrite) to install $REPO_REV." @@ -4041,9 +4141,9 @@ install_fedora_onedir_post() { # __install_saltstack_rhel_onedir_repository() { if [ "$ITYPE" = "stable" ]; then - repo_rev="$ONEDIR_REV" + REPO_REV="$ONEDIR_REV" else - repo_rev="latest" + REPO_REV="latest" fi if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -ne 3 ]; then @@ -4051,46 +4151,64 @@ __install_saltstack_rhel_onedir_repository() { return 1 fi - # Avoid using '$releasever' variable for yum. - # Instead, this should work correctly on all RHEL variants. - base_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/redhat/${DISTRO_MAJOR_VERSION}/\$basearch/${ONEDIR_REV}/" - if [ "${ONEDIR_REV}" = "nightly" ] ; then - base_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/redhat/${DISTRO_MAJOR_VERSION}/\$basearch/" - fi - - gpg_key="SALT-PROJECT-GPG-PUBKEY-2023.pub" - - gpg_key_urls="" - for key in $gpg_key; do - gpg_key_urls=$(printf "${base_url}${key},%s" "$gpg_key_urls") - done - - repo_file="/etc/yum.repos.d/salt.repo" - - if [ ! -s "$repo_file" ] || [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]; then - cat <<_eof > "$repo_file" -[saltstack] -name=SaltStack ${repo_rev} Release Channel for RHEL/CentOS \$releasever -baseurl=${base_url} -skip_if_unavailable=True -gpgcheck=1 -gpgkey=${gpg_key_urls} -enabled=1 -enabled_metadata=1 -_eof - - fetch_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/redhat/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/${ONEDIR_REV}/" - if [ "${ONEDIR_REV}" = "nightly" ] ; then - fetch_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/redhat/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/" +## DGM # Avoid using '$releasever' variable for yum. +## DGM # Instead, this should work correctly on all RHEL variants. +## DGM base_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/redhat/${DISTRO_MAJOR_VERSION}/\$basearch/${ONEDIR_REV}/" +## DGM if [ "${ONEDIR_REV}" = "nightly" ] ; then +## DGM base_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/redhat/${DISTRO_MAJOR_VERSION}/\$basearch/" +## DGM fi +## DGM +## DGM ## gpg_key="SALT-PROJECT-GPG-PUBKEY-2023.pub" +## DGM +## DGM gpg_key_urls="" +## DGM for key in $gpg_key; do +## DGM gpg_key_urls=$(printf "${base_url}${key},%s" "$gpg_key_urls") +## DGM done +## DGM +## DGM repo_file="/etc/yum.repos.d/salt.repo" +## DGM +## DGM if [ ! -s "$repo_file" ] || [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]; then +## DGM cat <<_eof > "$repo_file" +## DGM [saltstack] +## DGM name=SaltStack ${repo_rev} Release Channel for RHEL/CentOS \$releasever +## DGM baseurl=${base_url} +## DGM skip_if_unavailable=True +## DGM gpgcheck=1 +## DGM gpgkey=${gpg_key_urls} +## DGM enabled=1 +## DGM enabled_metadata=1 +## DGM _eof +## DGM +## DGM fetch_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/redhat/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/${ONEDIR_REV}/" +## DGM if [ "${ONEDIR_REV}" = "nightly" ] ; then +## DGM fetch_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/redhat/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/" +## DGM fi +## DGM for key in $gpg_key; do +## DGM __rpm_import_gpg "${fetch_url}${key}" || return 1 +## DGM done +## DGM +## DGM yum clean metadata || return 1 + if [ ! -s "$REPO_FILE" ] || [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]; then + REPO_FILE="/etc/yum.repos.d/salt.repo" + FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" + __fetch_url "${REPO_FILE}" "${FETCH_URL}.repo" + if [ "$REPO_REV" != "latest" ]; then + # 3006.x is default + REPO_REV_MAJOR=$(echo "$REPO_REV" | cut -d '.' -f 1) + if [ "$REPO_REV_MAJOR" -eq "3007" ]; then + # Enable the Salt 3007 STS repo + dnf config-manager --set-disable salt-repo-* + dnf config-manager --set-enabled salt-repo-3007-sts + fi + else + # Enable the Salt LATEST repo + dnf config-manager --set-disable salt-repo-* + dnf config-manager --set-enabled salt-repo-latest fi - for key in $gpg_key; do - __rpm_import_gpg "${fetch_url}${key}" || return 1 - done - - yum clean metadata || return 1 - elif [ "$repo_rev" != "latest" ]; then + dnf clean expire-cache || return 1 + elif [ "$REPO_REV" != "latest" ]; then echowarn "salt.repo already exists, ignoring salt version argument." - echowarn "Use -F (forced overwrite) to install $repo_rev." + echowarn "Use -F (forced overwrite) to install $REPO_REV." fi return 0 @@ -5376,9 +5494,9 @@ install_amazon_linux_ami_2_deps() { fi if [ "$ITYPE" = "stable" ]; then - repo_rev="$STABLE_REV" + REPO_REV="$STABLE_REV" else - repo_rev="latest" + REPO_REV="latest" fi # We need to install yum-utils before doing anything else when installing on @@ -5390,31 +5508,51 @@ install_amazon_linux_ami_2_deps() { yum -y update || return 1 fi +## DGM if [ $_DISABLE_REPOS -eq $BS_FALSE ] || [ "$_CUSTOM_REPO_URL" != "null" ]; then +## DGM __REPO_FILENAME="salt.repo" +## DGM PY_PKG_VER=3 +## DGM __PY_VERSION_REPO="py3" +## DGM repo_label="saltstack-py3-repo" +## DGM repo_name="SaltStack Python 3 repo for Amazon Linux 2" +## DGM +## DGM base_url="$HTTP_VAL://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/amazon/2/\$basearch/$repo_rev/" +## DGM gpg_key="${base_url}SALT-PROJECT-GPG-PUBKEY-2023.gpg" +## DGM +## DGM # This should prob be refactored to use __install_saltstack_rhel_onedir_repository() +## DGM # With args passed in to do the right thing. Reformatted to be more like the +## DGM # amazon linux yum file. +## DGM if [ ! -s "/etc/yum.repos.d/${__REPO_FILENAME}" ]; then +## DGM cat <<_eof > "/etc/yum.repos.d/${__REPO_FILENAME}" +## DGM [$repo_label] +## DGM name=$repo_name +## DGM failovermethod=priority +## DGM priority=10 +## DGM gpgcheck=1 +## DGM gpgkey=$gpg_key +## DGM baseurl=$base_url +## DGM _eof +## DGM fi + if [ $_DISABLE_REPOS -eq $BS_FALSE ] || [ "$_CUSTOM_REPO_URL" != "null" ]; then - __REPO_FILENAME="salt.repo" - PY_PKG_VER=3 - __PY_VERSION_REPO="py3" - repo_label="saltstack-py3-repo" - repo_name="SaltStack Python 3 repo for Amazon Linux 2" - - base_url="$HTTP_VAL://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/amazon/2/\$basearch/$repo_rev/" - gpg_key="${base_url}SALT-PROJECT-GPG-PUBKEY-2023.gpg" - - # This should prob be refactored to use __install_saltstack_rhel_onedir_repository() - # With args passed in to do the right thing. Reformatted to be more like the - # amazon linux yum file. - if [ ! -s "/etc/yum.repos.d/${__REPO_FILENAME}" ]; then - cat <<_eof > "/etc/yum.repos.d/${__REPO_FILENAME}" -[$repo_label] -name=$repo_name -failovermethod=priority -priority=10 -gpgcheck=1 -gpgkey=$gpg_key -baseurl=$base_url -_eof + REPO_FILE="/etc/yum.repos.d/salt.repo" + if [ ! -s "${REPO_FILE}" ]; then + FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" + __fetch_url "${REPO_FILE}" "${FETCH_URL}.repo" + if [ "$REPO_REV" != "latest" ]; then + # 3006.x is default + REPO_REV_MAJOR=$(echo "$REPO_REV" | cut -d '.' -f 1) + if [ "$REPO_REV_MAJOR" -eq "3007" ]; then + # Enable the Salt 3007 STS repo + dnf config-manager --set-disable salt-repo-* + dnf config-manager --set-enabled salt-repo-3007-sts + fi + else + # Enable the Salt LATEST repo + dnf config-manager --set-disable salt-repo-* + dnf config-manager --set-enabled salt-repo-latest + fi + dnf clean expire-cache || return 1 fi - fi if [ "${_EXTRA_PACKAGES}" != "" ]; then @@ -5431,9 +5569,9 @@ install_amazon_linux_ami_2_onedir_deps() { fi if [ "$ITYPE" = "onedir" ]; then - repo_rev="$ONEDIR_REV" + REPO_REV="$ONEDIR_REV" else - repo_rev="latest" + REPO_REV="latest" fi # We need to install yum-utils before doing anything else when installing on @@ -5445,34 +5583,56 @@ install_amazon_linux_ami_2_onedir_deps() { yum -y update || return 1 fi - if [ $_DISABLE_REPOS -eq $BS_FALSE ] || [ "$_CUSTOM_REPO_URL" != "null" ]; then - __REPO_FILENAME="salt.repo" - __PY_VERSION_REPO="py3" - PY_PKG_VER=3 - repo_label="saltstack-py3-repo" - repo_name="SaltStack Python 3 repo for Amazon Linux 2" - fi - - base_url="$HTTP_VAL://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/amazon/2/\$basearch/$repo_rev/" - if [ "${ONEDIR_REV}" = "nightly" ] ; then - base_url="$HTTP_VAL://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/amazon/2/\$basearch/" - fi +## DGM if [ $_DISABLE_REPOS -eq $BS_FALSE ] || [ "$_CUSTOM_REPO_URL" != "null" ]; then +## DGM __REPO_FILENAME="salt.repo" +## DGM __PY_VERSION_REPO="py3" +## DGM PY_PKG_VER=3 +## DGM repo_label="saltstack-py3-repo" +## DGM repo_name="SaltStack Python 3 repo for Amazon Linux 2" +## DGM fi +## DGM +## DGM base_url="$HTTP_VAL://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/amazon/2/\$basearch/$repo_rev/" +## DGM if [ "${ONEDIR_REV}" = "nightly" ] ; then +## DGM base_url="$HTTP_VAL://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/amazon/2/\$basearch/" +## DGM fi +## DGM +## DGM gpg_key="${base_url}SALT-PROJECT-GPG-PUBKEY-2023.pub" +## DGM +## DGM # With args passed in to do the right thing. Reformatted to be more like the +## DGM # amazon linux yum file. +## DGM if [ ! -s "/etc/yum.repos.d/${__REPO_FILENAME}" ]; then +## DGM cat <<_eof > "/etc/yum.repos.d/${__REPO_FILENAME}" +## DGM [$repo_label] +## DGM name=$repo_name +## DGM failovermethod=priority +## DGM priority=10 +## DGM gpgcheck=1 +## DGM gpgkey=$gpg_key +## DGM baseurl=$base_url +## DGM _eof +## DGM fi - gpg_key="${base_url}SALT-PROJECT-GPG-PUBKEY-2023.pub" - - # With args passed in to do the right thing. Reformatted to be more like the - # amazon linux yum file. - if [ ! -s "/etc/yum.repos.d/${__REPO_FILENAME}" ]; then - cat <<_eof > "/etc/yum.repos.d/${__REPO_FILENAME}" -[$repo_label] -name=$repo_name -failovermethod=priority -priority=10 -gpgcheck=1 -gpgkey=$gpg_key -baseurl=$base_url -_eof + if [ $_DISABLE_REPOS -eq $BS_FALSE ] || [ "$_CUSTOM_REPO_URL" != "null" ]; then + REPO_FILE="/etc/yum.repos.d/salt.repo" + if [ ! -s "${REPO_FILE}" ]; then + FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" + __fetch_url "${REPO_FILE}" "${FETCH_URL}.repo" + if [ "$REPO_REV" != "latest" ]; then + # 3006.x is default + REPO_REV_MAJOR=$(echo "$REPO_REV" | cut -d '.' -f 1) + if [ "$REPO_REV_MAJOR" -eq "3007" ]; then + # Enable the Salt 3007 STS repo + dnf config-manager --set-disable salt-repo-* + dnf config-manager --set-enabled salt-repo-3007-sts + fi + else + # Enable the Salt LATEST repo + dnf config-manager --set-disable salt-repo-* + dnf config-manager --set-enabled salt-repo-latest + fi + dnf clean expire-cache || return 1 fi + fi if [ "${_EXTRA_PACKAGES}" != "" ]; then echoinfo "Installing the following extra packages as requested: ${_EXTRA_PACKAGES}" @@ -5571,9 +5731,9 @@ install_amazon_linux_ami_2023_git_deps() { install_amazon_linux_ami_2023_onedir_deps() { if [ "$ITYPE" = "onedir" ]; then - repo_rev="$ONEDIR_REV" + REPO_REV="$ONEDIR_REV" else - repo_rev="latest" + REPO_REV="latest" fi # We need to install yum-utils before doing anything else when installing on @@ -5585,33 +5745,54 @@ install_amazon_linux_ami_2023_onedir_deps() { yum -y update || return 1 fi - if [ "$_DISABLE_REPOS" -eq $BS_FALSE ] || [ "$_CUSTOM_REPO_URL" != "null" ]; then - __REPO_FILENAME="salt.repo" - __PY_VERSION_REPO="py3" - PY_PKG_VER=3 - repo_label="saltstack-py3-repo" - repo_name="SaltStack Python 3 repo for Amazon Linux 2023" - - base_url="$HTTP_VAL://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/amazon/2023/\$basearch/$repo_rev/" - if [ "${ONEDIR_REV}" = "nightly" ] ; then - base_url="$HTTP_VAL://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/amazon/2023/\$basearch/" - fi - - gpg_key="${base_url}SALT-PROJECT-GPG-PUBKEY-2023.pub" - - # This should prob be refactored to use __install_saltstack_rhel_onedir_repository() - # With args passed in to do the right thing. Reformatted to be more like the - # amazon linux yum file. - if [ ! -s "/etc/yum.repos.d/${__REPO_FILENAME}" ]; then - cat <<_eof > "/etc/yum.repos.d/${__REPO_FILENAME}" -[$repo_label] -name=$repo_name -failovermethod=priority -priority=10 -gpgcheck=1 -gpgkey=$gpg_key -baseurl=$base_url -_eof +## DGM if [ "$_DISABLE_REPOS" -eq $BS_FALSE ] || [ "$_CUSTOM_REPO_URL" != "null" ]; then +## DGM __REPO_FILENAME="salt.repo" +## DGM __PY_VERSION_REPO="py3" +## DGM PY_PKG_VER=3 +## DGM repo_label="saltstack-py3-repo" +## DGM repo_name="SaltStack Python 3 repo for Amazon Linux 2023" +## DGM +## DGM base_url="$HTTP_VAL://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/amazon/2023/\$basearch/$repo_rev/" +## DGM if [ "${ONEDIR_REV}" = "nightly" ] ; then +## DGM base_url="$HTTP_VAL://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/amazon/2023/\$basearch/" +## DGM fi +## DGM +## DGM gpg_key="${base_url}SALT-PROJECT-GPG-PUBKEY-2023.pub" +## DGM +## DGM # This should prob be refactored to use __install_saltstack_rhel_onedir_repository() +## DGM # With args passed in to do the right thing. Reformatted to be more like the +## DGM # amazon linux yum file. +## DGM if [ ! -s "/etc/yum.repos.d/${__REPO_FILENAME}" ]; then +## DGM cat <<_eof > "/etc/yum.repos.d/${__REPO_FILENAME}" +## DGM [$repo_label] +## DGM name=$repo_name +## DGM failovermethod=priority +## DGM priority=10 +## DGM gpgcheck=1 +## DGM gpgkey=$gpg_key +## DGM baseurl=$base_url +## DGM _eof +## DGM fi +## DGM fi + if [ $_DISABLE_REPOS -eq $BS_FALSE ] || [ "$_CUSTOM_REPO_URL" != "null" ]; then + REPO_FILE="/etc/yum.repos.d/salt.repo" + if [ ! -s "${REPO_FILE}" ]; then + FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" + __fetch_url "${REPO_FILE}" "${FETCH_URL}.repo" + if [ "$REPO_REV" != "latest" ]; then + # 3006.x is default + REPO_REV_MAJOR=$(echo "$REPO_REV" | cut -d '.' -f 1) + if [ "$REPO_REV_MAJOR" -eq "3007" ]; then + # Enable the Salt 3007 STS repo + dnf config-manager --set-disable salt-repo-* + dnf config-manager --set-enabled salt-repo-3007-sts + fi + else + # Enable the Salt LATEST repo + dnf config-manager --set-disable salt-repo-* + dnf config-manager --set-enabled salt-repo-latest + fi + dnf clean expire-cache || return 1 fi fi @@ -5942,22 +6123,34 @@ __install_saltstack_photon_onedir_repository() { REPO_REV="latest" fi - __PY_VERSION_REPO="py3" + ## DGM __PY_VERSION_REPO="py3" REPO_FILE="/etc/yum.repos.d/salt.repo" if [ ! -s "$REPO_FILE" ] || [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]; then - ## salt repo 4 & 5 have issues, need the Major version dot Zero, eg: 4.0, 5.0 - FETCH_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/photon/${DISTRO_MAJOR_VERSION}.0/${CPU_ARCH_L}/${ONEDIR_REV}" - if [ "${ONEDIR_REV}" = "nightly" ] ; then - FETCH_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/photon/${DISTRO_MAJOR_VERSION}.0/${CPU_ARCH_L}/" - fi - + ## DGM ## salt repo 4 & 5 have issues, need the Major version dot Zero, eg: 4.0, 5.0 + ## DGM FETCH_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/photon/${DISTRO_MAJOR_VERSION}.0/${CPU_ARCH_L}/${ONEDIR_REV}" + ## DGM if [ "${ONEDIR_REV}" = "nightly" ] ; then + ## DGM FETCH_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/photon/${DISTRO_MAJOR_VERSION}.0/${CPU_ARCH_L}/" + ## DGM fi + ## DGM __fetch_url "${REPO_FILE}" "${FETCH_URL}.repo" + ## DGM GPG_KEY="SALT-PROJECT-GPG-PUBKEY-2023.pub" + ## DGM __rpm_import_gpg "${FETCH_URL}/${GPG_KEY}" || return 1 + + FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" __fetch_url "${REPO_FILE}" "${FETCH_URL}.repo" - - GPG_KEY="SALT-PROJECT-GPG-PUBKEY-2023.pub" - - __rpm_import_gpg "${FETCH_URL}/${GPG_KEY}" || return 1 - + if [ "$REPO_REV" != "latest" ]; then + # 3006.x is default + REPO_REV_MAJOR=$(echo "$REPO_REV" | cut -d '.' -f 1) + if [ "$REPO_REV_MAJOR" -eq "3007" ]; then + # Enable the Salt 3007 STS repo + dnf config-manager --set-disable salt-repo-* + dnf config-manager --set-enabled salt-repo-3007-sts + fi + else + # Enable the Salt LATEST repo + dnf config-manager --set-disable salt-repo-* + dnf config-manager --set-enabled salt-repo-latest + fi tdnf makecache || return 1 elif [ "$REPO_REV" != "latest" ]; then echowarn "salt.repo already exists, ignoring salt version argument." @@ -7132,22 +7325,43 @@ daemons_running_voidlinux() { # OS X / Darwin Install Functions # -__parse_repo_json_python() { - - # Using latest, grab the right - # version from the repo.json - _JSON_VERSION=$(python - <<-EOF -import json, urllib.request -url = "https://repo.saltproject.io/salt/py3/macos/repo.json" -response = urllib.request.urlopen(url) -data = json.loads(response.read()) -version = data["${_ONEDIR_REV}"][list(data["${_ONEDIR_REV}"])[0]]['version'] -print(version) -EOF -) -echo "${_JSON_VERSION}" +## DGM __parse_repo_json_python() { +## DGM +## DGM # Using latest, grab the right +## DGM # version from the repo.json +## DGM _JSON_VERSION=$(python - <<-EOF +## DGM import json, urllib.request +## DGM ## DGM url = "https://repo.saltproject.io/salt/py3/macos/repo.json" ## DGM note the use of repo.json here TBD handling this +## DGM url = "https://packages.broadcom.com/ui/repos/tree/General/saltproject-generic/macos" +## DGM response = urllib.request.urlopen(url) +## DGM +## DGM ## DGM No json file to process here TBD handling it +## DGM data = json.loads(response.read()) +## DGM version = data["${_ONEDIR_REV}"][list(data["${_ONEDIR_REV}"])[0]]['version'] +## DGM print(version) +## DGM EOF +## DGM ) +## DGM echo "${_JSON_VERSION}" +## DGM } + +__macosx_get_packagesite_onedir_latest() { + echodebug "Find latest MacOS release from repository" + + # get dir listing from url, sort and pick highest + macos_versions_tmpf=$(mktemp) + curr_pwd=$(pwd) + cd ${macos_versions_tmpf} || return 1 + wget -r -np -nH --exclude-directories=onedir,relenv,windows -x -l 1 "$SALT_MACOS_PKGDIR_URL/" + # shellcheck disable=SC2010 + LATEST_VERSION=$(ls artifactory/saltproject-generic/macos/ | grep -v 'index.html' | sort -V -u | tail -n 1) + cd ${curr_pwd} || return "${LATEST_VERSION}" + rm -fR ${macos_versions_tmpf} + + echodebug "latest MacOS release from repository found ${LATEST_VERSION}" + return "${LATEST_VERSION}" } + __macosx_get_packagesite_onedir() { if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -ne 3 ]; then echoerror "Python version is no longer supported, only Python 3" @@ -7156,16 +7370,35 @@ __macosx_get_packagesite_onedir() { DARWIN_ARCH=${CPU_ARCH_L} +## DGM if [ "$(echo "$_ONEDIR_REV" | grep -E '^(latest)$')" != "" ]; then +## DGM ## DGM TBD what to do here +## DGM _PKG_VERSION=$(__parse_repo_json_python) +## DGM elif [ "$(echo "$_ONEDIR_REV" | grep -E '^([3-9][0-9]{3}(\.[0-9]*))')" != "" ]; then +## DGM _PKG_VERSION=$_ONEDIR_REV +## DGM else +## DGM ## DGM TBD what to do here +## DGM _PKG_VERSION=$(__parse_repo_json_python) +## DGM fi +## DGM +## DGM ## DGM PKG="salt-${_PKG_VERSION}-${__PY_VERSION_REPO}-${DARWIN_ARCH}.pkg" +## DGM ## DGM SALTPKGCONFURL="https://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/macos/${ONEDIR_REV}/${PKG}" +## DGM _ONEDIR_TYPE="saltproject-generic" +## DGM PKG="salt-${ONEDIR_REV}-py3-${DARWIN_ARCH}.pkg" +## DGM SALTPKGCONFURL="https://${_REPO_URL}/${_ONEDIR_TYPE}/macos/${ONEDIR_REV}/${PKG}" + + _ONEDIR_TYPE="saltproject-generic" + SALT_MACOS_PKGDIR_URL="https://${_REPO_URL}/${_ONEDIR_TYPE}/macos" if [ "$(echo "$_ONEDIR_REV" | grep -E '^(latest)$')" != "" ]; then - _PKG_VERSION=$(__parse_repo_json_python) + _PKG_VERSION=$(__macosx_get_packagesite_onedir_latest) elif [ "$(echo "$_ONEDIR_REV" | grep -E '^([3-9][0-9]{3}(\.[0-9]*))')" != "" ]; then _PKG_VERSION=$_ONEDIR_REV else - _PKG_VERSION=$(__parse_repo_json_python) + _PKG_VERSION=$(__macosx_get_packagesite_onedir_latest) fi - PKG="salt-${_PKG_VERSION}-${__PY_VERSION_REPO}-${DARWIN_ARCH}.pkg" - SALTPKGCONFURL="https://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/macos/${ONEDIR_REV}/${PKG}" + PKG="salt-${_PKG_VERSION}-py3-${DARWIN_ARCH}.pkg" + SALTPKGCONFURL="${SALT_MACOS_PKGDIR_URL}/${ONEDIR_REV}/${PKG}" + } __configure_macosx_pkg_details_onedir() { From 0899e72c34958da284c29fd71eeddb52cc5abe9f Mon Sep 17 00:00:00 2001 From: David Murphy Date: Fri, 1 Nov 2024 15:11:22 -0600 Subject: [PATCH 002/173] Disabling Kitchen tests, and updated quick-start --- .github/workflows/test-linux.yml | 28 +++++++++++++------------- .github/workflows/test-macos.yml | 28 +++++++++++++------------- .github/workflows/test-windows.yml | 32 +++++++++++++++--------------- salt-quick-start.sh | 25 +++++++++++++++++++---- 4 files changed, 65 insertions(+), 48 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index c15a5bc30..b224e0eb3 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -54,22 +54,22 @@ jobs: python3 -m pip install -U pip python3 -m pip install -r tests/requirements.txt - - name: Create Test Instance - run: | - bundle exec kitchen create ${{ matrix.instance }}-${{ inputs.distro-slug }} || \ - (sleep 10 && bundle exec kitchen create ${{ matrix.instance }}-${{ inputs.distro-slug }}) - sleep 2 + ## DGM - name: Create Test Instance + ## DGM run: | + ## DGM bundle exec kitchen create ${{ matrix.instance }}-${{ inputs.distro-slug }} || \ + ## DGM (sleep 10 && bundle exec kitchen create ${{ matrix.instance }}-${{ inputs.distro-slug }}) + ## DGM sleep 2 - - name: Test Bootstrap - run: | - bundle exec kitchen verify ${{ matrix.instance }}-${{ inputs.distro-slug }} - sleep 2 + ## DGM - name: Test Bootstrap + ## DGM run: | + ## DGM bundle exec kitchen verify ${{ matrix.instance }}-${{ inputs.distro-slug }} + ## DGM sleep 2 - - name: Destroy Test Instance - if: always() - run: | - bundle exec kitchen destroy ${{ matrix.instance }}-${{ inputs.distro-slug }} - sleep 2 + ## DGM - name: Destroy Test Instance + ## DGM if: always() + ## DGM run: | + ## DGM bundle exec kitchen destroy ${{ matrix.instance }}-${{ inputs.distro-slug }} + ## DGM sleep 2 - name: Set Exit Status if: always() diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index ccabff32a..113e2063c 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -61,22 +61,22 @@ jobs: pip install -U pip pip install -r tests/requirements.txt - - name: Create Test Instance - run: | - bundle exec kitchen create ${{ matrix.instance }}-${{ inputs.distro-slug }} || \ - (sleep 10 && bundle exec kitchen create ${{ matrix.instance }}-${{ inputs.distro-slug }}) - sleep 2 + ## DGM - name: Create Test Instance + ## DGM run: | + ## DGM bundle exec kitchen create ${{ matrix.instance }}-${{ inputs.distro-slug }} || \ + ## DGM (sleep 10 && bundle exec kitchen create ${{ matrix.instance }}-${{ inputs.distro-slug }}) + ## DGM sleep 2 - - name: Test Bootstrap - run: | - bundle exec kitchen verify ${{ matrix.instance }}-${{ inputs.distro-slug }} - sleep 2 + ## DGM - name: Test Bootstrap + ## DGM run: | + ## DGM bundle exec kitchen verify ${{ matrix.instance }}-${{ inputs.distro-slug }} + ## DGM sleep 2 - - name: Destroy Test Instance - if: always() - run: | - bundle exec kitchen destroy ${{ matrix.instance }}-${{ inputs.distro-slug }} - sleep 2 + ## DGM - name: Destroy Test Instance + ## DGM if: always() + ## DGM run: | + ## DGM bundle exec kitchen destroy ${{ matrix.instance }}-${{ inputs.distro-slug }} + ## DGM sleep 2 - name: Set Exit Status if: always() diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index 25d300ce8..8a6bfe4ec 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -90,22 +90,22 @@ jobs: pip install -U pip pip install -r tests/requirements.txt - - name: Create Test Instance - run: | - bundle exec kitchen create ${{ matrix.instance }}-${{ inputs.distro-slug }} - sleep 2 - - - name: Test Bootstrap - run: | - env - bundle exec kitchen verify ${{ matrix.instance }}-${{ inputs.distro-slug }} - sleep 2 - - - name: Destroy Test Instance - if: always() - run: | - bundle exec kitchen destroy ${{ matrix.instance }}-${{ inputs.distro-slug }} - sleep 2 + ## DGM - name: Create Test Instance + ## DGM run: | + ## DGM bundle exec kitchen create ${{ matrix.instance }}-${{ inputs.distro-slug }} + ## DGM sleep 2 + + ## DGM - name: Test Bootstrap + ## DGM run: | + ## DGM env + ## DGM bundle exec kitchen verify ${{ matrix.instance }}-${{ inputs.distro-slug }} + ## DGM sleep 2 + + ## DGM - name: Destroy Test Instance + ## DGM if: always() + ## DGM run: | + ## DGM bundle exec kitchen destroy ${{ matrix.instance }}-${{ inputs.distro-slug }} + ## DGM sleep 2 - name: Set Exit Status if: always() diff --git a/salt-quick-start.sh b/salt-quick-start.sh index 8449abbf0..9be9e7afe 100755 --- a/salt-quick-start.sh +++ b/salt-quick-start.sh @@ -1,7 +1,8 @@ #!/bin/sh __ScriptName="salt-quick-start.sh" -SALT_REPO_URL="https://repo.saltproject.io/salt/py3/onedir" +## DGM SALT_REPO_URL="https://repo.saltproject.io/salt/py3/onedir" +SALT_REPO_URL="https://packages.broadcom.com/artifactory/salt-project-generic/onedir" _COLORS=${QS_COLORS:-$(tput colors 2>/dev/null || echo 0)} _LOCAL=0 @@ -89,8 +90,23 @@ if [[ "$_LOCAL" == "1" && "$_FULL" == "1" ]]; then fi __parse_repo_json_jq() { - _JSON_FILE="${SALT_REPO_URL}/repo.json" - _JSON_VERSION=$(curl -s ${_JSON_FILE} | jq -sr ".[].latest[] | select(.os == \"$1\") | select(.arch == \"$2\").version") + ## DGM TBD how to handle no repo.json + ## DGM _JSON_FILE="${SALT_REPO_URL}/repo.json" + ## DGM _JSON_VERSION=$(curl -s ${_JSON_FILE} | jq -sr ".[].latest[] | select(.os == \"$1\") | select(.arch == \"$2\").version") + + # $1 is OS_NAME + # $2 is ARCH + + # get dir listing from url, sort and pick highest + onedir_versions_tmpf=$(mktemp) + curr_pwd=$(pwd) + cd ${onedir_versions_tmpf} || return 1 + wget -r -np -nH --exclude-directories=onedir,relenv,windows -x -l 1 "$SALT_REPO_URL/" + # shellcheck disable=SC2010 + LATEST_VERSION=$(ls artifactory/saltproject-generic/onedir/ | grep -v 'index.html' | sort -V -u | tail -n 1) + cd ${curr_pwd} || return "${LATEST_VERSION}" + rm -fR ${onedir_versions_tmpf} + _JSON_VERSION="${LATEST_VERSION}" } __fetch_url() { @@ -146,7 +162,8 @@ fi __parse_repo_json_jq ${OS_NAME} ${CPU_ARCH_L} FILE="salt-${_JSON_VERSION}-onedir-${OS_NAME_L}-${CPU_ARCH_L}.tar.xz" -URL="${SALT_REPO_URL}/latest/${FILE}" +## DGM URL="${SALT_REPO_URL}/latest/${FILE}" +URL="${SALT_REPO_URL}/${_JSON_VERSION}/${FILE}" if [[ ! -f ${FILE} ]]; then echoinfo "Downloading Salt" From db974361f74e8e782f81c60a473f6adf051cbd56 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 4 Nov 2024 15:40:40 -0700 Subject: [PATCH 003/173] Initial Shell pytest --- .github/workflows/test-linux.yml | 40 ++++++---- .github/workflows/test-macos.yml | 47 +++++++----- .github/workflows/test-windows.yml | 100 +++++++++++++++---------- .gitignore | 1 + .pre-commit-config.yaml | 2 +- tests/conftest.py | 37 +-------- tests/integration/test_installation.py | 59 ++++++++++----- tests/requirements.txt | 12 +-- 8 files changed, 166 insertions(+), 132 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index b224e0eb3..5f49a6661 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -37,22 +37,22 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Setup Ruby - uses: ruby/setup-ruby@v1 - with: - ruby-version: 2.6.10 - ## ruby-version: 3.3.4 - bundler-cache: true + ## DGM - name: Setup Ruby + ## DGM uses: ruby/setup-ruby@v1 + ## DGM with: + ## DGM ruby-version: 2.6.10 + ## DGM ## ruby-version: 3.3.4 + ## DGM bundler-cache: true - - name: Set up Python 3.10 - uses: actions/setup-python@v5 - with: - python-version: "3.10" + ## DGM - name: Set up Python 3.10 + ## DGM uses: actions/setup-python@v5 + ## DGM with: + ## DGM python-version: "3.10" - - name: Install Python Dependencies - run: | - python3 -m pip install -U pip - python3 -m pip install -r tests/requirements.txt + ## DGM - name: Install Python Dependencies + ## DGM run: | + ## DGM python3 -m pip install -U pip + ## DGM python3 -m pip install -r tests/requirements.txt ## DGM - name: Create Test Instance ## DGM run: | @@ -71,6 +71,18 @@ jobs: ## DGM bundle exec kitchen destroy ${{ matrix.instance }}-${{ inputs.distro-slug }} ## DGM sleep 2 + - name: Install Pytest + run: | + pip install -U pytest + + - name: Bootstrap Salt + run: | + . ./bootstrap.sh + + - name: Test Bootstrap + run: | + pytest --cache-clear -v -s -ra --log-cli-level=debug tests/integration/ + - name: Set Exit Status if: always() run: | diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index 113e2063c..190ec2463 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -28,13 +28,14 @@ on: default: 20 description: The timeout(in minutes) for the workflow -env: - KITCHEN_LOCAL_YAML: 'kitchen.macos.yml' +## DGM env: +## DGM KITCHEN_LOCAL_YAML: 'kitchen.macos.yml' jobs: Test: name: ${{ matrix.instance }} - runs-on: ${{ inputs.runs-on }} + ## DGM runs-on: ${{ inputs.runs-on }} + runs-on: macos-12 timeout-minutes: ${{ inputs.timeout }} strategy: fail-fast: false @@ -44,22 +45,22 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Setup Ruby - uses: ruby/setup-ruby@v1 - with: - ruby-version: 2.6.10 - ## ruby-version: 3.3.4 - bundler-cache: true + ## DGM - name: Setup Ruby + ## DGM uses: ruby/setup-ruby@v1 + ## DGM with: + ## DGM ruby-version: 2.6.10 + ## DGM ## ruby-version: 3.3.4 + ## DGM bundler-cache: true - - name: Set up Python 3.10 - uses: actions/setup-python@v5 - with: - python-version: "3.10" + ## DGM - name: Set up Python 3.10 + ## DGM uses: actions/setup-python@v5 + ## DGM with: + ## DGM python-version: "3.10" - - name: Install Python Dependencies - run: | - pip install -U pip - pip install -r tests/requirements.txt + ## DGM - name: Install Python Dependencies + ## DGM run: | + ## DGM pip install -U pip + ## DGM pip install -r tests/requirements.txt ## DGM - name: Create Test Instance ## DGM run: | @@ -78,6 +79,18 @@ jobs: ## DGM bundle exec kitchen destroy ${{ matrix.instance }}-${{ inputs.distro-slug }} ## DGM sleep 2 + - name: Install Pytest + run: | + pip install -U pytest + + - name: Bootstrap Salt + run: | + . ./bootstrap.sh + + - name: Test Bootstrap + run: | + pytest --cache-clear -v -s -ra --log-cli-level=debug tests/integration/ + - name: Set Exit Status if: always() run: | diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index 8a6bfe4ec..8613744eb 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -28,16 +28,17 @@ on: default: 20 description: The timeout(in minutes) for the workflow -env: - machine_user: kitchen - machine_pass: Password1 - machine_port: 5985 - KITCHEN_LOCAL_YAML: 'kitchen.windows.yml' +## DGM env: +## DGM machine_user: kitchen +## DGM machine_pass: Password1 +## DGM machine_port: 5985 +## DGM KITCHEN_LOCAL_YAML: 'kitchen.windows.yml' jobs: Test: name: ${{ matrix.instance }} - runs-on: ${{ inputs.runs-on }} + ## DGM runs-on: ${{ inputs.runs-on }} + runs-on: windows-latest timeout-minutes: ${{ inputs.timeout }} strategy: fail-fast: false @@ -47,48 +48,50 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Setup Ruby - uses: ruby/setup-ruby@v1 - with: - ruby-version: 2.6.10 - ## ruby-version: 3.3.4 - bundler-cache: true - - - name: Install Chef - uses: actionshub/chef-install@1.1.0 - with: - project: chef - version: 16.10.8 - - - name: Add Chef bindir to PATH - uses: myci-actions/export-env-var-powershell@1 - with: - name: PATH - value: "C:\\opscode\\chef\\bin;C:\\opscode\\chef\\embedded\\bin;$env:PATH" - - - name: Setup test user - run: | - $password = ConvertTo-SecureString $env:machine_pass -AsPlainText -Force - New-LocalUser $env:machine_user -Password $password - Add-LocalGroupMember -Group "Administrators" -Member $env:machine_user - Get-LocalUser - Get-LocalGroupMember -Group "Administrators" + ## DGM - name: Setup Ruby + ## DGM uses: ruby/setup-ruby@v1 + ## DGM with: + ## DGM ruby-version: 2.6.10 + ## DGM ## ruby-version: 3.3.4 + ## DGM bundler-cache: true + + ## DGM - name: Install Chef + ## DGM uses: actionshub/chef-install@1.1.0 + ## DGM with: + ## DGM project: chef + ## DGM version: 16.10.8 + + ## DGM - name: Add Chef bindir to PATH + ## DGM uses: myci-actions/export-env-var-powershell@1 + ## DGM with: + ## DGM name: PATH + ## DGM value: "C:\\opscode\\chef\\bin;C:\\opscode\\chef\\embedded\\bin;$env:PATH" + + ## DGM - name: Setup test user + ## DGM run: | + ## DGM $password = ConvertTo-SecureString $env:machine_pass -AsPlainText -Force + ## DGM New-LocalUser $env:machine_user -Password $password + ## DGM Add-LocalGroupMember -Group "Administrators" -Member $env:machine_user + ## DGM Get-LocalUser + ## DGM Get-LocalGroupMember -Group "Administrators" - - name: Set up WinRM - run: | - Set-WSManQuickConfig -Force - winrm set winrm/config/service '@{AllowUnencrypted="True"}' - env + ## DGM - name: Set up WinRM + ## DGM run: | + ## DGM Set-WSManQuickConfig -Force + ## DGM winrm set winrm/config/service '@{AllowUnencrypted="True"}' + ## DGM env - name: Set up Python 3.10 uses: actions/setup-python@v5 with: python-version: "3.10" - - name: Install Python Dependencies + ## DGM - name: Install Python Dependencies + - name: Install Pytest run: | - pip install -U pip - pip install -r tests/requirements.txt + ## DGM pip install -U pip + ## DGM pip install -r tests/requirements.txt + pip install -U pytest ## DGM - name: Create Test Instance ## DGM run: | @@ -107,6 +110,23 @@ jobs: ## DGM bundle exec kitchen destroy ${{ matrix.instance }}-${{ inputs.distro-slug }} ## DGM sleep 2 + - name: Bootstrap Salt + run: | + . ./bootstrap-salt.ps1 -RunService $false + + - name: Set Environment Variables + run: | + $env:Path = "$env:Path;C:\Program Files\Salt Project\Salt" + $env:Instance = ${{ matrix.instance }} + Write-Output "Path:" + Write-Output $env:Path + Write-Output "Environment Variables" + Get-ChildItem Env: + + - name: Test Bootstrap + run: | + pytest --cache-clear -v -s -ra --log-cli-level=debug tests/integration/ + - name: Set Exit Status if: always() run: | diff --git a/.gitignore b/.gitignore index 9b7ebc0b0..f1af41c03 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ *.sw? *.un~ .vagrant +*__pycache__ tools venv diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9fb6a2607..033ac67b4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -58,7 +58,7 @@ repos: args: [--py310-plus] - repo: https://github.com/psf/black - rev: 24.1.0 + rev: 24.10.0 hooks: - id: black args: [] diff --git a/tests/conftest.py b/tests/conftest.py index 864307c03..3779d377d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,45 +1,11 @@ import logging import os -import pprint import pytest -import testinfra log = logging.getLogger(__name__) -@pytest.fixture(scope="session") -def host(): - if ( - os.environ.get("RUNNER_OS", "") == "macOS" - and os.environ.get("KITCHEN_LOCAL_YAML", "") == "kitchen.macos.yml" - ): - # Adjust the `PATH` so that the `salt-call` executable can be found - os.environ["PATH"] = "/opt/salt/bin{}{}".format(os.pathsep, os.environ["PATH"]) - return testinfra.get_host("local://", sudo=True) - - if os.environ.get("KITCHEN_USERNAME") == "vagrant" or "windows" in os.environ.get( - "KITCHEN_INSTANCE" - ): - if "windows" in os.environ.get("KITCHEN_INSTANCE"): - _url = "winrm://{KITCHEN_USERNAME}:{KITCHEN_PASSWORD}@{KITCHEN_HOSTNAME}:{KITCHEN_PORT}".format( - **os.environ - ) - return testinfra.get_host( - _url, - no_ssl=True, - ) - return testinfra.get_host( - "paramiko://{KITCHEN_USERNAME}@{KITCHEN_HOSTNAME}:{KITCHEN_PORT}".format( - **os.environ - ), - ssh_identity_file=os.environ.get("KITCHEN_SSH_KEY"), - ) - return testinfra.get_host( - "docker://{KITCHEN_USERNAME}@{KITCHEN_CONTAINER_ID}".format(**os.environ) - ) - - @pytest.fixture(scope="session") def target_python_version(): return 3 @@ -53,7 +19,8 @@ def target_salt_version(): target_salt = ".".join( [ item - for item in os.environ["KITCHEN_SUITE"].split("-") + ## DGM for item in os.environ["KITCHEN_SUITE"].split("-") + for item in os.environ.get("KITCHEN_SUITE", "").split("-") if item not in bootstrap_types ] ) diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index 9ee18d49c..6623fc613 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -1,31 +1,50 @@ +import json import logging -import os -from contextlib import nullcontext +import platform +import subprocess + +import pytest +from Demoa.winc32cred_demo import target log = logging.getLogger(__name__) -def selected_context_manager(host): - if "windows" in os.environ.get("KITCHEN_INSTANCE"): - return nullcontext() - return host.sudo() +@pytest.fixture +def path(): + if platform.system() == "Windows": + salt_path = "C:\\Program Files\\Salt Project\\Salt" + if salt_path not in os.environ["path"]: + os.environ["path"] = f'{os.environ["path"]};{salt_path}' + yield os.environ["path"] + + +def run_salt_call(cmd): + """ + Runs salt call command and returns a dictionary + Accepts cmd as a list + """ + cmd.append("--out=json") + result = subprocess.run(cmd, capture_output=True, text=True) + json_data = json.loads(result.stdout) + return json_data["local"] -def test_ping(host): - with selected_context_manager(host): - assert host.salt("test.ping", "--timeout=120") +def test_ping(): + cmd = ["salt-call", "--local", "test.ping"] + result = run_salt_call(cmd) + assert result == True -def test_target_python_version(host, target_python_version): - with selected_context_manager(host): - ret = host.salt("grains.item", "pythonversion", "--timeout=120") - assert ret["pythonversion"][0] == target_python_version +def test_target_python_version(target_python_version): + cmd = ["salt-call", "--local", "grains.item", "pythonversion", "--timeout=120"] + result = run_salt_call(cmd) + # Returns: {'pythonversion': [3, 10, 11, 'final', 0]} + py_maj_ver = result["pythonversion"][0] + assert py_maj_ver == target_python_version -def test_target_salt_version(host, target_salt_version): - with selected_context_manager(host): - ret = host.salt("grains.item", "saltversion", "--timeout=120") - if target_salt_version.endswith(".0") or target_salt_version.endswith(".x"): - assert ret["saltversion"] == ".".join(target_salt_version.split(".")[:-1]) - else: - assert ret["saltversion"].startswith(target_salt_version) +def test_target_salt_version(target_salt_version): + cmd = ["salt-call", "--local", "grains.item", "saltversion", "--timeout=120"] + result = run_salt_call(cmd) + # Returns: {'saltversion': '3006.9+217.g53cfa53040'} + assert result["saltversion"] == target_salt_version diff --git a/tests/requirements.txt b/tests/requirements.txt index 21d133da8..64223821a 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,5 +1,7 @@ -pytest-testinfra -paramiko -requests-ntlm==1.1.0; sys.platform == 'win32' -pywinrm; sys.platform == 'win32' -six>=1.10.0 +## DGM pytest-testinfra +## DGM paramiko +## DGM requests-ntlm==1.1.0; sys.platform == 'win32' +## DGM pywinrm; sys.platform == 'win32' +## DGM six>=1.10.0 + +pytest From 239f979957d7a5aa4769b9ccef780cbb6c15c0f2 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 4 Nov 2024 15:47:59 -0700 Subject: [PATCH 004/173] Debug --- .github/workflows/test-linux.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 5f49a6661..7bebc54ff 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -77,7 +77,8 @@ jobs: - name: Bootstrap Salt run: | - . ./bootstrap.sh + ls -alh + ## . ./bootstrap.sh - name: Test Bootstrap run: | From b4a632180ebc826327f146cb09860fc4e1dc4961 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 4 Nov 2024 15:54:01 -0700 Subject: [PATCH 005/173] Further debugging --- .github/workflows/ci.yml | 4 ++-- .github/workflows/templates/ci.yml | 4 ++-- tests/integration/test_installation.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c13c6fffe..d8539b20a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,8 +27,8 @@ jobs: separator: "," files: | bootstrap-salt.* - Gemfile - kitchen*.yml + ## DGM Gemfile + ## DGM kitchen*.yml tests/** .github/workflows/** files_ignore: | diff --git a/.github/workflows/templates/ci.yml b/.github/workflows/templates/ci.yml index e9c6be7b9..7f99aa1b2 100644 --- a/.github/workflows/templates/ci.yml +++ b/.github/workflows/templates/ci.yml @@ -27,8 +27,8 @@ jobs: separator: "," files: | bootstrap-salt.* - Gemfile - kitchen*.yml + ## DGM Gemfile + ## DGM kitchen*.yml tests/** .github/workflows/** files_ignore: | diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index 6623fc613..8b92ec89a 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -4,7 +4,7 @@ import subprocess import pytest -from Demoa.winc32cred_demo import target +from Demos.win32cred_demo import target log = logging.getLogger(__name__) From 63bdbaf39260f58dce38f4e54d46d1a982f1a650 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 4 Nov 2024 16:03:56 -0700 Subject: [PATCH 006/173] Debugging --- tests/integration/test_installation.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index 8b92ec89a..3a65144fb 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -4,7 +4,8 @@ import subprocess import pytest -from Demos.win32cred_demo import target + +## from Demos.win32cred_demo import target log = logging.getLogger(__name__) From 892c7cbadc42e3d16ab0da56bdebff761d31254e Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 4 Nov 2024 16:13:39 -0700 Subject: [PATCH 007/173] Updated to using bootstrap-salt.sh --- .github/workflows/test-linux.yml | 3 +-- .github/workflows/test-macos.yml | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 7bebc54ff..31e4c7741 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -77,8 +77,7 @@ jobs: - name: Bootstrap Salt run: | - ls -alh - ## . ./bootstrap.sh + . ./bootstrap-salt.sh - name: Test Bootstrap run: | diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index 190ec2463..26dd94f7b 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -85,7 +85,7 @@ jobs: - name: Bootstrap Salt run: | - . ./bootstrap.sh + . ./bootstrap-salt.sh - name: Test Bootstrap run: | From 0828aa5e413d5e01cd6806305028cd943aff48b5 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 4 Nov 2024 16:31:53 -0700 Subject: [PATCH 008/173] Added debugging output --- .github/workflows/test-linux.yml | 2 +- .github/workflows/test-macos.yml | 2 +- bootstrap-salt.sh | 584 +++++++++++++++++++++++++++++++ 3 files changed, 586 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 31e4c7741..000074322 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -77,7 +77,7 @@ jobs: - name: Bootstrap Salt run: | - . ./bootstrap-salt.sh + sh -x ./bootstrap-salt.sh - name: Test Bootstrap run: | diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index 26dd94f7b..f82f15590 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -85,7 +85,7 @@ jobs: - name: Bootstrap Salt run: | - . ./bootstrap-salt.sh + sh -x ./bootstrap-salt.sh - name: Test Bootstrap run: | diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 89ceb7424..e808e2fab 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -903,6 +903,10 @@ __fetch_url() { # DESCRIPTION: Retrieves a URL, verifies its content and writes it to standard output #---------------------------------------------------------------------------------------------------------------------- __fetch_verify() { + # DGM debug + set -v + set -x + fetch_verify_url="$1" fetch_verify_sum="$2" fetch_verify_size="$3" @@ -924,6 +928,10 @@ __fetch_verify() { # DESCRIPTION: Checks if a URL exists #---------------------------------------------------------------------------------------------------------------------- __check_url_exists() { + # DGM debug + set -v + set -x + _URL="$1" if curl --output /dev/null --silent --fail "${_URL}"; then return 0 @@ -1936,6 +1944,10 @@ __wait_for_apt(){ # PARAMETERS: packages #---------------------------------------------------------------------------------------------------------------------- __apt_get_install_noinput() { + # DGM debug + set -v + set -x + __wait_for_apt apt-get install -y -o DPkg::Options::=--force-confold "${@}"; return $? } # ---------- end of function __apt_get_install_noinput ---------- @@ -1945,6 +1957,10 @@ __apt_get_install_noinput() { # DESCRIPTION: (DRY) apt-get upgrade with noinput options #---------------------------------------------------------------------------------------------------------------------- __apt_get_upgrade_noinput() { + # DGM debug + set -v + set -x + __wait_for_apt apt-get upgrade -y -o DPkg::Options::=--force-confold; return $? } # ---------- end of function __apt_get_upgrade_noinput ---------- @@ -1975,6 +1991,10 @@ __temp_gpg_pub() { # PARAMETERS: url #---------------------------------------------------------------------------------------------------------------------- __apt_key_fetch() { + # DGM debug + set -v + set -x + url=$1 @@ -1996,6 +2016,10 @@ __apt_key_fetch() { # PARAMETERS: url #---------------------------------------------------------------------------------------------------------------------- __rpm_import_gpg() { + # DGM debug + set -v + set -x + url=$1 tempfile="$(__temp_gpg_pub)" @@ -2019,6 +2043,10 @@ __rpm_import_gpg() { # DESCRIPTION: (DRY) yum install with noinput options #---------------------------------------------------------------------------------------------------------------------- __yum_install_noinput() { + # DGM debug + set -v + set -x + if [ "$DISTRO_NAME_L" = "oracle_linux" ]; then # We need to install one package at a time because --enablerepo=X disables ALL OTHER REPOS!!!! for package in "${@}"; do @@ -2034,6 +2062,10 @@ __yum_install_noinput() { # DESCRIPTION: (DRY) dnf install with noinput options #---------------------------------------------------------------------------------------------------------------------- __dnf_install_noinput() { + # DGM debug + set -v + set -x + dnf -y install "${@}" || return $? } # ---------- end of function __dnf_install_noinput ---------- @@ -2042,6 +2074,10 @@ __dnf_install_noinput() { # DESCRIPTION: (DRY) tdnf install with noinput options #---------------------------------------------------------------------------------------------------------------------- __tdnf_install_noinput() { + # DGM debug + set -v + set -x + tdnf -y install "${@}" || return $? } # ---------- end of function __tdnf_install_noinput ---------- @@ -2051,6 +2087,10 @@ __tdnf_install_noinput() { # specific revision. #---------------------------------------------------------------------------------------------------------------------- __git_clone_and_checkout() { + # DGM debug + set -v + set -x + echodebug "Installed git version: $(git --version | awk '{ print $3 }')" # Turn off SSL verification if -I flag was set for insecure downloads if [ "$_INSECURE_DL" -eq $BS_TRUE ]; then @@ -2389,6 +2429,10 @@ __overwriteconfig() { # PARAMETERS: servicename #---------------------------------------------------------------------------------------------------------------------- __check_services_systemd() { + # DGM debug + set -v + set -x + if [ $# -eq 0 ]; then echoerror "You need to pass a service name to check!" exit 1 @@ -2415,6 +2459,10 @@ __check_services_systemd() { # PARAMETERS: servicename #---------------------------------------------------------------------------------------------------------------------- __check_services_upstart() { + # DGM debug + set -v + set -x + if [ $# -eq 0 ]; then echoerror "You need to pass a service name to check!" exit 1 @@ -2442,6 +2490,10 @@ __check_services_upstart() { # PARAMETERS: servicename #---------------------------------------------------------------------------------------------------------------------- __check_services_sysvinit() { + # DGM debug + set -v + set -x + if [ $# -eq 0 ]; then echoerror "You need to pass a service name to check!" exit 1 @@ -2468,6 +2520,10 @@ __check_services_sysvinit() { # PARAMETERS: servicename #---------------------------------------------------------------------------------------------------------------------- __check_services_debian() { + # DGM debug + set -v + set -x + if [ $# -eq 0 ]; then echoerror "You need to pass a service name to check!" exit 1 @@ -2495,6 +2551,10 @@ __check_services_debian() { # PARAMETERS: servicename #---------------------------------------------------------------------------------------------------------------------- __check_services_openrc() { + # DGM debug + set -v + set -x + if [ $# -eq 0 ]; then echoerror "You need to pass a service name to check!" exit 1 @@ -2521,6 +2581,10 @@ __check_services_openrc() { # DESCRIPTION: Return 0 or 1 depending on successful creation of virtualenv #---------------------------------------------------------------------------------------------------------------------- __create_virtualenv() { + # DGM debug + set -v + set -x + if [ ! -d "$_VIRTUALENV_DIR" ]; then echoinfo "Creating virtualenv ${_VIRTUALENV_DIR}" if [ "$_PIP_ALL" -eq $BS_TRUE ]; then @@ -2538,6 +2602,10 @@ __create_virtualenv() { # DESCRIPTION: Return 0 or 1 depending on successful activation of virtualenv #---------------------------------------------------------------------------------------------------------------------- __activate_virtualenv() { + # DGM debug + set -v + set -x + set +o nounset # Is virtualenv empty if [ -z "$_VIRTUALENV_DIR" ]; then @@ -2635,6 +2703,10 @@ __install_pip_deps() { # PARAMETERS: py_exe #---------------------------------------------------------------------------------------------------------------------- __install_salt_from_repo() { + # DGM debug + set -v + set -x + _py_exe="$1" if [ "${_py_exe}" = "" ]; then @@ -2881,6 +2953,10 @@ fi # Ubuntu Install Functions # __enable_universe_repository() { + # DGM debug + set -v + set -x + if [ "$(grep -R universe /etc/apt/sources.list /etc/apt/sources.list.d/ | grep -v '#')" != "" ]; then # The universe repository is already enabled return 0 @@ -2964,6 +3040,10 @@ __install_saltstack_ubuntu_repository() { } __install_saltstack_ubuntu_onedir_repository() { + # DGM debug + set -v + set -x + echodebug "__install_saltstack_ubuntu_onedir_repository() entry" # Workaround for latest non-LTS Ubuntu if { [ "$DISTRO_MAJOR_VERSION" -eq 20 ] && [ "$DISTRO_MINOR_VERSION" -eq 10 ]; } || \ @@ -3026,6 +3106,10 @@ __install_saltstack_ubuntu_onedir_repository() { } install_ubuntu_deps() { + # DGM debug + set -v + set -x + echodebug "install_ubuntu_deps() entry" if [ "$_DISABLE_REPOS" -eq $BS_FALSE ]; then # Install add-apt-repository @@ -3083,6 +3167,10 @@ install_ubuntu_deps() { } install_ubuntu_stable_deps() { + # DGM debug + set -v + set -x + echodebug "install_ubuntu_stable_deps() entry" if [ "$_START_DAEMONS" -eq $BS_FALSE ]; then @@ -3118,6 +3206,10 @@ install_ubuntu_stable_deps() { } install_ubuntu_git_deps() { + # DGM debug + set -v + set -x + echodebug "install_ubuntu_git_deps() entry" __wait_for_apt apt-get update || return 1 @@ -3160,6 +3252,10 @@ install_ubuntu_git_deps() { } install_ubuntu_onedir_deps() { + # DGM debug + set -v + set -x + if [ "$_START_DAEMONS" -eq $BS_FALSE ]; then echowarn "Not starting daemons on Debian based distributions is not working mostly because starting them is the default behaviour." fi @@ -3193,6 +3289,10 @@ install_ubuntu_onedir_deps() { } install_ubuntu_stable() { + # DGM debug + set -v + set -x + __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then @@ -3219,6 +3319,10 @@ install_ubuntu_stable() { } install_ubuntu_git() { + # DGM debug + set -v + set -x + # Activate virtualenv before install if [ "${_VIRTUALENV_DIR}" != "null" ]; then __activate_virtualenv || return 1 @@ -3248,6 +3352,10 @@ install_ubuntu_git() { } install_ubuntu_onedir() { + # DGM debug + set -v + set -x + __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then @@ -3274,6 +3382,10 @@ install_ubuntu_onedir() { } install_ubuntu_stable_post() { + # DGM debug + set -v + set -x + for fname in api master minion syndic; do # Skip salt-api since the service should be opt-in and not necessarily started on boot [ $fname = "api" ] && continue @@ -3300,6 +3412,10 @@ install_ubuntu_stable_post() { } install_ubuntu_git_post() { + # DGM debug + set -v + set -x + for fname in api master minion syndic; do # Skip if not meant to be installed [ $fname = "api" ] && \ @@ -3359,6 +3475,10 @@ install_ubuntu_git_post() { } install_ubuntu_restart_daemons() { + # DGM debug + set -v + set -x + [ "$_START_DAEMONS" -eq $BS_FALSE ] && return # Ensure upstart configs / systemd units are loaded @@ -3414,6 +3534,10 @@ install_ubuntu_restart_daemons() { } install_ubuntu_check_services() { + # DGM debug + set -v + set -x + for fname in api master minion syndic; do # Skip salt-api since the service should be opt-in and not necessarily started on boot [ $fname = "api" ] && continue @@ -3442,6 +3566,10 @@ install_ubuntu_check_services() { # Debian Install Functions # __install_saltstack_debian_repository() { + # DGM debug + set -v + set -x + echodebug "__install_saltstack_debian_repository() entry" ## DGM DEBIAN_RELEASE="$DISTRO_MAJOR_VERSION" @@ -3493,6 +3621,10 @@ __install_saltstack_debian_repository() { } __install_saltstack_debian_onedir_repository() { + # DGM debug + set -v + set -x + echodebug "__install_saltstack_debian_onedir_repository() entry" ## DGM DEBIAN_RELEASE="$DISTRO_MAJOR_VERSION" @@ -3548,6 +3680,10 @@ __install_saltstack_debian_onedir_repository() { } install_debian_onedir_deps() { + # DGM debug + set -v + set -x + echodebug "install_debian_onedir_git_deps() entry" if [ "$_START_DAEMONS" -eq $BS_FALSE ]; then @@ -3602,6 +3738,10 @@ install_debian_onedir_deps() { } install_debian_git_deps() { + # DGM debug + set -v + set -x + echodebug "install_debian_git_deps() entry" __wait_for_apt apt-get update || return 1 @@ -3637,6 +3777,10 @@ install_debian_git_deps() { } install_debian_stable() { + # DGM debug + set -v + set -x + __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then @@ -3663,18 +3807,30 @@ install_debian_stable() { } install_debian_11_git_deps() { + # DGM debug + set -v + set -x + install_debian_git_deps || return 1 return 0 } install_debian_12_git_deps() { + # DGM debug + set -v + set -x + install_debian_git_deps || return 1 return 0 } install_debian_git() { + # DGM debug + set -v + set -x + if [ -n "$_PY_EXE" ]; then _PYEXE=${_PY_EXE} else @@ -3702,16 +3858,28 @@ install_debian_git() { } install_debian_11_git() { + # DGM debug + set -v + set -x + install_debian_git || return 1 return 0 } install_debian_12_git() { + # DGM debug + set -v + set -x + install_debian_git || return 1 return 0 } install_debian_onedir() { + # DGM debug + set -v + set -x + __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then @@ -3738,6 +3906,10 @@ install_debian_onedir() { } install_debian_git_post() { + # DGM debug + set -v + set -x + for fname in api master minion syndic; do # Skip if not meant to be installed [ "$fname" = "api" ] && \ @@ -3777,6 +3949,10 @@ install_debian_git_post() { } install_debian_2021_post() { + # DGM debug + set -v + set -x + # Kali 2021 (debian derivative) disables all network services by default # Using archlinux post function to enable salt systemd services install_arch_linux_post || return 1 @@ -3784,6 +3960,10 @@ install_debian_2021_post() { } install_debian_restart_daemons() { + # DGM debug + set -v + set -x + [ "$_START_DAEMONS" -eq $BS_FALSE ] && return 0 for fname in api master minion syndic; do @@ -3812,6 +3992,10 @@ install_debian_restart_daemons() { } install_debian_check_services() { + # DGM debug + set -v + set -x + for fname in api master minion syndic; do # Skip salt-api since the service should be opt-in and not necessarily started on boot [ $fname = "api" ] && continue @@ -3840,6 +4024,10 @@ install_debian_check_services() { # __install_saltstack_fedora_onedir_repository() { + # DGM debug + set -v + set -x + if [ "$ITYPE" = "stable" ]; then REPO_REV="$ONEDIR_REV" else @@ -3892,6 +4080,10 @@ __install_saltstack_fedora_onedir_repository() { } install_fedora_deps() { + # DGM debug + set -v + set -x + if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then dnf -y update || return 1 fi @@ -3922,6 +4114,10 @@ install_fedora_deps() { } install_fedora_git_deps() { + # DGM debug + set -v + set -x + if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -ne 3 ]; then echoerror "Python version is no longer supported, only Python 3" return 1 @@ -3961,6 +4157,10 @@ install_fedora_git_deps() { } install_fedora_git() { + # DGM debug + set -v + set -x + if [ "${_PY_EXE}" != "" ]; then _PYEXE=${_PY_EXE} echoinfo "Using the following python version: ${_PY_EXE} to install salt" @@ -3975,6 +4175,10 @@ install_fedora_git() { } install_fedora_git_post() { + # DGM debug + set -v + set -x + for fname in api master minion syndic; do # Skip if not meant to be installed [ $fname = "api" ] && \ @@ -4006,6 +4210,10 @@ install_fedora_git_post() { } install_fedora_restart_daemons() { + # DGM debug + set -v + set -x + [ $_START_DAEMONS -eq $BS_FALSE ] && return for fname in api master minion syndic; do @@ -4028,6 +4236,10 @@ install_fedora_restart_daemons() { } install_fedora_check_services() { + # DGM debug + set -v + set -x + for fname in api master minion syndic; do # Skip salt-api since the service should be opt-in and not necessarily started on boot [ $fname = "api" ] && continue @@ -4044,6 +4256,10 @@ install_fedora_check_services() { } install_fedora_onedir_deps() { + # DGM debug + set -v + set -x + if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then yum -y update || return 1 fi @@ -4082,6 +4298,10 @@ install_fedora_onedir_deps() { install_fedora_onedir() { + # DGM debug + set -v + set -x + STABLE_REV=$ONEDIR_REV #install_fedora_stable || return 1 @@ -4111,6 +4331,10 @@ install_fedora_onedir() { } install_fedora_onedir_post() { + # DGM debug + set -v + set -x + STABLE_REV=$ONEDIR_REV for fname in api master minion syndic; do @@ -4140,6 +4364,10 @@ install_fedora_onedir_post() { # CentOS Install Functions # __install_saltstack_rhel_onedir_repository() { + # DGM debug + set -v + set -x + if [ "$ITYPE" = "stable" ]; then REPO_REV="$ONEDIR_REV" else @@ -4215,6 +4443,10 @@ __install_saltstack_rhel_onedir_repository() { } install_centos_stable_deps() { + # DGM debug + set -v + set -x + if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then yum -y update || return 1 fi @@ -4252,6 +4484,10 @@ install_centos_stable_deps() { } install_centos_stable() { + # DGM debug + set -v + set -x + __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then @@ -4285,6 +4521,10 @@ install_centos_stable() { } install_centos_stable_post() { + # DGM debug + set -v + set -x + SYSTEMD_RELOAD=$BS_FALSE for fname in api master minion syndic; do @@ -4316,6 +4556,10 @@ install_centos_stable_post() { } install_centos_git_deps() { + # DGM debug + set -v + set -x + # First try stable deps then fall back to onedir deps if that one fails # if we're installing on a Red Hat based host that doesn't have the classic # package repos available. @@ -4361,6 +4605,10 @@ install_centos_git_deps() { } install_centos_git() { + # DGM debug + set -v + set -x + if [ "${_PY_EXE}" != "" ]; then _PYEXE=${_PY_EXE} echoinfo "Using the following python version: ${_PY_EXE} to install salt" @@ -4376,6 +4624,10 @@ install_centos_git() { } install_centos_git_post() { + # DGM debug + set -v + set -x + SYSTEMD_RELOAD=$BS_FALSE for fname in api master minion syndic; do @@ -4416,6 +4668,10 @@ install_centos_git_post() { } install_centos_onedir_deps() { + # DGM debug + set -v + set -x + if [ "$_UPGRADE_SYS" -eq "$BS_TRUE" ]; then yum -y update || return 1 fi @@ -4452,6 +4708,10 @@ install_centos_onedir_deps() { } install_centos_onedir() { + # DGM debug + set -v + set -x + __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then @@ -4478,6 +4738,10 @@ install_centos_onedir() { } install_centos_onedir_post() { + # DGM debug + set -v + set -x + SYSTEMD_RELOAD=$BS_FALSE for fname in api master minion syndic; do @@ -4509,6 +4773,10 @@ install_centos_onedir_post() { } install_centos_restart_daemons() { + # DGM debug + set -v + set -x + [ "$_START_DAEMONS" -eq $BS_FALSE ] && return for fname in api master minion syndic; do @@ -4553,21 +4821,37 @@ install_centos_restart_daemons() { } install_centos_testing_deps() { + # DGM debug + set -v + set -x + install_centos_stable_deps || return 1 return 0 } install_centos_testing() { + # DGM debug + set -v + set -x + install_centos_stable || return 1 return 0 } install_centos_testing_post() { + # DGM debug + set -v + set -x + install_centos_stable_post || return 1 return 0 } install_centos_check_services() { + # DGM debug + set -v + set -x + for fname in api master minion syndic; do # Skip salt-api since the service should be opt-in and not necessarily started on boot [ $fname = "api" ] && continue @@ -4596,286 +4880,514 @@ install_centos_check_services() { # RedHat Install Functions # install_red_hat_linux_stable_deps() { + # DGM debug + set -v + set -x + install_centos_stable_deps || return 1 return 0 } install_red_hat_linux_git_deps() { + # DGM debug + set -v + set -x + install_centos_git_deps || return 1 return 0 } install_red_hat_linux_onedir_deps() { + # DGM debug + set -v + set -x + install_centos_onedir_deps || return 1 return 0 } install_red_hat_enterprise_stable_deps() { + # DGM debug + set -v + set -x + install_red_hat_linux_stable_deps || return 1 return 0 } install_red_hat_enterprise_git_deps() { + # DGM debug + set -v + set -x + install_red_hat_linux_git_deps || return 1 return 0 } install_red_hat_enterprise_onedir_deps() { + # DGM debug + set -v + set -x + install_red_hat_linux_onedir_deps || return 1 return 0 } install_red_hat_enterprise_linux_stable_deps() { + # DGM debug + set -v + set -x + install_red_hat_linux_stable_deps || return 1 return 0 } install_red_hat_enterprise_linux_git_deps() { + # DGM debug + set -v + set -x + install_red_hat_linux_git_deps || return 1 return 0 } install_red_hat_enterprise_linux_onedir_deps() { + # DGM debug + set -v + set -x + install_red_hat_linux_onedir_deps || return 1 return 0 } install_red_hat_enterprise_server_stable_deps() { + # DGM debug + set -v + set -x + install_red_hat_linux_stable_deps || return 1 return 0 } install_red_hat_enterprise_server_git_deps() { + # DGM debug + set -v + set -x + install_red_hat_linux_git_deps || return 1 return 0 } install_red_hat_enterprise_server_onedir_deps() { + # DGM debug + set -v + set -x + install_red_hat_linux_onedir_deps || return 1 return 0 } install_red_hat_enterprise_workstation_stable_deps() { + # DGM debug + set -v + set -x + install_red_hat_linux_stable_deps || return 1 return 0 } install_red_hat_enterprise_workstation_git_deps() { + # DGM debug + set -v + set -x + install_red_hat_linux_git_deps || return 1 return 0 } install_red_hat_enterprise_workstation_onedir_deps() { + # DGM debug + set -v + set -x + install_red_hat_linux_timat_deps || return 1 return 0 } install_red_hat_linux_stable() { + # DGM debug + set -v + set -x + install_centos_stable || return 1 return 0 } install_red_hat_linux_git() { + # DGM debug + set -v + set -x + install_centos_git || return 1 return 0 } install_red_hat_linux_onedir() { + # DGM debug + set -v + set -x + install_centos_onedir || return 1 return 0 } install_red_hat_enterprise_stable() { + # DGM debug + set -v + set -x + install_red_hat_linux_stable || return 1 return 0 } install_red_hat_enterprise_git() { + # DGM debug + set -v + set -x + install_red_hat_linux_git || return 1 return 0 } install_red_hat_enterprise_onedir() { + # DGM debug + set -v + set -x + install_red_hat_linux_onedir || return 1 return 0 } install_red_hat_enterprise_linux_stable() { + # DGM debug + set -v + set -x + install_red_hat_linux_stable || return 1 return 0 } install_red_hat_enterprise_linux_git() { + # DGM debug + set -v + set -x + install_red_hat_linux_git || return 1 return 0 } install_red_hat_enterprise_linux_onedir() { + # DGM debug + set -v + set -x + install_red_hat_linux_onedir || return 1 return 0 } install_red_hat_enterprise_server_stable() { + # DGM debug + set -v + set -x + install_red_hat_linux_stable || return 1 return 0 } install_red_hat_enterprise_server_git() { + # DGM debug + set -v + set -x + install_red_hat_linux_git || return 1 return 0 } install_red_hat_enterprise_server_onedir() { + # DGM debug + set -v + set -x + install_red_hat_linux_onedir || return 1 return 0 } install_red_hat_enterprise_workstation_stable() { + # DGM debug + set -v + set -x + install_red_hat_linux_stable || return 1 return 0 } install_red_hat_enterprise_workstation_git() { + # DGM debug + set -v + set -x + install_red_hat_linux_git || return 1 return 0 } install_red_hat_enterprise_workstation_onedir() { + # DGM debug + set -v + set -x + install_red_hat_linux_onedir || return 1 return 0 } install_red_hat_linux_stable_post() { + # DGM debug + set -v + set -x + install_centos_stable_post || return 1 return 0 } install_red_hat_linux_restart_daemons() { + # DGM debug + set -v + set -x + install_centos_restart_daemons || return 1 return 0 } install_red_hat_linux_git_post() { + # DGM debug + set -v + set -x + install_centos_git_post || return 1 return 0 } install_red_hat_enterprise_stable_post() { + # DGM debug + set -v + set -x + install_red_hat_linux_stable_post || return 1 return 0 } install_red_hat_enterprise_restart_daemons() { + # DGM debug + set -v + set -x + install_red_hat_linux_restart_daemons || return 1 return 0 } install_red_hat_enterprise_git_post() { + # DGM debug + set -v + set -x + install_red_hat_linux_git_post || return 1 return 0 } install_red_hat_enterprise_linux_stable_post() { + # DGM debug + set -v + set -x + install_red_hat_linux_stable_post || return 1 return 0 } install_red_hat_enterprise_linux_restart_daemons() { + # DGM debug + set -v + set -x + install_red_hat_linux_restart_daemons || return 1 return 0 } install_red_hat_enterprise_linux_git_post() { + # DGM debug + set -v + set -x + install_red_hat_linux_git_post || return 1 return 0 } install_red_hat_enterprise_server_stable_post() { + # DGM debug + set -v + set -x + install_red_hat_linux_stable_post || return 1 return 0 } install_red_hat_enterprise_server_restart_daemons() { + # DGM debug + set -v + set -x + install_red_hat_linux_restart_daemons || return 1 return 0 } install_red_hat_enterprise_server_git_post() { + # DGM debug + set -v + set -x + install_red_hat_linux_git_post || return 1 return 0 } install_red_hat_enterprise_workstation_stable_post() { + # DGM debug + set -v + set -x + install_red_hat_linux_stable_post || return 1 return 0 } install_red_hat_enterprise_workstation_restart_daemons() { + # DGM debug + set -v + set -x + install_red_hat_linux_restart_daemons || return 1 return 0 } install_red_hat_enterprise_workstation_git_post() { + # DGM debug + set -v + set -x + install_red_hat_linux_git_post || return 1 return 0 } install_red_hat_linux_testing_deps() { + # DGM debug + set -v + set -x + install_centos_testing_deps || return 1 return 0 } install_red_hat_linux_testing() { + # DGM debug + set -v + set -x + install_centos_testing || return 1 return 0 } install_red_hat_linux_testing_post() { + # DGM debug + set -v + set -x + install_centos_testing_post || return 1 return 0 } install_red_hat_enterprise_testing_deps() { + # DGM debug + set -v + set -x + install_centos_testing_deps || return 1 return 0 } install_red_hat_enterprise_testing() { + # DGM debug + set -v + set -x + install_centos_testing || return 1 return 0 } install_red_hat_enterprise_testing_post() { + # DGM debug + set -v + set -x + install_centos_testing_post || return 1 return 0 } install_red_hat_enterprise_server_testing_deps() { + # DGM debug + set -v + set -x + install_centos_testing_deps || return 1 return 0 } install_red_hat_enterprise_server_testing() { + # DGM debug + set -v + set -x + install_centos_testing || return 1 return 0 } install_red_hat_enterprise_server_testing_post() { + # DGM debug + set -v + set -x + install_centos_testing_post || return 1 return 0 } install_red_hat_enterprise_workstation_testing_deps() { + # DGM debug + set -v + set -x + install_centos_testing_deps || return 1 return 0 } install_red_hat_enterprise_workstation_testing() { + # DGM debug + set -v + set -x + install_centos_testing || return 1 return 0 } install_red_hat_enterprise_workstation_testing_post() { + # DGM debug + set -v + set -x + install_centos_testing_post || return 1 return 0 } @@ -4889,6 +5401,10 @@ install_red_hat_enterprise_workstation_testing_post() { # Oracle Linux Install Functions # install_oracle_linux_stable_deps() { + # DGM debug + set -v + set -x + # Install Oracle's EPEL. if [ "${_EPEL_REPOS_INSTALLED}" -eq $BS_FALSE ]; then _EPEL_REPO=oracle-epel-release-el${DISTRO_MAJOR_VERSION} @@ -7345,6 +7861,10 @@ daemons_running_voidlinux() { ## DGM } __macosx_get_packagesite_onedir_latest() { + # DGM debug + set -v + set -x + echodebug "Find latest MacOS release from repository" # get dir listing from url, sort and pick highest @@ -7363,6 +7883,10 @@ __macosx_get_packagesite_onedir_latest() { __macosx_get_packagesite_onedir() { + # DGM debug + set -v + set -x + if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -ne 3 ]; then echoerror "Python version is no longer supported, only Python 3" return 1 @@ -7402,21 +7926,37 @@ __macosx_get_packagesite_onedir() { } __configure_macosx_pkg_details_onedir() { + # DGM debug + set -v + set -x + __macosx_get_packagesite_onedir || return 1 return 0 } install_macosx_stable_deps() { + # DGM debug + set -v + set -x + __configure_macosx_pkg_details_onedir || return 1 return 0 } install_macosx_onedir_deps() { + # DGM debug + set -v + set -x + __configure_macosx_pkg_details_onedir || return 1 return 0 } install_macosx_git_deps() { + # DGM debug + set -v + set -x + install_macosx_stable_deps || return 1 if ! echo "$PATH" | grep -q /usr/local/bin; then @@ -7442,6 +7982,10 @@ install_macosx_git_deps() { } install_macosx_stable() { + # DGM debug + set -v + set -x + install_macosx_stable_deps || return 1 __fetch_url "/tmp/${PKG}" "${SALTPKGCONFURL}" || return 1 @@ -7452,6 +7996,10 @@ install_macosx_stable() { } install_macosx_onedir() { + # DGM debug + set -v + set -x + install_macosx_onedir_deps || return 1 __fetch_url "/tmp/${PKG}" "${SALTPKGCONFURL}" || return 1 @@ -7462,6 +8010,10 @@ install_macosx_onedir() { } install_macosx_git() { + # DGM debug + set -v + set -x + if [ -n "$_PY_EXE" ]; then _PYEXE="${_PY_EXE}" @@ -7475,6 +8027,10 @@ install_macosx_git() { } install_macosx_stable_post() { + # DGM debug + set -v + set -x + if [ ! -f /etc/paths.d/salt ]; then print "%s\n" "/opt/salt/bin" "/usr/local/sbin" > /etc/paths.d/salt fi @@ -7490,16 +8046,28 @@ install_macosx_stable_post() { } install_macosx_onedir_post() { + # DGM debug + set -v + set -x + install_macosx_stable_post || return 1 return 0 } install_macosx_git_post() { + # DGM debug + set -v + set -x + install_macosx_stable_post || return 1 return 0 } install_macosx_restart_daemons() { + # DGM debug + set -v + set -x + [ "$_START_DAEMONS" -eq $BS_FALSE ] && return if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then @@ -7525,6 +8093,10 @@ install_macosx_restart_daemons() { # the -c options is passed. # config_salt() { + # DGM debug + set -v + set -x + # If the configuration directory is not passed, return [ "$_TEMP_CONFIG_DIR" = "null" ] && return @@ -7686,6 +8258,10 @@ config_salt() { # as long as the -k option is passed. # preseed_master() { + # DGM debug + set -v + set -x + # Create the PKI directory if [ "$(find "$_TEMP_KEYS_DIR" -maxdepth 1 -type f | wc -l)" -lt 1 ]; then @@ -7720,6 +8296,10 @@ preseed_master() { # This function checks if all of the installed daemons are running or not. # daemons_running_onedir() { + # DGM debug + set -v + set -x + [ "$_START_DAEMONS" -eq $BS_FALSE ] && return 0 FAILED_DAEMONS=0 @@ -7757,6 +8337,10 @@ daemons_running_onedir() { # This function checks if all of the installed daemons are running or not. # daemons_running() { + # DGM debug + set -v + set -x + [ "$_START_DAEMONS" -eq $BS_FALSE ] && return 0 FAILED_DAEMONS=0 From 0a3b4f9b64201b7bbb133c9fa4c85f802278c46a Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 4 Nov 2024 16:50:23 -0700 Subject: [PATCH 009/173] Trying to run scripts as root, using sudo --- .github/workflows/test-linux.yml | 2 +- .github/workflows/test-macos.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 000074322..93e83f766 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -77,7 +77,7 @@ jobs: - name: Bootstrap Salt run: | - sh -x ./bootstrap-salt.sh + sudo sh -x ./bootstrap-salt.sh - name: Test Bootstrap run: | diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index f82f15590..7d201a9ae 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -85,7 +85,7 @@ jobs: - name: Bootstrap Salt run: | - sh -x ./bootstrap-salt.sh + sudo sh -x ./bootstrap-salt.sh - name: Test Bootstrap run: | From 0c7cf7353c326db1d30b0480d5d1ad9820fd9fde Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 4 Nov 2024 16:59:20 -0700 Subject: [PATCH 010/173] Updated security url for single slash --- bootstrap-salt.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index e808e2fab..1cd4282bf 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -3017,7 +3017,7 @@ __install_saltstack_ubuntu_repository() { ## DGM __apt_key_fetch "$SALTSTACK_UBUNTU_URL/SALT-PROJECT-GPG-PUBKEY-2023.gpg" || return 1 __fetch_url "/etc/apt/sources.list.d/salt.sources" "https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.sources" - __apt_key_fetch "${HTTP_VAL}://${_REPO_URL}//api/security/keypair/SaltProjectKey/public" || return 1 + __apt_key_fetch "${HTTP_VAL}://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" || return 1 __wait_for_apt apt-get update || return 1 if [ "$STABLE_REV" != "latest" ]; then @@ -3087,7 +3087,7 @@ __install_saltstack_ubuntu_onedir_repository() { ## DGM echo "$__REPO_ARCH_DEB $SALTSTACK_UBUNTU_URL $UBUNTU_CODENAME main" > /etc/apt/sources.list.d/salt.list __fetch_url "/etc/apt/sources.list.d/salt.sources" "https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.sources" - __apt_key_fetch "${HTTP_VAL}://${_REPO_URL}//api/security/keypair/SaltProjectKey/public" || return 1 + __apt_key_fetch "${HTTP_VAL}://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" || return 1 __wait_for_apt apt-get update || return 1 if [ "$STABLE_REV" != "latest" ]; then @@ -3602,7 +3602,7 @@ __install_saltstack_debian_repository() { ## __wait_for_apt apt-get update || return 1 __fetch_url "/etc/apt/sources.list.d/salt.sources" "https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.sources" - __apt_key_fetch "${HTTP_VAL}://${_REPO_URL}//api/security/keypair/SaltProjectKey/public" || return 1 + __apt_key_fetch "${HTTP_VAL}://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" || return 1 __wait_for_apt apt-get update || return 1 if [ "$STABLE_REV" != "latest" ]; then @@ -3661,7 +3661,7 @@ __install_saltstack_debian_onedir_repository() { ## DGM __wait_for_apt apt-get update || return 1 __fetch_url "/etc/apt/sources.list.d/salt.sources" "https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.sources" - __apt_key_fetch "${HTTP_VAL}://${_REPO_URL}//api/security/keypair/SaltProjectKey/public" || return 1 + __apt_key_fetch "${HTTP_VAL}://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" || return 1 __wait_for_apt apt-get update || return 1 if [ "$STABLE_REV" != "latest" ]; then From d5ff514a6b67275959b830da7ac7b68a544a8c19 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 5 Nov 2024 09:39:33 -0700 Subject: [PATCH 011/173] Fixed typo gpg -> pgp --- bootstrap-salt.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 1cd4282bf..63ac557dd 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -2003,7 +2003,7 @@ __apt_key_fetch() { ## DGM cp -f "$tempfile" /usr/share/keyrings/salt-archive-keyring.gpg && chmod 644 /usr/share/keyrings/salt-archive-keyring.gpg || return 1 __fetch_url "$tempfile" "$url" || return 1 mkdir -p /etc/apt/keyrings - cp -f "$tempfile" /etc/apt/keyrings/salt-archive-keyring.gpg && chmod 644 /etc/apt/keyrings/salt-archive-keyring.pgp || return 1 + cp -f "$tempfile" /etc/apt/keyrings/salt-archive-keyring.pgp && chmod 644 /etc/apt/keyrings/salt-archive-keyring.pgp || return 1 rm -f "$tempfile" return 0 From 581f650d6e613ecb64516e839f475af907f688b0 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 5 Nov 2024 09:59:50 -0700 Subject: [PATCH 012/173] Set version defaults to 'latest' --- bootstrap-salt.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 63ac557dd..ff9fea5e8 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -26,7 +26,7 @@ #====================================================================================================================== set -o nounset # Treat unset variables as an error -__ScriptVersion="2024.11.01" +__ScriptVersion="2024.11.05" __ScriptName="bootstrap-salt.sh" __ScriptFullName="$0" @@ -582,6 +582,11 @@ echoinfo "Executed by: ${CALLER}" echoinfo "Command line: '${__ScriptFullName} ${__ScriptArgs}'" echowarn "Running the unstable version of ${__ScriptName}" +# Defaults +STABLE_REV="latest" +ONEDIR_REV="latest" +_ONEDIR_REV="latest" + # Define installation type if [ "$#" -gt 0 ];then __check_unparsed_options "$*" From ded5813ead5c033531947a717f5d3d94c30b2f91 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 5 Nov 2024 11:18:29 -0700 Subject: [PATCH 013/173] Debugging tests --- tests/integration/test_installation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index 3a65144fb..1904d0422 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -26,6 +26,7 @@ def run_salt_call(cmd): """ cmd.append("--out=json") result = subprocess.run(cmd, capture_output=True, text=True) + print(f"DGM run_salt_call, cmd '{cmd}', result '{result}'", flush=True) json_data = json.loads(result.stdout) return json_data["local"] From ef6d6b6d7eb789abbd836297ddb8eabc3e369dc0 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 5 Nov 2024 11:35:37 -0700 Subject: [PATCH 014/173] Trying container --- .github/workflows/test-linux.yml | 4 +++- tests/integration/test_installation.py | 6 ++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 93e83f766..37e199959 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -26,7 +26,9 @@ on: jobs: Test: name: ${{ matrix.instance }} - runs-on: ubuntu-20.04 + ## DGM runs-on: ubuntu-20.04 + runs-on: ubuntu-latest + container: ubuntu-22.04 timeout-minutes: ${{ inputs.timeout }} strategy: max-parallel: 2 diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index 1904d0422..00da09587 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -27,8 +27,10 @@ def run_salt_call(cmd): cmd.append("--out=json") result = subprocess.run(cmd, capture_output=True, text=True) print(f"DGM run_salt_call, cmd '{cmd}', result '{result}'", flush=True) - json_data = json.loads(result.stdout) - return json_data["local"] + if result.stdout: + json_data = json.loads(result.stdout) + return json_data["local"] + return None def test_ping(): From 7d5b7c1ace77d4c9b131baef5a9a7de985708dc9 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 5 Nov 2024 11:42:47 -0700 Subject: [PATCH 015/173] Trying Rocky Linux 9 container --- .github/workflows/test-linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 37e199959..1e6426b34 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -28,7 +28,7 @@ jobs: name: ${{ matrix.instance }} ## DGM runs-on: ubuntu-20.04 runs-on: ubuntu-latest - container: ubuntu-22.04 + container: rockylinux-9 timeout-minutes: ${{ inputs.timeout }} strategy: max-parallel: 2 From 46fdd9ed68bb2c4e3cea8239a084e1a2e2f3dc01 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 5 Nov 2024 13:44:54 -0700 Subject: [PATCH 016/173] Using salt-ci-containers --- .github/workflows/test-linux.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 1e6426b34..2240b5547 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -28,7 +28,9 @@ jobs: name: ${{ matrix.instance }} ## DGM runs-on: ubuntu-20.04 runs-on: ubuntu-latest - container: rockylinux-9 + container: + image: ghcr.io/saltstack/salt-ci-containers/packaging:rockylinux-9 + timeout-minutes: ${{ inputs.timeout }} strategy: max-parallel: 2 From efe775e630d9ba5d365ae4903e7919a48395bcab Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 5 Nov 2024 13:57:10 -0700 Subject: [PATCH 017/173] Removed use of sudo --- .github/workflows/test-linux.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 2240b5547..3cd736fa4 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -81,7 +81,8 @@ jobs: - name: Bootstrap Salt run: | - sudo sh -x ./bootstrap-salt.sh + ## sudo sh -x ./bootstrap-salt.sh + sh -x ./bootstrap-salt.sh - name: Test Bootstrap run: | From 6050a0b21731347b694d0f12eab8f82a067d4e50 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 5 Nov 2024 14:32:58 -0700 Subject: [PATCH 018/173] Renamed REPO_FILE to YUM_REPO_FILE and created default --- bootstrap-salt.sh | 60 ++++++++++++++++++++++++----------------------- 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index ff9fea5e8..1f12fbeea 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -586,6 +586,7 @@ echowarn "Running the unstable version of ${__ScriptName}" STABLE_REV="latest" ONEDIR_REV="latest" _ONEDIR_REV="latest" +YUM_REPO_FILE="/etc/yum.repos.d/salt.repo" # Define installation type if [ "$#" -gt 0 ];then @@ -4048,19 +4049,19 @@ __install_saltstack_fedora_onedir_repository() { ## DGM GPG_KEY="SALT-PROJECT-GPG-PUBKEY-2023.pub" - REPO_FILE="/etc/yum.repos.d/salt.repo" + YUM_REPO_FILE="/etc/yum.repos.d/salt.repo" - if [ ! -s "$REPO_FILE" ] || [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]; then + if [ ! -s "$YUM_REPO_FILE" ] || [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]; then ## DGM FETCH_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/fedora/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/${ONEDIR_REV}" ## DGM if [ "${ONEDIR_REV}" = "nightly" ] ; then ## DGM FETCH_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/fedora/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/" ## DGM fi - ## DGM __fetch_url "${REPO_FILE}" "${FETCH_URL}.repo" + ## DGM __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}.repo" ## DGM __rpm_import_gpg "${FETCH_URL}/${GPG_KEY}" || return 1 ## DGM yum clean metadata || return 1 FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" - __fetch_url "${REPO_FILE}" "${FETCH_URL}.repo" + __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}.repo" if [ "$REPO_REV" != "latest" ]; then # 3006.x is default REPO_REV_MAJOR=$(echo "$REPO_REV" | cut -d '.' -f 1) @@ -4421,10 +4422,11 @@ __install_saltstack_rhel_onedir_repository() { ## DGM done ## DGM ## DGM yum clean metadata || return 1 - if [ ! -s "$REPO_FILE" ] || [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]; then - REPO_FILE="/etc/yum.repos.d/salt.repo" + YUM_REPO_FILE="/etc/yum.repos.d/salt.repo" + + if [ ! -s "$YUM_REPO_FILE" ] || [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]; then FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" - __fetch_url "${REPO_FILE}" "${FETCH_URL}.repo" + __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}.repo" if [ "$REPO_REV" != "latest" ]; then # 3006.x is default REPO_REV_MAJOR=$(echo "$REPO_REV" | cut -d '.' -f 1) @@ -6030,7 +6032,7 @@ install_amazon_linux_ami_2_deps() { fi ## DGM if [ $_DISABLE_REPOS -eq $BS_FALSE ] || [ "$_CUSTOM_REPO_URL" != "null" ]; then -## DGM __REPO_FILENAME="salt.repo" +## DGM __YUM_REPO_FILENAME="salt.repo" ## DGM PY_PKG_VER=3 ## DGM __PY_VERSION_REPO="py3" ## DGM repo_label="saltstack-py3-repo" @@ -6042,8 +6044,8 @@ install_amazon_linux_ami_2_deps() { ## DGM # This should prob be refactored to use __install_saltstack_rhel_onedir_repository() ## DGM # With args passed in to do the right thing. Reformatted to be more like the ## DGM # amazon linux yum file. -## DGM if [ ! -s "/etc/yum.repos.d/${__REPO_FILENAME}" ]; then -## DGM cat <<_eof > "/etc/yum.repos.d/${__REPO_FILENAME}" +## DGM if [ ! -s "/etc/yum.repos.d/${__YUM_REPO_FILENAME}" ]; then +## DGM cat <<_eof > "/etc/yum.repos.d/${__YUM_REPO_FILENAME}" ## DGM [$repo_label] ## DGM name=$repo_name ## DGM failovermethod=priority @@ -6055,10 +6057,10 @@ install_amazon_linux_ami_2_deps() { ## DGM fi if [ $_DISABLE_REPOS -eq $BS_FALSE ] || [ "$_CUSTOM_REPO_URL" != "null" ]; then - REPO_FILE="/etc/yum.repos.d/salt.repo" - if [ ! -s "${REPO_FILE}" ]; then + YUM_REPO_FILE="/etc/yum.repos.d/salt.repo" + if [ ! -s "${YUM_REPO_FILE}" ]; then FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" - __fetch_url "${REPO_FILE}" "${FETCH_URL}.repo" + __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}.repo" if [ "$REPO_REV" != "latest" ]; then # 3006.x is default REPO_REV_MAJOR=$(echo "$REPO_REV" | cut -d '.' -f 1) @@ -6105,7 +6107,7 @@ install_amazon_linux_ami_2_onedir_deps() { fi ## DGM if [ $_DISABLE_REPOS -eq $BS_FALSE ] || [ "$_CUSTOM_REPO_URL" != "null" ]; then -## DGM __REPO_FILENAME="salt.repo" +## DGM __YUM_REPO_FILENAME="salt.repo" ## DGM __PY_VERSION_REPO="py3" ## DGM PY_PKG_VER=3 ## DGM repo_label="saltstack-py3-repo" @@ -6121,8 +6123,8 @@ install_amazon_linux_ami_2_onedir_deps() { ## DGM ## DGM # With args passed in to do the right thing. Reformatted to be more like the ## DGM # amazon linux yum file. -## DGM if [ ! -s "/etc/yum.repos.d/${__REPO_FILENAME}" ]; then -## DGM cat <<_eof > "/etc/yum.repos.d/${__REPO_FILENAME}" +## DGM if [ ! -s "/etc/yum.repos.d/${__YUM_REPO_FILENAME}" ]; then +## DGM cat <<_eof > "/etc/yum.repos.d/${__YUM_REPO_FILENAME}" ## DGM [$repo_label] ## DGM name=$repo_name ## DGM failovermethod=priority @@ -6134,10 +6136,10 @@ install_amazon_linux_ami_2_onedir_deps() { ## DGM fi if [ $_DISABLE_REPOS -eq $BS_FALSE ] || [ "$_CUSTOM_REPO_URL" != "null" ]; then - REPO_FILE="/etc/yum.repos.d/salt.repo" - if [ ! -s "${REPO_FILE}" ]; then + YUM_REPO_FILE="/etc/yum.repos.d/salt.repo" + if [ ! -s "${YUM_REPO_FILE}" ]; then FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" - __fetch_url "${REPO_FILE}" "${FETCH_URL}.repo" + __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}.repo" if [ "$REPO_REV" != "latest" ]; then # 3006.x is default REPO_REV_MAJOR=$(echo "$REPO_REV" | cut -d '.' -f 1) @@ -6267,7 +6269,7 @@ install_amazon_linux_ami_2023_onedir_deps() { fi ## DGM if [ "$_DISABLE_REPOS" -eq $BS_FALSE ] || [ "$_CUSTOM_REPO_URL" != "null" ]; then -## DGM __REPO_FILENAME="salt.repo" +## DGM __YUM_REPO_FILENAME="salt.repo" ## DGM __PY_VERSION_REPO="py3" ## DGM PY_PKG_VER=3 ## DGM repo_label="saltstack-py3-repo" @@ -6283,8 +6285,8 @@ install_amazon_linux_ami_2023_onedir_deps() { ## DGM # This should prob be refactored to use __install_saltstack_rhel_onedir_repository() ## DGM # With args passed in to do the right thing. Reformatted to be more like the ## DGM # amazon linux yum file. -## DGM if [ ! -s "/etc/yum.repos.d/${__REPO_FILENAME}" ]; then -## DGM cat <<_eof > "/etc/yum.repos.d/${__REPO_FILENAME}" +## DGM if [ ! -s "/etc/yum.repos.d/${__YUM_REPO_FILENAME}" ]; then +## DGM cat <<_eof > "/etc/yum.repos.d/${__YUM_REPO_FILENAME}" ## DGM [$repo_label] ## DGM name=$repo_name ## DGM failovermethod=priority @@ -6296,10 +6298,10 @@ install_amazon_linux_ami_2023_onedir_deps() { ## DGM fi ## DGM fi if [ $_DISABLE_REPOS -eq $BS_FALSE ] || [ "$_CUSTOM_REPO_URL" != "null" ]; then - REPO_FILE="/etc/yum.repos.d/salt.repo" - if [ ! -s "${REPO_FILE}" ]; then + YUM_REPO_FILE="/etc/yum.repos.d/salt.repo" + if [ ! -s "${YUM_REPO_FILE}" ]; then FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" - __fetch_url "${REPO_FILE}" "${FETCH_URL}.repo" + __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}.repo" if [ "$REPO_REV" != "latest" ]; then # 3006.x is default REPO_REV_MAJOR=$(echo "$REPO_REV" | cut -d '.' -f 1) @@ -6645,20 +6647,20 @@ __install_saltstack_photon_onedir_repository() { fi ## DGM __PY_VERSION_REPO="py3" - REPO_FILE="/etc/yum.repos.d/salt.repo" + YUM_REPO_FILE="/etc/yum.repos.d/salt.repo" - if [ ! -s "$REPO_FILE" ] || [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]; then + if [ ! -s "$YUM_REPO_FILE" ] || [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]; then ## DGM ## salt repo 4 & 5 have issues, need the Major version dot Zero, eg: 4.0, 5.0 ## DGM FETCH_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/photon/${DISTRO_MAJOR_VERSION}.0/${CPU_ARCH_L}/${ONEDIR_REV}" ## DGM if [ "${ONEDIR_REV}" = "nightly" ] ; then ## DGM FETCH_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/photon/${DISTRO_MAJOR_VERSION}.0/${CPU_ARCH_L}/" ## DGM fi - ## DGM __fetch_url "${REPO_FILE}" "${FETCH_URL}.repo" + ## DGM __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}.repo" ## DGM GPG_KEY="SALT-PROJECT-GPG-PUBKEY-2023.pub" ## DGM __rpm_import_gpg "${FETCH_URL}/${GPG_KEY}" || return 1 FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" - __fetch_url "${REPO_FILE}" "${FETCH_URL}.repo" + __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}.repo" if [ "$REPO_REV" != "latest" ]; then # 3006.x is default REPO_REV_MAJOR=$(echo "$REPO_REV" | cut -d '.' -f 1) From 67a0bef81d8b4e9b2cbfd346d3e50f5d20d4f4b8 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 5 Nov 2024 15:19:26 -0700 Subject: [PATCH 019/173] Trying to use init.rc to start / stop in case using container which do not have full systemd implementation, but have systemctl present --- bootstrap-salt.sh | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 1f12fbeea..37501bb3a 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -3408,8 +3408,11 @@ install_ubuntu_stable_post() { /bin/systemctl enable salt-$fname.service > /dev/null 2>&1 ) sleep 1 - /bin/systemctl daemon-reload - elif [ -f /etc/init.d/salt-$fname ]; then + /bin/systemctl daemon-reload && continue + fi + + # if here, try using init.rc in case using container without systemd + if [ -f /etc/init.d/salt-$fname ]; then update-rc.d salt-$fname defaults fi done @@ -3445,7 +3448,7 @@ install_ubuntu_git_post() { systemctl is-enabled salt-$fname.service || (systemctl preset salt-$fname.service && systemctl enable salt-$fname.service) sleep 1 - systemctl daemon-reload + systemctl daemon-reload && continue elif [ -f /sbin/initctl ]; then _upstart_conf="/etc/init/salt-$fname.conf" # We have upstart support @@ -3461,8 +3464,10 @@ install_ubuntu_git_post() { fi /sbin/initctl reload-configuration || return 1 fi + fi # No upstart support in Ubuntu!? - elif [ -f "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.init" ]; then + # if here, try using init.rc in case using container without systemd + if [ -f "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.init" ]; then echodebug "There's NO upstart support!?" echodebug "Copying ${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.init to /etc/init.d/salt-$fname" __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.init" "/etc/init.d/salt-$fname" @@ -3989,7 +3994,9 @@ install_debian_restart_daemons() { systemctl status salt-$fname.service journalctl -xe fi - elif [ -f /etc/init.d/salt-$fname ]; then + fi + # if here, try using init.rc in case using container without systemd + if [ -f /etc/init.d/salt-$fname ]; then # Still in SysV init /etc/init.d/salt-$fname stop > /dev/null 2>&1 /etc/init.d/salt-$fname start @@ -4211,7 +4218,12 @@ install_fedora_git_post() { systemctl is-enabled salt-$fname.service || (systemctl preset salt-$fname.service && systemctl enable salt-$fname.service) sleep 1 - systemctl daemon-reload + systemctl daemon-reload && continue + + # if here, try using init.rc in case using container without systemd + if [ -f /etc/init.d/salt-$fname ]; then + update-rc.d salt-$fname defaults + fi done } @@ -4238,6 +4250,11 @@ install_fedora_restart_daemons() { systemctl status salt-$fname.service journalctl -xe fi + # if here, try using init.rc in case using container without systemd + if [ -f /etc/init.d/salt-$fname ]; then + /etc/init.d/salt-$fname stop > /dev/null 2>&1 + /etc/init.d/salt-$fname start && continue + fi done } @@ -4658,7 +4675,9 @@ install_centos_git_post() { fi SYSTEMD_RELOAD=$BS_TRUE - elif [ ! -f "/etc/init.d/salt-$fname" ] || \ + fi + # ensure init.rc in case using container without systemd full support + if [ ! -f "/etc/init.d/salt-$fname" ] || \ { [ -f "/etc/init.d/salt-$fname" ] && [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]; }; then __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/rpm/salt-${fname}" /etc/init.d chmod +x /etc/init.d/salt-${fname} From ff6c817bfeae4341db0e612333e5edcf4a0b6ce7 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 5 Nov 2024 15:42:35 -0700 Subject: [PATCH 020/173] Revert to ubuntu 22.04 container, and fix multiple .repo typo --- .github/workflows/test-linux.yml | 2 +- bootstrap-salt.sh | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 3cd736fa4..75c80a8dd 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -29,7 +29,7 @@ jobs: ## DGM runs-on: ubuntu-20.04 runs-on: ubuntu-latest container: - image: ghcr.io/saltstack/salt-ci-containers/packaging:rockylinux-9 + image: ghcr.io/saltstack/salt-ci-containers/packaging:ubuntu-22.04 timeout-minutes: ${{ inputs.timeout }} strategy: diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 37501bb3a..74de3e239 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -4068,7 +4068,7 @@ __install_saltstack_fedora_onedir_repository() { ## DGM yum clean metadata || return 1 FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" - __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}.repo" + __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" if [ "$REPO_REV" != "latest" ]; then # 3006.x is default REPO_REV_MAJOR=$(echo "$REPO_REV" | cut -d '.' -f 1) @@ -4443,7 +4443,7 @@ __install_saltstack_rhel_onedir_repository() { if [ ! -s "$YUM_REPO_FILE" ] || [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]; then FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" - __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}.repo" + __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" if [ "$REPO_REV" != "latest" ]; then # 3006.x is default REPO_REV_MAJOR=$(echo "$REPO_REV" | cut -d '.' -f 1) @@ -6079,7 +6079,7 @@ install_amazon_linux_ami_2_deps() { YUM_REPO_FILE="/etc/yum.repos.d/salt.repo" if [ ! -s "${YUM_REPO_FILE}" ]; then FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" - __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}.repo" + __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" if [ "$REPO_REV" != "latest" ]; then # 3006.x is default REPO_REV_MAJOR=$(echo "$REPO_REV" | cut -d '.' -f 1) @@ -6158,7 +6158,7 @@ install_amazon_linux_ami_2_onedir_deps() { YUM_REPO_FILE="/etc/yum.repos.d/salt.repo" if [ ! -s "${YUM_REPO_FILE}" ]; then FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" - __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}.repo" + __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" if [ "$REPO_REV" != "latest" ]; then # 3006.x is default REPO_REV_MAJOR=$(echo "$REPO_REV" | cut -d '.' -f 1) @@ -6320,7 +6320,7 @@ install_amazon_linux_ami_2023_onedir_deps() { YUM_REPO_FILE="/etc/yum.repos.d/salt.repo" if [ ! -s "${YUM_REPO_FILE}" ]; then FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" - __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}.repo" + __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" if [ "$REPO_REV" != "latest" ]; then # 3006.x is default REPO_REV_MAJOR=$(echo "$REPO_REV" | cut -d '.' -f 1) @@ -6679,7 +6679,7 @@ __install_saltstack_photon_onedir_repository() { ## DGM __rpm_import_gpg "${FETCH_URL}/${GPG_KEY}" || return 1 FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" - __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}.repo" + __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" if [ "$REPO_REV" != "latest" ]; then # 3006.x is default REPO_REV_MAJOR=$(echo "$REPO_REV" | cut -d '.' -f 1) From 3c2af3b920c3d64b3a041ab09ef0ae0767a427a3 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 5 Nov 2024 16:56:32 -0700 Subject: [PATCH 021/173] Updated how systemd is assummed functional, do not rely on systemctl present --- bootstrap-salt.sh | 240 ++++++++++++++++++++++++++++------------------ 1 file changed, 145 insertions(+), 95 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 74de3e239..9ef7ec686 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -124,6 +124,31 @@ __check_command_exists() { command -v "$1" > /dev/null 2>&1 } +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: __check_services_systemd_functional +# DESCRIPTION: Set _SYSTEMD_FUNCTIONAL = BS_TRUE or BS_FALSE case where systemd is functional (for example: container may not have systemd) +#---------------------------------------------------------------------------------------------------------------------- +__check_services_systemd_functional() { + # DGM debug + set -v + set -x + + # check if systemd is functional, having systemctl present is insufficient + + if [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_FALSE ]; then + # already determined systemd is not functional, default is 1 + return + fi + + if __check_command_exists systemctl; then + # shellcheck disable=SC2034 + _SYSTEMD_HELP="$(systemctl --help)" + else + echoerror "systemctl: command not found, assume systemd not implemented" + _SYSTEMD_FUNCTIONAL=$BS_FALSE + fi +} # ---------- end of function __check_services_systemd_functional ---------- + #--- FUNCTION ------------------------------------------------------------------------------------------------------- # NAME: __check_pip_allowed # DESCRIPTION: Simple function to let the users know that -P needs to be used. @@ -279,6 +304,7 @@ _PIP_INSTALL_ARGS="--prefix=/usr" _PIP_DOWNLOAD_ARGS="" _QUICK_START="$BS_FALSE" _AUTO_ACCEPT_MINION_KEYS="$BS_FALSE" +_SYSTEMD_FUNCTIONAL=$BS_TRUE # Defaults for install arguments ITYPE="stable" @@ -588,6 +614,9 @@ ONEDIR_REV="latest" _ONEDIR_REV="latest" YUM_REPO_FILE="/etc/yum.repos.d/salt.repo" +# check if systemd is functional +__check_services_systemd_functional + # Define installation type if [ "$#" -gt 0 ];then __check_unparsed_options "$*" @@ -2446,6 +2475,20 @@ __check_services_systemd() { echoerror "You need to pass a service name to check as the single argument to the function" fi + # check if systemd is functional, having systemctl present is insufficient + + if [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_FALSE ]; then + # already determined systemd is not functional, default is 1 + return 1 + fi + + _SYSTEMD_ACTIVE=$(/bin/systemctl daemon-reload 2>&1 | grep 'System has not been booted with systemd') + if [ "$_SYSTEMD_ACTIVE" != "" ]; then + echodebug "systemd is not functional, despite systemctl being present" + _SYSTEMD_FUNCTIONAL="$BS_FALSE" + return 1 + fi + servicename=$1 echodebug "Checking if service ${servicename} is enabled" @@ -2876,6 +2919,7 @@ if [ "x${_PY_MAJOR_VERSION}" = "x" ]; then _PY_MAJOR_VERSION=3 fi + ####################################################################################################################### # # Distribution install functions @@ -3401,18 +3445,16 @@ install_ubuntu_stable_post() { [ $fname = "master" ] && [ "$_INSTALL_MASTER" -eq $BS_FALSE ] && continue [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue - if [ -f /bin/systemctl ]; then + ## if [ -f /bin/systemctl ]; then + if [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_TRUE ]; then # Using systemd /bin/systemctl is-enabled salt-$fname.service > /dev/null 2>&1 || ( /bin/systemctl preset salt-$fname.service > /dev/null 2>&1 && /bin/systemctl enable salt-$fname.service > /dev/null 2>&1 ) sleep 1 - /bin/systemctl daemon-reload && continue - fi - - # if here, try using init.rc in case using container without systemd - if [ -f /etc/init.d/salt-$fname ]; then + /bin/systemctl daemon-reload + elif [ -f /etc/init.d/salt-$fname ]; then update-rc.d salt-$fname defaults fi done @@ -3440,7 +3482,8 @@ install_ubuntu_git_post() { _SERVICE_DIR="${_SALT_GIT_CHECKOUT_DIR}/pkg" fi - if [ -f /bin/systemctl ] && [ "$DISTRO_MAJOR_VERSION" -ge 16 ]; then + ## DGM if [ -f /bin/systemctl ] && [ "$DISTRO_MAJOR_VERSION" -ge 16 ]; then + if [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_TRUE ] && [ "$DISTRO_MAJOR_VERSION" -ge 16 ]; then __copyfile "${_SERVICE_DIR}/salt-${fname}.service" "/lib/systemd/system/salt-${fname}.service" # Skip salt-api since the service should be opt-in and not necessarily started on boot @@ -3448,26 +3491,24 @@ install_ubuntu_git_post() { systemctl is-enabled salt-$fname.service || (systemctl preset salt-$fname.service && systemctl enable salt-$fname.service) sleep 1 - systemctl daemon-reload && continue - elif [ -f /sbin/initctl ]; then - _upstart_conf="/etc/init/salt-$fname.conf" - # We have upstart support - echodebug "There's upstart support" - if [ ! -f $_upstart_conf ]; then - # upstart does not know about our service, let's copy the proper file - echowarn "Upstart does not appear to know about salt-$fname" - echodebug "Copying ${_SERVICE_DIR}/salt-$fname.upstart to $_upstart_conf" - __copyfile "${_SERVICE_DIR}/salt-${fname}.upstart" "$_upstart_conf" - # Set service to know about virtualenv - if [ "${_VIRTUALENV_DIR}" != "null" ]; then - echo "SALT_USE_VIRTUALENV=${_VIRTUALENV_DIR}" > /etc/default/salt-${fname} - fi - /sbin/initctl reload-configuration || return 1 - fi - fi + systemctl daemon-reload + ## DGM elif [ -f /sbin/initctl ]; then + ## DGM _upstart_conf="/etc/init/salt-$fname.conf" + ## DGM # We have upstart support + ## DGM echodebug "There's upstart support" + ## DGM if [ ! -f $_upstart_conf ]; then + ## DGM # upstart does not know about our service, let's copy the proper file + ## DGM echowarn "Upstart does not appear to know about salt-$fname" + ## DGM echodebug "Copying ${_SERVICE_DIR}/salt-$fname.upstart to $_upstart_conf" + ## DGM __copyfile "${_SERVICE_DIR}/salt-${fname}.upstart" "$_upstart_conf" + ## DGM # Set service to know about virtualenv + ## DGM if [ "${_VIRTUALENV_DIR}" != "null" ]; then + ## DGM echo "SALT_USE_VIRTUALENV=${_VIRTUALENV_DIR}" > /etc/default/salt-${fname} + ## DGM fi + ## DGM /sbin/initctl reload-configuration || return 1 + ## DGM fi # No upstart support in Ubuntu!? - # if here, try using init.rc in case using container without systemd - if [ -f "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.init" ]; then + elif [ -f "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.init" ]; then echodebug "There's NO upstart support!?" echodebug "Copying ${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.init to /etc/init.d/salt-$fname" __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.init" "/etc/init.d/salt-$fname" @@ -3478,7 +3519,8 @@ install_ubuntu_git_post() { update-rc.d salt-$fname defaults else - echoerror "Neither upstart nor init.d was setup for salt-$fname" + ## DGM echoerror "Neither upstart nor init.d was setup for salt-$fname" + echoerror "No init.d was setup for salt-$fname" fi done @@ -3493,10 +3535,11 @@ install_ubuntu_restart_daemons() { [ "$_START_DAEMONS" -eq $BS_FALSE ] && return # Ensure upstart configs / systemd units are loaded - if [ -f /bin/systemctl ] && [ "$DISTRO_MAJOR_VERSION" -ge 16 ]; then + ## DGM if [ -f /bin/systemctl ] && [ "$DISTRO_MAJOR_VERSION" -ge 16 ]; then + if [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_TRUE ] && [ "$DISTRO_MAJOR_VERSION" -ge 16 ]; then systemctl daemon-reload - elif [ -f /sbin/initctl ]; then - /sbin/initctl reload-configuration + ## DGM elif [ -f /sbin/initctl ]; then + ## DGM /sbin/initctl reload-configuration fi for fname in api master minion syndic; do @@ -3508,7 +3551,8 @@ install_ubuntu_restart_daemons() { [ $fname = "minion" ] && [ "$_INSTALL_MINION" -eq $BS_FALSE ] && continue [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue - if [ -f /bin/systemctl ] && [ "$DISTRO_MAJOR_VERSION" -ge 16 ]; then + ## DGM if [ -f /bin/systemctl ] && [ "$DISTRO_MAJOR_VERSION" -ge 16 ]; then + if [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_TRUE ] && [ "$DISTRO_MAJOR_VERSION" -ge 16 ]; then echodebug "There's systemd support while checking salt-$fname" systemctl stop salt-$fname > /dev/null 2>&1 systemctl start salt-$fname.service && continue @@ -3520,17 +3564,17 @@ install_ubuntu_restart_daemons() { fi fi - if [ -f /sbin/initctl ]; then - echodebug "There's upstart support while checking salt-$fname" + ## DGM if [ -f /sbin/initctl ]; then + ## DGM echodebug "There's upstart support while checking salt-$fname" - if status salt-$fname 2>/dev/null | grep -q running; then - stop salt-$fname || (echodebug "Failed to stop salt-$fname" && return 1) - fi + ## DGM if status salt-$fname 2>/dev/null | grep -q running; then + ## DGM stop salt-$fname || (echodebug "Failed to stop salt-$fname" && return 1) + ## DGM fi - start salt-$fname && continue - # We failed to start the service, let's test the SysV code below - echodebug "Failed to start salt-$fname using Upstart" - fi + ## DGM start salt-$fname && continue + ## DGM # We failed to start the service, let's test the SysV code below + ## DGM echodebug "Failed to start salt-$fname using Upstart" + ## DGM fi if [ ! -f /etc/init.d/salt-$fname ]; then echoerror "No init.d support for salt-$fname was found" @@ -3558,7 +3602,8 @@ install_ubuntu_check_services() { [ $fname = "master" ] && [ "$_INSTALL_MASTER" -eq $BS_FALSE ] && continue [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue - if [ -f /bin/systemctl ] && [ "$DISTRO_MAJOR_VERSION" -ge 16 ]; then + ## DGM if [ -f /bin/systemctl ] && [ "$DISTRO_MAJOR_VERSION" -ge 16 ]; then + if [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_TRUE ] && [ "$DISTRO_MAJOR_VERSION" -ge 16 ]; then __check_services_systemd salt-$fname || return 1 elif [ -f /etc/init.d/salt-$fname ]; then __check_services_debian salt-$fname || return 1 @@ -3937,7 +3982,8 @@ install_debian_git_post() { fi # Configure SystemD for Debian 8 "Jessie" and later - if [ -f /bin/systemctl ]; then + ## DGM if [ -f /bin/systemctl ]; then + if [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_TRUE ]; then if [ ! -f /lib/systemd/system/salt-${fname}.service ] || \ { [ -f /lib/systemd/system/salt-${fname}.service ] && [ $_FORCE_OVERWRITE -eq $BS_TRUE ]; }; then if [ -f "${_SERVICE_DIR}/salt-${fname}.service" ]; then @@ -3986,7 +4032,8 @@ install_debian_restart_daemons() { [ $fname = "minion" ] && [ "$_INSTALL_MINION" -eq $BS_FALSE ] && continue [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue - if [ -f /bin/systemctl ]; then + ## DGM if [ -f /bin/systemctl ]; then + if [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_TRUE ]; then # Debian 8 and above uses systemd /bin/systemctl stop salt-$fname > /dev/null 2>&1 /bin/systemctl start salt-$fname.service && continue @@ -3994,9 +4041,7 @@ install_debian_restart_daemons() { systemctl status salt-$fname.service journalctl -xe fi - fi - # if here, try using init.rc in case using container without systemd - if [ -f /etc/init.d/salt-$fname ]; then + elif [ -f /etc/init.d/salt-$fname ]; then # Still in SysV init /etc/init.d/salt-$fname stop > /dev/null 2>&1 /etc/init.d/salt-$fname start @@ -4018,7 +4063,8 @@ install_debian_check_services() { [ $fname = "minion" ] && [ "$_INSTALL_MINION" -eq $BS_FALSE ] && continue [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue - if [ -f /bin/systemctl ]; then + ## DGM if [ -f /bin/systemctl ]; then + if [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_TRUE ]; then __check_services_systemd salt-$fname || return 1 elif [ -f /etc/init.d/salt-$fname ]; then __check_services_debian salt-$fname || return 1 @@ -4056,7 +4102,7 @@ __install_saltstack_fedora_onedir_repository() { ## DGM GPG_KEY="SALT-PROJECT-GPG-PUBKEY-2023.pub" - YUM_REPO_FILE="/etc/yum.repos.d/salt.repo" + ## DGM YUM_REPO_FILE="/etc/yum.repos.d/salt.repo" if [ ! -s "$YUM_REPO_FILE" ] || [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]; then ## DGM FETCH_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/fedora/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/${ONEDIR_REV}" @@ -4218,12 +4264,8 @@ install_fedora_git_post() { systemctl is-enabled salt-$fname.service || (systemctl preset salt-$fname.service && systemctl enable salt-$fname.service) sleep 1 - systemctl daemon-reload && continue + systemctl daemon-reload - # if here, try using init.rc in case using container without systemd - if [ -f /etc/init.d/salt-$fname ]; then - update-rc.d salt-$fname defaults - fi done } @@ -4250,11 +4292,6 @@ install_fedora_restart_daemons() { systemctl status salt-$fname.service journalctl -xe fi - # if here, try using init.rc in case using container without systemd - if [ -f /etc/init.d/salt-$fname ]; then - /etc/init.d/salt-$fname stop > /dev/null 2>&1 - /etc/init.d/salt-$fname start && continue - fi done } @@ -4439,7 +4476,7 @@ __install_saltstack_rhel_onedir_repository() { ## DGM done ## DGM ## DGM yum clean metadata || return 1 - YUM_REPO_FILE="/etc/yum.repos.d/salt.repo" + ## DGM YUM_REPO_FILE="/etc/yum.repos.d/salt.repo" if [ ! -s "$YUM_REPO_FILE" ] || [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]; then FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" @@ -4560,7 +4597,8 @@ install_centos_stable_post() { [ $fname = "minion" ] && [ "$_INSTALL_MINION" -eq $BS_FALSE ] && continue [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue - if [ -f /bin/systemctl ]; then + ## DGM if [ -f /bin/systemctl ]; then + if [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_TRUE ]; then /bin/systemctl is-enabled salt-${fname}.service > /dev/null 2>&1 || ( /bin/systemctl preset salt-${fname}.service > /dev/null 2>&1 && /bin/systemctl enable salt-${fname}.service > /dev/null 2>&1 @@ -4668,16 +4706,16 @@ install_centos_git_post() { else _SERVICE_FILE="${_SALT_GIT_CHECKOUT_DIR}/pkg/rpm/salt-${fname}.service" fi - if [ -f /bin/systemctl ]; then + + ## DGM if [ -f /bin/systemctl ]; then + if [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_TRUE ]; then if [ ! -f "/usr/lib/systemd/system/salt-${fname}.service" ] || \ { [ -f "/usr/lib/systemd/system/salt-${fname}.service" ] && [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]; }; then __copyfile "${_SERVICE_FILE}" /usr/lib/systemd/system fi SYSTEMD_RELOAD=$BS_TRUE - fi - # ensure init.rc in case using container without systemd full support - if [ ! -f "/etc/init.d/salt-$fname" ] || \ + elif [ ! -f "/etc/init.d/salt-$fname" ] || \ { [ -f "/etc/init.d/salt-$fname" ] && [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]; }; then __copyfile "${_SALT_GIT_CHECKOUT_DIR}/pkg/rpm/salt-${fname}" /etc/init.d chmod +x /etc/init.d/salt-${fname} @@ -4779,7 +4817,8 @@ install_centos_onedir_post() { [ $fname = "minion" ] && [ "$_INSTALL_MINION" -eq $BS_FALSE ] && continue [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue - if [ -f /bin/systemctl ]; then + ## DGM if [ -f /bin/systemctl ]; then + if [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_TRUE ]; then /bin/systemctl is-enabled salt-${fname}.service > /dev/null 2>&1 || ( /bin/systemctl preset salt-${fname}.service > /dev/null 2>&1 && /bin/systemctl enable salt-${fname}.service > /dev/null 2>&1 @@ -4814,26 +4853,28 @@ install_centos_restart_daemons() { [ $fname = "minion" ] && [ "$_INSTALL_MINION" -eq $BS_FALSE ] && continue [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue - if [ -f /sbin/initctl ] && [ -f /etc/init/salt-${fname}.conf ]; then - # We have upstart support and upstart knows about our service - if ! /sbin/initctl status salt-$fname > /dev/null 2>&1; then - # Everything is in place and upstart gave us an error code? Fail! - return 1 - fi - - # upstart knows about this service. - # Let's try to stop it, and then start it - /sbin/initctl stop salt-$fname > /dev/null 2>&1 - # Restart service - if ! /sbin/initctl start salt-$fname > /dev/null 2>&1; then - # Failed the restart?! - return 1 - fi - elif [ -f /etc/init.d/salt-$fname ]; then + ## DGM if [ -f /sbin/initctl ] && [ -f /etc/init/salt-${fname}.conf ]; then + ## DGM # We have upstart support and upstart knows about our service + ## DGM if ! /sbin/initctl status salt-$fname > /dev/null 2>&1; then + ## DGM # Everything is in place and upstart gave us an error code? Fail! + ## DGM return 1 + ## DGM fi + + ## DGM # upstart knows about this service. + ## DGM # Let's try to stop it, and then start it + ## DGM /sbin/initctl stop salt-$fname > /dev/null 2>&1 + ## DGM # Restart service + ## DGM if ! /sbin/initctl start salt-$fname > /dev/null 2>&1; then + ## DGM # Failed the restart?! + ## DGM return 1 + ## DGM fi + ## DGM elif [ -f /etc/init.d/salt-$fname ]; then + if [ -f /etc/init.d/salt-$fname ]; then # Disable stdin to fix shell session hang on killing tee pipe service salt-$fname stop < /dev/null > /dev/null 2>&1 service salt-$fname start < /dev/null - elif [ -f /usr/bin/systemctl ]; then + ## DGM elif [ -f /usr/bin/systemctl ]; then + elif [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_TRUE ]; then # CentOS 7 uses systemd /usr/bin/systemctl stop salt-$fname > /dev/null 2>&1 /usr/bin/systemctl start salt-$fname.service && continue @@ -4889,7 +4930,8 @@ install_centos_check_services() { if [ -f "/etc/init.d/salt-$fname" ]; then __check_services_sysvinit "salt-$fname" || return 1 - elif [ -f /usr/bin/systemctl ]; then + ## DGM elif [ -f /usr/bin/systemctl ]; then + elif [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_TRUE ]; then __check_services_systemd "salt-$fname" || return 1 fi done @@ -6076,7 +6118,7 @@ install_amazon_linux_ami_2_deps() { ## DGM fi if [ $_DISABLE_REPOS -eq $BS_FALSE ] || [ "$_CUSTOM_REPO_URL" != "null" ]; then - YUM_REPO_FILE="/etc/yum.repos.d/salt.repo" + ## DGM YUM_REPO_FILE="/etc/yum.repos.d/salt.repo" if [ ! -s "${YUM_REPO_FILE}" ]; then FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" @@ -6155,7 +6197,7 @@ install_amazon_linux_ami_2_onedir_deps() { ## DGM fi if [ $_DISABLE_REPOS -eq $BS_FALSE ] || [ "$_CUSTOM_REPO_URL" != "null" ]; then - YUM_REPO_FILE="/etc/yum.repos.d/salt.repo" + ## DGM YUM_REPO_FILE="/etc/yum.repos.d/salt.repo" if [ ! -s "${YUM_REPO_FILE}" ]; then FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" @@ -6317,7 +6359,7 @@ install_amazon_linux_ami_2023_onedir_deps() { ## DGM fi ## DGM fi if [ $_DISABLE_REPOS -eq $BS_FALSE ] || [ "$_CUSTOM_REPO_URL" != "null" ]; then - YUM_REPO_FILE="/etc/yum.repos.d/salt.repo" + ## DGM YUM_REPO_FILE="/etc/yum.repos.d/salt.repo" if [ ! -s "${YUM_REPO_FILE}" ]; then FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" @@ -6530,7 +6572,8 @@ install_arch_linux_post() { # Skip salt-api since the service should be opt-in and not necessarily started on boot [ $fname = "api" ] && continue - if [ -f /usr/bin/systemctl ]; then + ## DGM if [ -f /usr/bin/systemctl ]; then + if [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_TRUE ]; then # Using systemd /usr/bin/systemctl is-enabled salt-$fname.service > /dev/null 2>&1 || ( /usr/bin/systemctl preset salt-$fname.service > /dev/null 2>&1 && @@ -6561,7 +6604,8 @@ install_arch_linux_git_post() { _SERVICE_DIR="${_SALT_GIT_CHECKOUT_DIR}/pkg/rpm" fi - if [ -f /usr/bin/systemctl ]; then + ## DGM if [ -f /usr/bin/systemctl ]; then + if [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_TRUE ]; then __copyfile "${_SERVICE_DIR}/salt-${fname}.service" "/lib/systemd/system/salt-${fname}.service" # Skip salt-api since the service should be opt-in and not necessarily started on boot @@ -6594,7 +6638,8 @@ install_arch_linux_restart_daemons() { [ $fname = "minion" ] && [ "$_INSTALL_MINION" -eq $BS_FALSE ] && continue [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue - if [ -f /usr/bin/systemctl ]; then + ## DGM if [ -f /usr/bin/systemctl ]; then + if [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_TRUE ]; then /usr/bin/systemctl stop salt-$fname.service > /dev/null 2>&1 /usr/bin/systemctl start salt-$fname.service && continue echodebug "Failed to start salt-$fname using systemd" @@ -6610,7 +6655,8 @@ install_arch_linux_restart_daemons() { } install_arch_check_services() { - if [ ! -f /usr/bin/systemctl ]; then + ## DGM if [ ! -f /usr/bin/systemctl ]; then + if [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_TRUE ]; then # Not running systemd!? Don't check! return 0 fi @@ -6666,7 +6712,7 @@ __install_saltstack_photon_onedir_repository() { fi ## DGM __PY_VERSION_REPO="py3" - YUM_REPO_FILE="/etc/yum.repos.d/salt.repo" + ## DGM YUM_REPO_FILE="/etc/yum.repos.d/salt.repo" if [ ! -s "$YUM_REPO_FILE" ] || [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]; then ## DGM ## salt repo 4 & 5 have issues, need the Major version dot Zero, eg: 4.0, 5.0 @@ -7190,7 +7236,8 @@ install_opensuse_stable_post() { [ $fname = "minion" ] && [ "$_INSTALL_MINION" -eq $BS_FALSE ] && continue [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue - if [ -f /bin/systemctl ] || [ -f /usr/bin/systemctl ]; then + ## DGM if [ -f /bin/systemctl ] || [ -f /usr/bin/systemctl ]; then + if [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_TRUE ]; then systemctl is-enabled salt-$fname.service || (systemctl preset salt-$fname.service && systemctl enable salt-$fname.service) sleep 1 systemctl daemon-reload @@ -7213,7 +7260,8 @@ install_opensuse_git_post() { [ $fname = "minion" ] && [ "$_INSTALL_MINION" -eq $BS_FALSE ] && continue [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue - if command -v systemctl; then + ## DGM if command -v systemctl; then + if [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_TRUE ]; then use_usr_lib=$BS_FALSE if [ "${DISTRO_MAJOR_VERSION}" -ge 15 ]; then @@ -7265,7 +7313,8 @@ install_opensuse_restart_daemons() { [ $fname = "minion" ] && [ "$_INSTALL_MINION" -eq $BS_FALSE ] && continue [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue - if [ -f /bin/systemctl ]; then + ## DGM if [ -f /bin/systemctl ]; then + if [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_TRUE ]; then systemctl stop salt-$fname > /dev/null 2>&1 systemctl start salt-$fname.service && continue echodebug "Failed to start salt-$fname using systemd" @@ -7281,7 +7330,8 @@ install_opensuse_restart_daemons() { } install_opensuse_check_services() { - if [ ! -f /bin/systemctl ]; then + ## DGM if [ ! -f /bin/systemctl ]; then + if [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_TRUE ]; then # Not running systemd!? Don't check! return 0 fi From 6335ea1eabd99cc7b1fbbe27e10e314407f59b06 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 6 Nov 2024 11:02:23 -0700 Subject: [PATCH 022/173] Updated debugging output --- bootstrap-salt.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 9ef7ec686..f26bdeed9 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -144,8 +144,8 @@ __check_services_systemd_functional() { # shellcheck disable=SC2034 _SYSTEMD_HELP="$(systemctl --help)" else - echoerror "systemctl: command not found, assume systemd not implemented" _SYSTEMD_FUNCTIONAL=$BS_FALSE + echoerror "systemctl: command not found, assume systemd not implemented, _SYSTEMD_FUNCTIONAL $_SYSTEMD_FUNCTIONAL" fi } # ---------- end of function __check_services_systemd_functional ---------- @@ -2483,9 +2483,10 @@ __check_services_systemd() { fi _SYSTEMD_ACTIVE=$(/bin/systemctl daemon-reload 2>&1 | grep 'System has not been booted with systemd') + echodebug "__check_services_systemd _SYSTEMD_ACTIVE result ,$_SYSTEMD_ACTIVE," if [ "$_SYSTEMD_ACTIVE" != "" ]; then - echodebug "systemd is not functional, despite systemctl being present" - _SYSTEMD_FUNCTIONAL="$BS_FALSE" + _SYSTEMD_FUNCTIONAL=$BS_FALSE + echodebug "systemd is not functional, despite systemctl being present, setting _SYSTEMD_FUNCTIONAL false, $_SYSTEMD_FUNCTIONAL" return 1 fi From 687144336a57ec6d1a29da80a1b696aa999ea689 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 6 Nov 2024 17:19:43 -0700 Subject: [PATCH 023/173] Saving functioning Linux shell script with debug output --- bootstrap-salt.sh | 120 ++++++++++++++++++++++++---------------------- 1 file changed, 63 insertions(+), 57 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index f26bdeed9..65ce4f2f4 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -26,7 +26,7 @@ #====================================================================================================================== set -o nounset # Treat unset variables as an error -__ScriptVersion="2024.11.05" +__ScriptVersion="2024.11.06" __ScriptName="bootstrap-salt.sh" __ScriptFullName="$0" @@ -645,17 +645,20 @@ if [ "$ITYPE" = "git" ]; then # If doing stable install, check if version specified elif [ "$ITYPE" = "stable" ]; then if [ "$#" -eq 0 ];then + STABLE_REV="latest" ONEDIR_REV="latest" _ONEDIR_REV="latest" ITYPE="onedir" else if [ "$(echo "$1" | grep -E '^(latest|3006|3007)$')" != "" ]; then + STABLE_REV="$1" ONEDIR_REV="$1" _ONEDIR_REV="$1" ITYPE="onedir" shift elif [ "$(echo "$1" | grep -E '^([3-9][0-5]{2}[5-9](\.[0-9]*)?)')" != "" ]; then ## DGM ONEDIR_REV="minor/$1" don't have minor directory anymore + STABLE_REV="$1" ONEDIR_REV="$1" _ONEDIR_REV="$1" ITYPE="onedir" @@ -669,13 +672,16 @@ elif [ "$ITYPE" = "stable" ]; then elif [ "$ITYPE" = "onedir" ]; then if [ "$#" -eq 0 ];then ONEDIR_REV="latest" + STABLE_REV="latest" else if [ "$(echo "$1" | grep -E '^(latest|3006|3007)$')" != "" ]; then ONEDIR_REV="$1" + STABLE_REV="$1" shift elif [ "$(echo "$1" | grep -E '^([3-9][0-9]{3}(\.[0-9]*)?)')" != "" ]; then ## DGM ONEDIR_REV="minor/$1" don't have minor directory anymore ONEDIR_REV="$1" + STABLE_REV="$1" shift else echo "Unknown onedir version: $1 (valid: 3006, 3007, latest.)" @@ -3141,14 +3147,14 @@ __install_saltstack_ubuntu_onedir_repository() { __apt_key_fetch "${HTTP_VAL}://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" || return 1 __wait_for_apt apt-get update || return 1 - if [ "$STABLE_REV" != "latest" ]; then + if [ "$ONEDIR_REV" != "latest" ]; then # latest is default - STABLE_REV_MAJOR=$(echo "$STABLE_REV" | cut -d '.' -f 1) - if [ "$STABLE_REV_MAJOR" -eq "3006" ]; then + ONEDIR_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) + if [ "$ONEDIR_REV_MAJOR" -eq "3006" ]; then echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 echo "Pin: version 3006.*" >> /etc/apt/preferences.d/salt-pin-1001 echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 - elif [ "$STABLE_REV_MAJOR" -eq "3007" ]; then + elif [ "$ONEDIR_REV_MAJOR" -eq "3007" ]; then echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 echo "Pin: version 3007.*" >> /etc/apt/preferences.d/salt-pin-1001 echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 @@ -3721,14 +3727,14 @@ __install_saltstack_debian_onedir_repository() { __apt_key_fetch "${HTTP_VAL}://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" || return 1 __wait_for_apt apt-get update || return 1 - if [ "$STABLE_REV" != "latest" ]; then + if [ "$ONEDIR_REV" != "latest" ]; then # latest is default - STABLE_REV_MAJOR=$(echo "$STABLE_REV" | cut -d '.' -f 1) - if [ "$STABLE_REV_MAJOR" -eq "3006" ]; then + ONEDIR_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) + if [ "$ONEDIR_REV_MAJOR" -eq "3006" ]; then echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 echo "Pin: version 3006.*" >> /etc/apt/preferences.d/salt-pin-1001 echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 - elif [ "$STABLE_REV_MAJOR" -eq "3007" ]; then + elif [ "$ONEDIR_REV_MAJOR" -eq "3007" ]; then echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 echo "Pin: version 3007.*" >> /etc/apt/preferences.d/salt-pin-1001 echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 @@ -4088,11 +4094,11 @@ __install_saltstack_fedora_onedir_repository() { set -v set -x - if [ "$ITYPE" = "stable" ]; then - REPO_REV="$ONEDIR_REV" - else - REPO_REV="latest" - fi + ## DGM if [ "$ITYPE" = "stable" ]; then + ## DGM REPO_REV="$ONEDIR_REV" + ## DGM else + ## DGM REPO_REV="latest" + ## DGM fi if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -ne 3 ]; then echoerror "Python version is no longer supported, only Python 3" @@ -4116,9 +4122,9 @@ __install_saltstack_fedora_onedir_repository() { FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" - if [ "$REPO_REV" != "latest" ]; then + if [ "$ONEDIR_REV" != "latest" ]; then # 3006.x is default - REPO_REV_MAJOR=$(echo "$REPO_REV" | cut -d '.' -f 1) + REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) if [ "$REPO_REV_MAJOR" -eq "3007" ]; then # Enable the Salt 3007 STS repo dnf config-manager --set-disable salt-repo-* @@ -4131,9 +4137,9 @@ __install_saltstack_fedora_onedir_repository() { fi dnf clean expire-cache || return 1 - elif [ "$REPO_REV" != "latest" ]; then + elif [ "$ONEDIR_REV" != "latest" ]; then echowarn "salt.repo already exists, ignoring salt version argument." - echowarn "Use -F (forced overwrite) to install $REPO_REV." + echowarn "Use -F (forced overwrite) to install $ONEDIR_REV." fi return 0 @@ -4429,11 +4435,11 @@ __install_saltstack_rhel_onedir_repository() { set -v set -x - if [ "$ITYPE" = "stable" ]; then - REPO_REV="$ONEDIR_REV" - else - REPO_REV="latest" - fi + ## DGM if [ "$ITYPE" = "stable" ]; then + ## DGM REPO_REV="$ONEDIR_REV" + ## DGM else + ## DGM REPO_REV="latest" + ## DGM fi if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -ne 3 ]; then echoerror "Python version is no longer supported, only Python 3" @@ -4482,9 +4488,9 @@ __install_saltstack_rhel_onedir_repository() { if [ ! -s "$YUM_REPO_FILE" ] || [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]; then FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" - if [ "$REPO_REV" != "latest" ]; then + if [ "$ONEDIR_REV" != "latest" ]; then # 3006.x is default - REPO_REV_MAJOR=$(echo "$REPO_REV" | cut -d '.' -f 1) + REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) if [ "$REPO_REV_MAJOR" -eq "3007" ]; then # Enable the Salt 3007 STS repo dnf config-manager --set-disable salt-repo-* @@ -4496,9 +4502,9 @@ __install_saltstack_rhel_onedir_repository() { dnf config-manager --set-enabled salt-repo-latest fi dnf clean expire-cache || return 1 - elif [ "$REPO_REV" != "latest" ]; then + elif [ "$ONEDIR_REV" != "latest" ]; then echowarn "salt.repo already exists, ignoring salt version argument." - echowarn "Use -F (forced overwrite) to install $REPO_REV." + echowarn "Use -F (forced overwrite) to install $ONEDIR_REV." fi return 0 @@ -6078,11 +6084,11 @@ install_amazon_linux_ami_2_deps() { return 1 fi - if [ "$ITYPE" = "stable" ]; then - REPO_REV="$STABLE_REV" - else - REPO_REV="latest" - fi + ## DGM if [ "$ITYPE" = "stable" ]; then + ## DGM REPO_REV="$STABLE_REV" + ## DGM else + ## DGM REPO_REV="latest" + ## DGM fi # We need to install yum-utils before doing anything else when installing on # Amazon Linux ECS-optimized images. See issue #974. @@ -6123,9 +6129,9 @@ install_amazon_linux_ami_2_deps() { if [ ! -s "${YUM_REPO_FILE}" ]; then FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" - if [ "$REPO_REV" != "latest" ]; then + if [ "$STABLE_REV" != "latest" ]; then # 3006.x is default - REPO_REV_MAJOR=$(echo "$REPO_REV" | cut -d '.' -f 1) + REPO_REV_MAJOR=$(echo "$STABLE_REV" | cut -d '.' -f 1) if [ "$REPO_REV_MAJOR" -eq "3007" ]; then # Enable the Salt 3007 STS repo dnf config-manager --set-disable salt-repo-* @@ -6153,11 +6159,11 @@ install_amazon_linux_ami_2_onedir_deps() { return 1 fi - if [ "$ITYPE" = "onedir" ]; then - REPO_REV="$ONEDIR_REV" - else - REPO_REV="latest" - fi + ## DGM if [ "$ITYPE" = "onedir" ]; then + ## DGM REPO_REV="$ONEDIR_REV" + ## DGM else + ## DGM REPO_REV="latest" + ## DGM fi # We need to install yum-utils before doing anything else when installing on # Amazon Linux ECS-optimized images. See issue #974. @@ -6202,9 +6208,9 @@ install_amazon_linux_ami_2_onedir_deps() { if [ ! -s "${YUM_REPO_FILE}" ]; then FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" - if [ "$REPO_REV" != "latest" ]; then + if [ "$ONEDIR_REV" != "latest" ]; then # 3006.x is default - REPO_REV_MAJOR=$(echo "$REPO_REV" | cut -d '.' -f 1) + REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) if [ "$REPO_REV_MAJOR" -eq "3007" ]; then # Enable the Salt 3007 STS repo dnf config-manager --set-disable salt-repo-* @@ -6315,11 +6321,11 @@ install_amazon_linux_ami_2023_git_deps() { } install_amazon_linux_ami_2023_onedir_deps() { - if [ "$ITYPE" = "onedir" ]; then - REPO_REV="$ONEDIR_REV" - else - REPO_REV="latest" - fi + ## DGM if [ "$ITYPE" = "onedir" ]; then + ## DGM REPO_REV="$ONEDIR_REV" + ## DGM else + ## DGM REPO_REV="latest" + ## DGM fi # We need to install yum-utils before doing anything else when installing on # Amazon Linux ECS-optimized images. See issue #974. @@ -6364,9 +6370,9 @@ install_amazon_linux_ami_2023_onedir_deps() { if [ ! -s "${YUM_REPO_FILE}" ]; then FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" - if [ "$REPO_REV" != "latest" ]; then + if [ "$ONEDIR_REV" != "latest" ]; then # 3006.x is default - REPO_REV_MAJOR=$(echo "$REPO_REV" | cut -d '.' -f 1) + REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) if [ "$REPO_REV_MAJOR" -eq "3007" ]; then # Enable the Salt 3007 STS repo dnf config-manager --set-disable salt-repo-* @@ -6706,11 +6712,11 @@ __install_saltstack_photon_onedir_repository() { return 1 fi - if [ "$ITYPE" = "stable" ]; then - REPO_REV="$ONEDIR_REV" - else - REPO_REV="latest" - fi + ## DGM if [ "$ITYPE" = "stable" ]; then + ## DGM REPO_REV="$ONEDIR_REV" + ## DGM else + ## DGM REPO_REV="latest" + ## DGM fi ## DGM __PY_VERSION_REPO="py3" ## DGM YUM_REPO_FILE="/etc/yum.repos.d/salt.repo" @@ -6727,9 +6733,9 @@ __install_saltstack_photon_onedir_repository() { FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" - if [ "$REPO_REV" != "latest" ]; then + if [ "$ONEDIR_REV" != "latest" ]; then # 3006.x is default - REPO_REV_MAJOR=$(echo "$REPO_REV" | cut -d '.' -f 1) + REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) if [ "$REPO_REV_MAJOR" -eq "3007" ]; then # Enable the Salt 3007 STS repo dnf config-manager --set-disable salt-repo-* @@ -6741,9 +6747,9 @@ __install_saltstack_photon_onedir_repository() { dnf config-manager --set-enabled salt-repo-latest fi tdnf makecache || return 1 - elif [ "$REPO_REV" != "latest" ]; then + elif [ "$ONEDIR_REV" != "latest" ]; then echowarn "salt.repo already exists, ignoring salt version argument." - echowarn "Use -F (forced overwrite) to install $REPO_REV." + echowarn "Use -F (forced overwrite) to install $ONEDIR_REV." fi return 0 From 1b8b9076365c624137d44f22c5e4abe8afb1eb36 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 6 Nov 2024 17:55:51 -0700 Subject: [PATCH 024/173] Removed debugging apart from macos latest pick, disabled CI/CI testing --- .github/workflows/ci.yml | 2 - .github/workflows/templates/ci.yml | 2 - .github/workflows/test-linux.yml | 50 +- .github/workflows/test-macos.yml | 54 +- .github/workflows/test-windows.yml | 59 -- .pre-commit-config.yaml | 7 - bootstrap-salt.sh | 798 +------------------------ salt-quick-start.sh | 5 - tests/conftest.py | 1 - tests/integration/test_installation.py | 1 - tests/requirements.txt | 6 - 11 files changed, 24 insertions(+), 961 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d8539b20a..fc4b306f0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,8 +27,6 @@ jobs: separator: "," files: | bootstrap-salt.* - ## DGM Gemfile - ## DGM kitchen*.yml tests/** .github/workflows/** files_ignore: | diff --git a/.github/workflows/templates/ci.yml b/.github/workflows/templates/ci.yml index 7f99aa1b2..c3c029c98 100644 --- a/.github/workflows/templates/ci.yml +++ b/.github/workflows/templates/ci.yml @@ -27,8 +27,6 @@ jobs: separator: "," files: | bootstrap-salt.* - ## DGM Gemfile - ## DGM kitchen*.yml tests/** .github/workflows/** files_ignore: | diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 75c80a8dd..d31ef758f 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -26,7 +26,7 @@ on: jobs: Test: name: ${{ matrix.instance }} - ## DGM runs-on: ubuntu-20.04 + ## runs-on: ubuntu-20.04 runs-on: ubuntu-latest container: image: ghcr.io/saltstack/salt-ci-containers/packaging:ubuntu-22.04 @@ -41,52 +41,18 @@ jobs: steps: - uses: actions/checkout@v4 - ## DGM - name: Setup Ruby - ## DGM uses: ruby/setup-ruby@v1 - ## DGM with: - ## DGM ruby-version: 2.6.10 - ## DGM ## ruby-version: 3.3.4 - ## DGM bundler-cache: true - - ## DGM - name: Set up Python 3.10 - ## DGM uses: actions/setup-python@v5 - ## DGM with: - ## DGM python-version: "3.10" - - ## DGM - name: Install Python Dependencies - ## DGM run: | - ## DGM python3 -m pip install -U pip - ## DGM python3 -m pip install -r tests/requirements.txt - - ## DGM - name: Create Test Instance - ## DGM run: | - ## DGM bundle exec kitchen create ${{ matrix.instance }}-${{ inputs.distro-slug }} || \ - ## DGM (sleep 10 && bundle exec kitchen create ${{ matrix.instance }}-${{ inputs.distro-slug }}) - ## DGM sleep 2 - - ## DGM - name: Test Bootstrap - ## DGM run: | - ## DGM bundle exec kitchen verify ${{ matrix.instance }}-${{ inputs.distro-slug }} - ## DGM sleep 2 - - ## DGM - name: Destroy Test Instance - ## DGM if: always() - ## DGM run: | - ## DGM bundle exec kitchen destroy ${{ matrix.instance }}-${{ inputs.distro-slug }} - ## DGM sleep 2 - - name: Install Pytest run: | pip install -U pytest - - name: Bootstrap Salt - run: | - ## sudo sh -x ./bootstrap-salt.sh - sh -x ./bootstrap-salt.sh + ## TBD Disabled until CI/CD is functional + ## - name: Bootstrap Salt + ## run: | + ## sh -x ./bootstrap-salt.sh - - name: Test Bootstrap - run: | - pytest --cache-clear -v -s -ra --log-cli-level=debug tests/integration/ + ## - name: Test Bootstrap + ## run: | + ## pytest --cache-clear -v -s -ra --log-cli-level=debug tests/integration/ - name: Set Exit Status if: always() diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index 7d201a9ae..d07416d3d 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -28,14 +28,11 @@ on: default: 20 description: The timeout(in minutes) for the workflow -## DGM env: -## DGM KITCHEN_LOCAL_YAML: 'kitchen.macos.yml' - jobs: Test: name: ${{ matrix.instance }} - ## DGM runs-on: ${{ inputs.runs-on }} - runs-on: macos-12 + ## runs-on: ${{ inputs.runs-on }} + runs-on: macos-13 timeout-minutes: ${{ inputs.timeout }} strategy: fail-fast: false @@ -45,51 +42,18 @@ jobs: steps: - uses: actions/checkout@v4 - ## DGM - name: Setup Ruby - ## DGM uses: ruby/setup-ruby@v1 - ## DGM with: - ## DGM ruby-version: 2.6.10 - ## DGM ## ruby-version: 3.3.4 - ## DGM bundler-cache: true - - ## DGM - name: Set up Python 3.10 - ## DGM uses: actions/setup-python@v5 - ## DGM with: - ## DGM python-version: "3.10" - - ## DGM - name: Install Python Dependencies - ## DGM run: | - ## DGM pip install -U pip - ## DGM pip install -r tests/requirements.txt - - ## DGM - name: Create Test Instance - ## DGM run: | - ## DGM bundle exec kitchen create ${{ matrix.instance }}-${{ inputs.distro-slug }} || \ - ## DGM (sleep 10 && bundle exec kitchen create ${{ matrix.instance }}-${{ inputs.distro-slug }}) - ## DGM sleep 2 - - ## DGM - name: Test Bootstrap - ## DGM run: | - ## DGM bundle exec kitchen verify ${{ matrix.instance }}-${{ inputs.distro-slug }} - ## DGM sleep 2 - - ## DGM - name: Destroy Test Instance - ## DGM if: always() - ## DGM run: | - ## DGM bundle exec kitchen destroy ${{ matrix.instance }}-${{ inputs.distro-slug }} - ## DGM sleep 2 - - name: Install Pytest run: | pip install -U pytest - - name: Bootstrap Salt - run: | - sudo sh -x ./bootstrap-salt.sh + ## TBD Disabled until CI/CD is functional + ## - name: Bootstrap Salt + ## run: | + ## sudo sh -x ./bootstrap-salt.sh - - name: Test Bootstrap - run: | - pytest --cache-clear -v -s -ra --log-cli-level=debug tests/integration/ + ## - name: Test Bootstrap + ## run: | + ## pytest --cache-clear -v -s -ra --log-cli-level=debug tests/integration/ - name: Set Exit Status if: always() diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index 8613744eb..4bc98f4d4 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -28,16 +28,9 @@ on: default: 20 description: The timeout(in minutes) for the workflow -## DGM env: -## DGM machine_user: kitchen -## DGM machine_pass: Password1 -## DGM machine_port: 5985 -## DGM KITCHEN_LOCAL_YAML: 'kitchen.windows.yml' - jobs: Test: name: ${{ matrix.instance }} - ## DGM runs-on: ${{ inputs.runs-on }} runs-on: windows-latest timeout-minutes: ${{ inputs.timeout }} strategy: @@ -48,68 +41,16 @@ jobs: steps: - uses: actions/checkout@v4 - ## DGM - name: Setup Ruby - ## DGM uses: ruby/setup-ruby@v1 - ## DGM with: - ## DGM ruby-version: 2.6.10 - ## DGM ## ruby-version: 3.3.4 - ## DGM bundler-cache: true - - ## DGM - name: Install Chef - ## DGM uses: actionshub/chef-install@1.1.0 - ## DGM with: - ## DGM project: chef - ## DGM version: 16.10.8 - - ## DGM - name: Add Chef bindir to PATH - ## DGM uses: myci-actions/export-env-var-powershell@1 - ## DGM with: - ## DGM name: PATH - ## DGM value: "C:\\opscode\\chef\\bin;C:\\opscode\\chef\\embedded\\bin;$env:PATH" - - ## DGM - name: Setup test user - ## DGM run: | - ## DGM $password = ConvertTo-SecureString $env:machine_pass -AsPlainText -Force - ## DGM New-LocalUser $env:machine_user -Password $password - ## DGM Add-LocalGroupMember -Group "Administrators" -Member $env:machine_user - ## DGM Get-LocalUser - ## DGM Get-LocalGroupMember -Group "Administrators" - - ## DGM - name: Set up WinRM - ## DGM run: | - ## DGM Set-WSManQuickConfig -Force - ## DGM winrm set winrm/config/service '@{AllowUnencrypted="True"}' - ## DGM env - name: Set up Python 3.10 uses: actions/setup-python@v5 with: python-version: "3.10" - ## DGM - name: Install Python Dependencies - name: Install Pytest run: | - ## DGM pip install -U pip - ## DGM pip install -r tests/requirements.txt pip install -U pytest - ## DGM - name: Create Test Instance - ## DGM run: | - ## DGM bundle exec kitchen create ${{ matrix.instance }}-${{ inputs.distro-slug }} - ## DGM sleep 2 - - ## DGM - name: Test Bootstrap - ## DGM run: | - ## DGM env - ## DGM bundle exec kitchen verify ${{ matrix.instance }}-${{ inputs.distro-slug }} - ## DGM sleep 2 - - ## DGM - name: Destroy Test Instance - ## DGM if: always() - ## DGM run: | - ## DGM bundle exec kitchen destroy ${{ matrix.instance }}-${{ inputs.distro-slug }} - ## DGM sleep 2 - - name: Bootstrap Salt run: | . ./bootstrap-salt.ps1 -RunService $false diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 033ac67b4..85e4d1787 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -37,13 +37,6 @@ repos: args: - requirements/release.in -## DGM - repo: https://github.com/asottile/pyupgrade -## DGM rev: v3.15.0 -## DGM hooks: -## DGM - id: pyupgrade -## DGM name: Rewrite Code to be Py3.10+ -## DGM args: [--py310-plus] - - repo: https://github.com/asottile/pyupgrade rev: v3.15.0 hooks: diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 65ce4f2f4..47e596a3a 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -129,9 +129,6 @@ __check_command_exists() { # DESCRIPTION: Set _SYSTEMD_FUNCTIONAL = BS_TRUE or BS_FALSE case where systemd is functional (for example: container may not have systemd) #---------------------------------------------------------------------------------------------------------------------- __check_services_systemd_functional() { - # DGM debug - set -v - set -x # check if systemd is functional, having systemctl present is insufficient @@ -292,9 +289,6 @@ _CUSTOM_REPO_URL="null" _CUSTOM_MASTER_CONFIG="null" _CUSTOM_MINION_CONFIG="null" _QUIET_GIT_INSTALLATION=$BS_FALSE -## DGM _REPO_URL="repo.saltproject.io" -## DGM _ONEDIR_DIR="salt" -## DGM _ONEDIR_NIGHTLY_DIR="salt-dev/${_ONEDIR_DIR}" _REPO_URL="packages.broadcom.com/artifactory" _PY_EXE="python3" _MINIMUM_PIP_VERSION="9.0.1" @@ -657,7 +651,6 @@ elif [ "$ITYPE" = "stable" ]; then ITYPE="onedir" shift elif [ "$(echo "$1" | grep -E '^([3-9][0-5]{2}[5-9](\.[0-9]*)?)')" != "" ]; then - ## DGM ONEDIR_REV="minor/$1" don't have minor directory anymore STABLE_REV="$1" ONEDIR_REV="$1" _ONEDIR_REV="$1" @@ -679,7 +672,6 @@ elif [ "$ITYPE" = "onedir" ]; then STABLE_REV="$1" shift elif [ "$(echo "$1" | grep -E '^([3-9][0-9]{3}(\.[0-9]*)?)')" != "" ]; then - ## DGM ONEDIR_REV="minor/$1" don't have minor directory anymore ONEDIR_REV="$1" STABLE_REV="$1" shift @@ -707,13 +699,13 @@ elif [ "$ITYPE" = "onedir_rc" ]; then ## elif [ "$(echo "$1" | grep -E '^([3-9][0-9]{3}?rc[0-9]-[0-9]$)')" != "" ]; then ## # Handle the 3xxx.0 version as 3xxx archive (pin to minor) and strip the fake ".0" suffix ## #ONEDIR_REV=$(echo "$1" | sed -E 's/^([3-9][0-9]{3})\.0$/\1/') -## ## DGM ONEDIR_REV="minor/$1" don't have minor directory anymore +## ## ONEDIR_REV="minor/$1" don't have minor directory anymore ## ONEDIR_REV="$1" ## shift ## elif [ "$(echo "$1" | grep -E '^([3-9][0-9]{3}\.[0-9]?rc[0-9]$)')" != "" ]; then ## # Handle the 3xxx.0 version as 3xxx archive (pin to minor) and strip the fake ".0" suffix ## #ONEDIR_REV=$(echo "$1" | sed -E 's/^([3-9][0-9]{3})\.0$/\1/') -## ## DGM ONEDIR_REV="minor/$1" don't have minor directory anymore +## ## ONEDIR_REV="minor/$1" don't have minor directory anymore ## ONEDIR_REV="$1" ## shift ## else @@ -808,7 +800,6 @@ fi PY_PKG_VER=3 _PY_PKG_VER="python3" _PY_MAJOR_VERSION="3" -## DGM __PY_VERSION_REPO="py3" # Check if we're installing via a different Python executable and set major version variables if [ -n "$_PY_EXE" ]; then @@ -944,9 +935,6 @@ __fetch_url() { # DESCRIPTION: Retrieves a URL, verifies its content and writes it to standard output #---------------------------------------------------------------------------------------------------------------------- __fetch_verify() { - # DGM debug - set -v - set -x fetch_verify_url="$1" fetch_verify_sum="$2" @@ -969,9 +957,6 @@ __fetch_verify() { # DESCRIPTION: Checks if a URL exists #---------------------------------------------------------------------------------------------------------------------- __check_url_exists() { - # DGM debug - set -v - set -x _URL="$1" if curl --output /dev/null --silent --fail "${_URL}"; then @@ -1396,10 +1381,6 @@ __check_dpkg_architecture() { return 1 fi - ## DGM __REPO_ARCH="$DPKG_ARCHITECTURE" - ## DGM __REPO_ARCH_DEB='deb [signed-by=/usr/share/keyrings/salt-archive-keyring.gpg]' - ## DGM TBD wonder what to do here given use of salt.sources ????, this value's use has been commented out - ## DGM for now __REPO_ARCH_DEB='deb [signed-by=/etc/apt/keyrings/salt-archive-keyring.gpg]' __return_code=0 case $DPKG_ARCHITECTURE in @@ -1414,10 +1395,6 @@ __check_dpkg_architecture() { "arm64") # Saltstack official repository has full arm64 support since 3006 error_msg="" - ## DGM __REPO_ARCH="arm64" - ## __REPO_ARCH_DEB="deb [signed-by=/usr/share/keyrings/salt-archive-keyring.gpg arch=$__REPO_ARCH]" - ## DGM TBD wonder what to do here given use of salt.sources ???, this value's use has been commented out? - ## DGM for now __REPO_ARCH_DEB="deb [signed-by=/etc/apt/keyrings/salt-archive-keyring.gpg arch=$__REPO_ARCH]" ;; "armhf") error_msg="$_REPO_URL doesn't have packages for your system architecture: $DPKG_ARCHITECTURE." @@ -1985,9 +1962,6 @@ __wait_for_apt(){ # PARAMETERS: packages #---------------------------------------------------------------------------------------------------------------------- __apt_get_install_noinput() { - # DGM debug - set -v - set -x __wait_for_apt apt-get install -y -o DPkg::Options::=--force-confold "${@}"; return $? } # ---------- end of function __apt_get_install_noinput ---------- @@ -1998,9 +1972,6 @@ __apt_get_install_noinput() { # DESCRIPTION: (DRY) apt-get upgrade with noinput options #---------------------------------------------------------------------------------------------------------------------- __apt_get_upgrade_noinput() { - # DGM debug - set -v - set -x __wait_for_apt apt-get upgrade -y -o DPkg::Options::=--force-confold; return $? } # ---------- end of function __apt_get_upgrade_noinput ---------- @@ -2032,16 +2003,11 @@ __temp_gpg_pub() { # PARAMETERS: url #---------------------------------------------------------------------------------------------------------------------- __apt_key_fetch() { - # DGM debug - set -v - set -x url=$1 tempfile="$(__temp_gpg_pub)" - ## DGM __fetch_url "$tempfile" "$url" || return 1 - ## DGM cp -f "$tempfile" /usr/share/keyrings/salt-archive-keyring.gpg && chmod 644 /usr/share/keyrings/salt-archive-keyring.gpg || return 1 __fetch_url "$tempfile" "$url" || return 1 mkdir -p /etc/apt/keyrings cp -f "$tempfile" /etc/apt/keyrings/salt-archive-keyring.pgp && chmod 644 /etc/apt/keyrings/salt-archive-keyring.pgp || return 1 @@ -2057,9 +2023,6 @@ __apt_key_fetch() { # PARAMETERS: url #---------------------------------------------------------------------------------------------------------------------- __rpm_import_gpg() { - # DGM debug - set -v - set -x url=$1 @@ -2084,9 +2047,6 @@ __rpm_import_gpg() { # DESCRIPTION: (DRY) yum install with noinput options #---------------------------------------------------------------------------------------------------------------------- __yum_install_noinput() { - # DGM debug - set -v - set -x if [ "$DISTRO_NAME_L" = "oracle_linux" ]; then # We need to install one package at a time because --enablerepo=X disables ALL OTHER REPOS!!!! @@ -2103,9 +2063,6 @@ __yum_install_noinput() { # DESCRIPTION: (DRY) dnf install with noinput options #---------------------------------------------------------------------------------------------------------------------- __dnf_install_noinput() { - # DGM debug - set -v - set -x dnf -y install "${@}" || return $? } # ---------- end of function __dnf_install_noinput ---------- @@ -2115,9 +2072,6 @@ __dnf_install_noinput() { # DESCRIPTION: (DRY) tdnf install with noinput options #---------------------------------------------------------------------------------------------------------------------- __tdnf_install_noinput() { - # DGM debug - set -v - set -x tdnf -y install "${@}" || return $? } # ---------- end of function __tdnf_install_noinput ---------- @@ -2128,9 +2082,6 @@ __tdnf_install_noinput() { # specific revision. #---------------------------------------------------------------------------------------------------------------------- __git_clone_and_checkout() { - # DGM debug - set -v - set -x echodebug "Installed git version: $(git --version | awk '{ print $3 }')" # Turn off SSL verification if -I flag was set for insecure downloads @@ -2470,9 +2421,6 @@ __overwriteconfig() { # PARAMETERS: servicename #---------------------------------------------------------------------------------------------------------------------- __check_services_systemd() { - # DGM debug - set -v - set -x if [ $# -eq 0 ]; then echoerror "You need to pass a service name to check!" @@ -2515,9 +2463,6 @@ __check_services_systemd() { # PARAMETERS: servicename #---------------------------------------------------------------------------------------------------------------------- __check_services_upstart() { - # DGM debug - set -v - set -x if [ $# -eq 0 ]; then echoerror "You need to pass a service name to check!" @@ -2546,9 +2491,6 @@ __check_services_upstart() { # PARAMETERS: servicename #---------------------------------------------------------------------------------------------------------------------- __check_services_sysvinit() { - # DGM debug - set -v - set -x if [ $# -eq 0 ]; then echoerror "You need to pass a service name to check!" @@ -2576,9 +2518,6 @@ __check_services_sysvinit() { # PARAMETERS: servicename #---------------------------------------------------------------------------------------------------------------------- __check_services_debian() { - # DGM debug - set -v - set -x if [ $# -eq 0 ]; then echoerror "You need to pass a service name to check!" @@ -2607,9 +2546,6 @@ __check_services_debian() { # PARAMETERS: servicename #---------------------------------------------------------------------------------------------------------------------- __check_services_openrc() { - # DGM debug - set -v - set -x if [ $# -eq 0 ]; then echoerror "You need to pass a service name to check!" @@ -2637,9 +2573,6 @@ __check_services_openrc() { # DESCRIPTION: Return 0 or 1 depending on successful creation of virtualenv #---------------------------------------------------------------------------------------------------------------------- __create_virtualenv() { - # DGM debug - set -v - set -x if [ ! -d "$_VIRTUALENV_DIR" ]; then echoinfo "Creating virtualenv ${_VIRTUALENV_DIR}" @@ -2658,9 +2591,6 @@ __create_virtualenv() { # DESCRIPTION: Return 0 or 1 depending on successful activation of virtualenv #---------------------------------------------------------------------------------------------------------------------- __activate_virtualenv() { - # DGM debug - set -v - set -x set +o nounset # Is virtualenv empty @@ -2759,9 +2689,6 @@ __install_pip_deps() { # PARAMETERS: py_exe #---------------------------------------------------------------------------------------------------------------------- __install_salt_from_repo() { - # DGM debug - set -v - set -x _py_exe="$1" @@ -3010,9 +2937,6 @@ fi # Ubuntu Install Functions # __enable_universe_repository() { - # DGM debug - set -v - set -x if [ "$(grep -R universe /etc/apt/sources.list /etc/apt/sources.list.d/ | grep -v '#')" != "" ]; then # The universe repository is already enabled @@ -3035,11 +2959,6 @@ __install_saltstack_ubuntu_repository() { { [ "$DISTRO_MAJOR_VERSION" -eq 24 ] && [ "$DISTRO_MINOR_VERSION" -eq 10 ]; } || \ [ "$DISTRO_MAJOR_VERSION" -eq 21 ] || [ "$DISTRO_MAJOR_VERSION" -eq 23 ] || [ "$DISTRO_MAJOR_VERSION" -eq 25 ]; then echowarn "Non-LTS Ubuntu detected, but stable packages requested. Trying packages for previous LTS release. You may experience problems." - ## DGM UBUNTU_VERSION=24.04 - ## DGM UBUNTU_CODENAME="noble" - ## DGM else - ## DGM UBUNTU_VERSION=${DISTRO_VERSION} - ## DGM UBUNTU_CODENAME=${DISTRO_CODENAME} fi # Install downloader backend for GPG keys fetching @@ -3069,10 +2988,6 @@ __install_saltstack_ubuntu_repository() { fi # SaltStack's stable Ubuntu repository: - ## DGM SALTSTACK_UBUNTU_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/ubuntu/${UBUNTU_VERSION}/${__REPO_ARCH}/${STABLE_REV}" - ## DGM echo "$__REPO_ARCH_DEB $SALTSTACK_UBUNTU_URL $UBUNTU_CODENAME main" > /etc/apt/sources.list.d/salt.list - ## DGM __apt_key_fetch "$SALTSTACK_UBUNTU_URL/SALT-PROJECT-GPG-PUBKEY-2023.gpg" || return 1 - __fetch_url "/etc/apt/sources.list.d/salt.sources" "https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.sources" __apt_key_fetch "${HTTP_VAL}://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" || return 1 __wait_for_apt apt-get update || return 1 @@ -3091,15 +3006,9 @@ __install_saltstack_ubuntu_repository() { fi fi - ## DGM _ONEDIR_TYPE="saltproject-deb" - ## DGM SALTSTACK_UBUNTU_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_TYPE}/pool/" - } __install_saltstack_ubuntu_onedir_repository() { - # DGM debug - set -v - set -x echodebug "__install_saltstack_ubuntu_onedir_repository() entry" # Workaround for latest non-LTS Ubuntu @@ -3107,11 +3016,6 @@ __install_saltstack_ubuntu_onedir_repository() { { [ "$DISTRO_MAJOR_VERSION" -eq 22 ] && [ "$DISTRO_MINOR_VERSION" -eq 10 ]; } || \ [ "$DISTRO_MAJOR_VERSION" -eq 21 ] || [ "$DISTRO_MAJOR_VERSION" -eq 23 ] || [ "$DISTRO_MAJOR_VERSION" -eq 25 ]; then echowarn "Non-LTS Ubuntu detected, but stable packages requested. Trying packages for previous LTS release. You may experience problems." - ## DGM UBUNTU_VERSION=24.04 - ## DGM UBUNTU_CODENAME="noble" - ## DGM else - ## DGM UBUNTU_VERSION=${DISTRO_VERSION} - ## DGM UBUNTU_CODENAME=${DISTRO_CODENAME} fi # Install downloader backend for GPG keys fetching @@ -3136,13 +3040,6 @@ __install_saltstack_ubuntu_onedir_repository() { __apt_get_install_noinput ${__PACKAGES} || return 1 # SaltStack's stable Ubuntu repository: - ## DGM TBD need deb location - ## DGM SALTSTACK_UBUNTU_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/ubuntu/${UBUNTU_VERSION}/${__REPO_ARCH}/${ONEDIR_REV}/" - ## DGM if [ "${ONEDIR_REV}" = "nightly" ] ; then - ## DGM SALTSTACK_UBUNTU_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/ubuntu/${UBUNTU_VERSION}/${__REPO_ARCH}/" - ## DGM fi - ## DGM echo "$__REPO_ARCH_DEB $SALTSTACK_UBUNTU_URL $UBUNTU_CODENAME main" > /etc/apt/sources.list.d/salt.list - __fetch_url "/etc/apt/sources.list.d/salt.sources" "https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.sources" __apt_key_fetch "${HTTP_VAL}://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" || return 1 __wait_for_apt apt-get update || return 1 @@ -3163,9 +3060,6 @@ __install_saltstack_ubuntu_onedir_repository() { } install_ubuntu_deps() { - # DGM debug - set -v - set -x echodebug "install_ubuntu_deps() entry" if [ "$_DISABLE_REPOS" -eq $BS_FALSE ]; then @@ -3224,9 +3118,6 @@ install_ubuntu_deps() { } install_ubuntu_stable_deps() { - # DGM debug - set -v - set -x echodebug "install_ubuntu_stable_deps() entry" @@ -3242,7 +3133,6 @@ install_ubuntu_stable_deps() { if [ "${_UPGRADE_SYS}" -eq $BS_TRUE ]; then if [ "${_INSECURE_DL}" -eq $BS_TRUE ]; then ## apt-key is deprecated - ## DGM if [ "$DISTRO_MAJOR_VERSION" -ge 20 ] || [ "$DISTRO_MAJOR_VERSION" -ge 21 ] || [ "$DISTRO_MAJOR_VERSION" -ge 22 ] || [ "$DISTRO_MAJOR_VERSION" -ge 23 ] || [ "$DISTRO_MAJOR_VERSION" -ge 24 ]; then if [ "$DISTRO_MAJOR_VERSION" -ge 20 ]; then __apt_get_install_noinput --allow-unauthenticated debian-archive-keyring && apt-get update || return 1 else @@ -3263,9 +3153,6 @@ install_ubuntu_stable_deps() { } install_ubuntu_git_deps() { - # DGM debug - set -v - set -x echodebug "install_ubuntu_git_deps() entry" @@ -3309,9 +3196,6 @@ install_ubuntu_git_deps() { } install_ubuntu_onedir_deps() { - # DGM debug - set -v - set -x if [ "$_START_DAEMONS" -eq $BS_FALSE ]; then echowarn "Not starting daemons on Debian based distributions is not working mostly because starting them is the default behaviour." @@ -3325,7 +3209,6 @@ install_ubuntu_onedir_deps() { if [ "${_UPGRADE_SYS}" -eq $BS_TRUE ]; then if [ "${_INSECURE_DL}" -eq $BS_TRUE ]; then ## apt-key is deprecated - ## DGM if [ "$DISTRO_MAJOR_VERSION" -ge 20 ] || [ "$DISTRO_MAJOR_VERSION" -ge 21 ] || [ "$DISTRO_MAJOR_VERSION" -ge 22 ] || [ "$DISTRO_MAJOR_VERSION" -ge 23 ] || [ "$DISTRO_MAJOR_VERSION" -ge 24 ]; then if [ "$DISTRO_MAJOR_VERSION" -ge 20 ]; then __apt_get_install_noinput --allow-unauthenticated debian-archive-keyring && apt-get update || return 1 else @@ -3346,9 +3229,6 @@ install_ubuntu_onedir_deps() { } install_ubuntu_stable() { - # DGM debug - set -v - set -x __PACKAGES="" @@ -3376,9 +3256,6 @@ install_ubuntu_stable() { } install_ubuntu_git() { - # DGM debug - set -v - set -x # Activate virtualenv before install if [ "${_VIRTUALENV_DIR}" != "null" ]; then @@ -3409,9 +3286,6 @@ install_ubuntu_git() { } install_ubuntu_onedir() { - # DGM debug - set -v - set -x __PACKAGES="" @@ -3439,9 +3313,6 @@ install_ubuntu_onedir() { } install_ubuntu_stable_post() { - # DGM debug - set -v - set -x for fname in api master minion syndic; do # Skip salt-api since the service should be opt-in and not necessarily started on boot @@ -3470,9 +3341,6 @@ install_ubuntu_stable_post() { } install_ubuntu_git_post() { - # DGM debug - set -v - set -x for fname in api master minion syndic; do # Skip if not meant to be installed @@ -3489,7 +3357,6 @@ install_ubuntu_git_post() { _SERVICE_DIR="${_SALT_GIT_CHECKOUT_DIR}/pkg" fi - ## DGM if [ -f /bin/systemctl ] && [ "$DISTRO_MAJOR_VERSION" -ge 16 ]; then if [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_TRUE ] && [ "$DISTRO_MAJOR_VERSION" -ge 16 ]; then __copyfile "${_SERVICE_DIR}/salt-${fname}.service" "/lib/systemd/system/salt-${fname}.service" @@ -3499,21 +3366,6 @@ install_ubuntu_git_post() { systemctl is-enabled salt-$fname.service || (systemctl preset salt-$fname.service && systemctl enable salt-$fname.service) sleep 1 systemctl daemon-reload - ## DGM elif [ -f /sbin/initctl ]; then - ## DGM _upstart_conf="/etc/init/salt-$fname.conf" - ## DGM # We have upstart support - ## DGM echodebug "There's upstart support" - ## DGM if [ ! -f $_upstart_conf ]; then - ## DGM # upstart does not know about our service, let's copy the proper file - ## DGM echowarn "Upstart does not appear to know about salt-$fname" - ## DGM echodebug "Copying ${_SERVICE_DIR}/salt-$fname.upstart to $_upstart_conf" - ## DGM __copyfile "${_SERVICE_DIR}/salt-${fname}.upstart" "$_upstart_conf" - ## DGM # Set service to know about virtualenv - ## DGM if [ "${_VIRTUALENV_DIR}" != "null" ]; then - ## DGM echo "SALT_USE_VIRTUALENV=${_VIRTUALENV_DIR}" > /etc/default/salt-${fname} - ## DGM fi - ## DGM /sbin/initctl reload-configuration || return 1 - ## DGM fi # No upstart support in Ubuntu!? elif [ -f "${_SALT_GIT_CHECKOUT_DIR}/pkg/salt-${fname}.init" ]; then echodebug "There's NO upstart support!?" @@ -3526,7 +3378,6 @@ install_ubuntu_git_post() { update-rc.d salt-$fname defaults else - ## DGM echoerror "Neither upstart nor init.d was setup for salt-$fname" echoerror "No init.d was setup for salt-$fname" fi done @@ -3535,18 +3386,12 @@ install_ubuntu_git_post() { } install_ubuntu_restart_daemons() { - # DGM debug - set -v - set -x [ "$_START_DAEMONS" -eq $BS_FALSE ] && return - # Ensure upstart configs / systemd units are loaded - ## DGM if [ -f /bin/systemctl ] && [ "$DISTRO_MAJOR_VERSION" -ge 16 ]; then + # Ensure systemd units are loaded if [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_TRUE ] && [ "$DISTRO_MAJOR_VERSION" -ge 16 ]; then systemctl daemon-reload - ## DGM elif [ -f /sbin/initctl ]; then - ## DGM /sbin/initctl reload-configuration fi for fname in api master minion syndic; do @@ -3558,7 +3403,6 @@ install_ubuntu_restart_daemons() { [ $fname = "minion" ] && [ "$_INSTALL_MINION" -eq $BS_FALSE ] && continue [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue - ## DGM if [ -f /bin/systemctl ] && [ "$DISTRO_MAJOR_VERSION" -ge 16 ]; then if [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_TRUE ] && [ "$DISTRO_MAJOR_VERSION" -ge 16 ]; then echodebug "There's systemd support while checking salt-$fname" systemctl stop salt-$fname > /dev/null 2>&1 @@ -3571,18 +3415,6 @@ install_ubuntu_restart_daemons() { fi fi - ## DGM if [ -f /sbin/initctl ]; then - ## DGM echodebug "There's upstart support while checking salt-$fname" - - ## DGM if status salt-$fname 2>/dev/null | grep -q running; then - ## DGM stop salt-$fname || (echodebug "Failed to stop salt-$fname" && return 1) - ## DGM fi - - ## DGM start salt-$fname && continue - ## DGM # We failed to start the service, let's test the SysV code below - ## DGM echodebug "Failed to start salt-$fname using Upstart" - ## DGM fi - if [ ! -f /etc/init.d/salt-$fname ]; then echoerror "No init.d support for salt-$fname was found" return 1 @@ -3596,9 +3428,6 @@ install_ubuntu_restart_daemons() { } install_ubuntu_check_services() { - # DGM debug - set -v - set -x for fname in api master minion syndic; do # Skip salt-api since the service should be opt-in and not necessarily started on boot @@ -3609,7 +3438,6 @@ install_ubuntu_check_services() { [ $fname = "master" ] && [ "$_INSTALL_MASTER" -eq $BS_FALSE ] && continue [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue - ## DGM if [ -f /bin/systemctl ] && [ "$DISTRO_MAJOR_VERSION" -ge 16 ]; then if [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_TRUE ] && [ "$DISTRO_MAJOR_VERSION" -ge 16 ]; then __check_services_systemd salt-$fname || return 1 elif [ -f /etc/init.d/salt-$fname ]; then @@ -3629,15 +3457,9 @@ install_ubuntu_check_services() { # Debian Install Functions # __install_saltstack_debian_repository() { - # DGM debug - set -v - set -x echodebug "__install_saltstack_debian_repository() entry" - ## DGM DEBIAN_RELEASE="$DISTRO_MAJOR_VERSION" - ## DGM DEBIAN_CODENAME="$DISTRO_CODENAME" - if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -ne 3 ]; then echoerror "Python version is no longer supported, only Python 3" return 1 @@ -3684,15 +3506,9 @@ __install_saltstack_debian_repository() { } __install_saltstack_debian_onedir_repository() { - # DGM debug - set -v - set -x echodebug "__install_saltstack_debian_onedir_repository() entry" - ## DGM DEBIAN_RELEASE="$DISTRO_MAJOR_VERSION" - ## DGM DEBIAN_CODENAME="$DISTRO_CODENAME" - if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -ne 3 ]; then echoerror "Python version is no longer supported, only Python 3" return 1 @@ -3714,15 +3530,6 @@ __install_saltstack_debian_onedir_repository() { # shellcheck disable=SC2086,SC2090 __apt_get_install_noinput ${__PACKAGES} || return 1 - # amd64 is just a part of repository URI - ## DGM SALTSTACK_DEBIAN_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/debian/${DEBIAN_RELEASE}/${__REPO_ARCH}/${ONEDIR_REV}" - ## DGM if [ "${ONEDIR_REV}" = "nightly" ] ; then - ## DGM SALTSTACK_DEBIAN_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/debian/${DEBIAN_RELEASE}/${__REPO_ARCH}" - ## DGM fi - ## DGM echo "$__REPO_ARCH_DEB $SALTSTACK_DEBIAN_URL $DEBIAN_CODENAME main" > "/etc/apt/sources.list.d/salt.list" - ## DGM __apt_key_fetch "${SALTSTACK_DEBIAN_URL}/SALT-PROJECT-GPG-PUBKEY-2023.gpg" || return 1 - ## DGM __wait_for_apt apt-get update || return 1 - __fetch_url "/etc/apt/sources.list.d/salt.sources" "https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.sources" __apt_key_fetch "${HTTP_VAL}://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" || return 1 __wait_for_apt apt-get update || return 1 @@ -3743,9 +3550,6 @@ __install_saltstack_debian_onedir_repository() { } install_debian_onedir_deps() { - # DGM debug - set -v - set -x echodebug "install_debian_onedir_git_deps() entry" @@ -3801,9 +3605,6 @@ install_debian_onedir_deps() { } install_debian_git_deps() { - # DGM debug - set -v - set -x echodebug "install_debian_git_deps() entry" @@ -3840,9 +3641,6 @@ install_debian_git_deps() { } install_debian_stable() { - # DGM debug - set -v - set -x __PACKAGES="" @@ -3870,29 +3668,18 @@ install_debian_stable() { } install_debian_11_git_deps() { - # DGM debug - set -v - set -x - install_debian_git_deps || return 1 return 0 } install_debian_12_git_deps() { - # DGM debug - set -v - set -x - install_debian_git_deps || return 1 return 0 } install_debian_git() { - # DGM debug - set -v - set -x if [ -n "$_PY_EXE" ]; then _PYEXE=${_PY_EXE} @@ -3921,27 +3708,18 @@ install_debian_git() { } install_debian_11_git() { - # DGM debug - set -v - set -x install_debian_git || return 1 return 0 } install_debian_12_git() { - # DGM debug - set -v - set -x install_debian_git || return 1 return 0 } install_debian_onedir() { - # DGM debug - set -v - set -x __PACKAGES="" @@ -3969,9 +3747,6 @@ install_debian_onedir() { } install_debian_git_post() { - # DGM debug - set -v - set -x for fname in api master minion syndic; do # Skip if not meant to be installed @@ -3989,7 +3764,6 @@ install_debian_git_post() { fi # Configure SystemD for Debian 8 "Jessie" and later - ## DGM if [ -f /bin/systemctl ]; then if [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_TRUE ]; then if [ ! -f /lib/systemd/system/salt-${fname}.service ] || \ { [ -f /lib/systemd/system/salt-${fname}.service ] && [ $_FORCE_OVERWRITE -eq $BS_TRUE ]; }; then @@ -4013,9 +3787,6 @@ install_debian_git_post() { } install_debian_2021_post() { - # DGM debug - set -v - set -x # Kali 2021 (debian derivative) disables all network services by default # Using archlinux post function to enable salt systemd services @@ -4024,9 +3795,6 @@ install_debian_2021_post() { } install_debian_restart_daemons() { - # DGM debug - set -v - set -x [ "$_START_DAEMONS" -eq $BS_FALSE ] && return 0 @@ -4039,7 +3807,6 @@ install_debian_restart_daemons() { [ $fname = "minion" ] && [ "$_INSTALL_MINION" -eq $BS_FALSE ] && continue [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue - ## DGM if [ -f /bin/systemctl ]; then if [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_TRUE ]; then # Debian 8 and above uses systemd /bin/systemctl stop salt-$fname > /dev/null 2>&1 @@ -4057,9 +3824,6 @@ install_debian_restart_daemons() { } install_debian_check_services() { - # DGM debug - set -v - set -x for fname in api master minion syndic; do # Skip salt-api since the service should be opt-in and not necessarily started on boot @@ -4070,7 +3834,6 @@ install_debian_check_services() { [ $fname = "minion" ] && [ "$_INSTALL_MINION" -eq $BS_FALSE ] && continue [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue - ## DGM if [ -f /bin/systemctl ]; then if [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_TRUE ]; then __check_services_systemd salt-$fname || return 1 elif [ -f /etc/init.d/salt-$fname ]; then @@ -4090,36 +3853,13 @@ install_debian_check_services() { # __install_saltstack_fedora_onedir_repository() { - # DGM debug - set -v - set -x - - ## DGM if [ "$ITYPE" = "stable" ]; then - ## DGM REPO_REV="$ONEDIR_REV" - ## DGM else - ## DGM REPO_REV="latest" - ## DGM fi if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -ne 3 ]; then echoerror "Python version is no longer supported, only Python 3" return 1 fi - ## DGM __PY_VERSION_REPO="py3" - - ## DGM GPG_KEY="SALT-PROJECT-GPG-PUBKEY-2023.pub" - - ## DGM YUM_REPO_FILE="/etc/yum.repos.d/salt.repo" - if [ ! -s "$YUM_REPO_FILE" ] || [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]; then - ## DGM FETCH_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/fedora/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/${ONEDIR_REV}" - ## DGM if [ "${ONEDIR_REV}" = "nightly" ] ; then - ## DGM FETCH_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/fedora/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/" - ## DGM fi - ## DGM __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}.repo" - ## DGM __rpm_import_gpg "${FETCH_URL}/${GPG_KEY}" || return 1 - ## DGM yum clean metadata || return 1 - FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" if [ "$ONEDIR_REV" != "latest" ]; then @@ -4146,9 +3886,6 @@ __install_saltstack_fedora_onedir_repository() { } install_fedora_deps() { - # DGM debug - set -v - set -x if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then dnf -y update || return 1 @@ -4180,9 +3917,6 @@ install_fedora_deps() { } install_fedora_git_deps() { - # DGM debug - set -v - set -x if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -ne 3 ]; then echoerror "Python version is no longer supported, only Python 3" @@ -4223,9 +3957,6 @@ install_fedora_git_deps() { } install_fedora_git() { - # DGM debug - set -v - set -x if [ "${_PY_EXE}" != "" ]; then _PYEXE=${_PY_EXE} @@ -4241,9 +3972,6 @@ install_fedora_git() { } install_fedora_git_post() { - # DGM debug - set -v - set -x for fname in api master minion syndic; do # Skip if not meant to be installed @@ -4277,9 +4005,6 @@ install_fedora_git_post() { } install_fedora_restart_daemons() { - # DGM debug - set -v - set -x [ $_START_DAEMONS -eq $BS_FALSE ] && return @@ -4303,9 +4028,6 @@ install_fedora_restart_daemons() { } install_fedora_check_services() { - # DGM debug - set -v - set -x for fname in api master minion syndic; do # Skip salt-api since the service should be opt-in and not necessarily started on boot @@ -4323,9 +4045,6 @@ install_fedora_check_services() { } install_fedora_onedir_deps() { - # DGM debug - set -v - set -x if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then yum -y update || return 1 @@ -4365,9 +4084,6 @@ install_fedora_onedir_deps() { install_fedora_onedir() { - # DGM debug - set -v - set -x STABLE_REV=$ONEDIR_REV #install_fedora_stable || return 1 @@ -4398,9 +4114,6 @@ install_fedora_onedir() { } install_fedora_onedir_post() { - # DGM debug - set -v - set -x STABLE_REV=$ONEDIR_REV @@ -4431,60 +4144,12 @@ install_fedora_onedir_post() { # CentOS Install Functions # __install_saltstack_rhel_onedir_repository() { - # DGM debug - set -v - set -x - - ## DGM if [ "$ITYPE" = "stable" ]; then - ## DGM REPO_REV="$ONEDIR_REV" - ## DGM else - ## DGM REPO_REV="latest" - ## DGM fi if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -ne 3 ]; then echoerror "Python version is no longer supported, only Python 3" return 1 fi -## DGM # Avoid using '$releasever' variable for yum. -## DGM # Instead, this should work correctly on all RHEL variants. -## DGM base_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/redhat/${DISTRO_MAJOR_VERSION}/\$basearch/${ONEDIR_REV}/" -## DGM if [ "${ONEDIR_REV}" = "nightly" ] ; then -## DGM base_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/redhat/${DISTRO_MAJOR_VERSION}/\$basearch/" -## DGM fi -## DGM -## DGM ## gpg_key="SALT-PROJECT-GPG-PUBKEY-2023.pub" -## DGM -## DGM gpg_key_urls="" -## DGM for key in $gpg_key; do -## DGM gpg_key_urls=$(printf "${base_url}${key},%s" "$gpg_key_urls") -## DGM done -## DGM -## DGM repo_file="/etc/yum.repos.d/salt.repo" -## DGM -## DGM if [ ! -s "$repo_file" ] || [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]; then -## DGM cat <<_eof > "$repo_file" -## DGM [saltstack] -## DGM name=SaltStack ${repo_rev} Release Channel for RHEL/CentOS \$releasever -## DGM baseurl=${base_url} -## DGM skip_if_unavailable=True -## DGM gpgcheck=1 -## DGM gpgkey=${gpg_key_urls} -## DGM enabled=1 -## DGM enabled_metadata=1 -## DGM _eof -## DGM -## DGM fetch_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/redhat/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/${ONEDIR_REV}/" -## DGM if [ "${ONEDIR_REV}" = "nightly" ] ; then -## DGM fetch_url="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/redhat/${DISTRO_MAJOR_VERSION}/${CPU_ARCH_L}/" -## DGM fi -## DGM for key in $gpg_key; do -## DGM __rpm_import_gpg "${fetch_url}${key}" || return 1 -## DGM done -## DGM -## DGM yum clean metadata || return 1 - ## DGM YUM_REPO_FILE="/etc/yum.repos.d/salt.repo" - if [ ! -s "$YUM_REPO_FILE" ] || [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]; then FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" @@ -4511,9 +4176,6 @@ __install_saltstack_rhel_onedir_repository() { } install_centos_stable_deps() { - # DGM debug - set -v - set -x if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then yum -y update || return 1 @@ -4552,9 +4214,6 @@ install_centos_stable_deps() { } install_centos_stable() { - # DGM debug - set -v - set -x __PACKAGES="" @@ -4589,9 +4248,6 @@ install_centos_stable() { } install_centos_stable_post() { - # DGM debug - set -v - set -x SYSTEMD_RELOAD=$BS_FALSE @@ -4604,7 +4260,6 @@ install_centos_stable_post() { [ $fname = "minion" ] && [ "$_INSTALL_MINION" -eq $BS_FALSE ] && continue [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue - ## DGM if [ -f /bin/systemctl ]; then if [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_TRUE ]; then /bin/systemctl is-enabled salt-${fname}.service > /dev/null 2>&1 || ( /bin/systemctl preset salt-${fname}.service > /dev/null 2>&1 && @@ -4625,9 +4280,6 @@ install_centos_stable_post() { } install_centos_git_deps() { - # DGM debug - set -v - set -x # First try stable deps then fall back to onedir deps if that one fails # if we're installing on a Red Hat based host that doesn't have the classic @@ -4674,9 +4326,6 @@ install_centos_git_deps() { } install_centos_git() { - # DGM debug - set -v - set -x if [ "${_PY_EXE}" != "" ]; then _PYEXE=${_PY_EXE} @@ -4693,9 +4342,6 @@ install_centos_git() { } install_centos_git_post() { - # DGM debug - set -v - set -x SYSTEMD_RELOAD=$BS_FALSE @@ -4714,7 +4360,6 @@ install_centos_git_post() { _SERVICE_FILE="${_SALT_GIT_CHECKOUT_DIR}/pkg/rpm/salt-${fname}.service" fi - ## DGM if [ -f /bin/systemctl ]; then if [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_TRUE ]; then if [ ! -f "/usr/lib/systemd/system/salt-${fname}.service" ] || \ { [ -f "/usr/lib/systemd/system/salt-${fname}.service" ] && [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]; }; then @@ -4739,9 +4384,6 @@ install_centos_git_post() { } install_centos_onedir_deps() { - # DGM debug - set -v - set -x if [ "$_UPGRADE_SYS" -eq "$BS_TRUE" ]; then yum -y update || return 1 @@ -4779,9 +4421,6 @@ install_centos_onedir_deps() { } install_centos_onedir() { - # DGM debug - set -v - set -x __PACKAGES="" @@ -4809,9 +4448,6 @@ install_centos_onedir() { } install_centos_onedir_post() { - # DGM debug - set -v - set -x SYSTEMD_RELOAD=$BS_FALSE @@ -4824,7 +4460,6 @@ install_centos_onedir_post() { [ $fname = "minion" ] && [ "$_INSTALL_MINION" -eq $BS_FALSE ] && continue [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue - ## DGM if [ -f /bin/systemctl ]; then if [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_TRUE ]; then /bin/systemctl is-enabled salt-${fname}.service > /dev/null 2>&1 || ( /bin/systemctl preset salt-${fname}.service > /dev/null 2>&1 && @@ -4845,9 +4480,6 @@ install_centos_onedir_post() { } install_centos_restart_daemons() { - # DGM debug - set -v - set -x [ "$_START_DAEMONS" -eq $BS_FALSE ] && return @@ -4860,27 +4492,10 @@ install_centos_restart_daemons() { [ $fname = "minion" ] && [ "$_INSTALL_MINION" -eq $BS_FALSE ] && continue [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue - ## DGM if [ -f /sbin/initctl ] && [ -f /etc/init/salt-${fname}.conf ]; then - ## DGM # We have upstart support and upstart knows about our service - ## DGM if ! /sbin/initctl status salt-$fname > /dev/null 2>&1; then - ## DGM # Everything is in place and upstart gave us an error code? Fail! - ## DGM return 1 - ## DGM fi - - ## DGM # upstart knows about this service. - ## DGM # Let's try to stop it, and then start it - ## DGM /sbin/initctl stop salt-$fname > /dev/null 2>&1 - ## DGM # Restart service - ## DGM if ! /sbin/initctl start salt-$fname > /dev/null 2>&1; then - ## DGM # Failed the restart?! - ## DGM return 1 - ## DGM fi - ## DGM elif [ -f /etc/init.d/salt-$fname ]; then if [ -f /etc/init.d/salt-$fname ]; then # Disable stdin to fix shell session hang on killing tee pipe service salt-$fname stop < /dev/null > /dev/null 2>&1 service salt-$fname start < /dev/null - ## DGM elif [ -f /usr/bin/systemctl ]; then elif [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_TRUE ]; then # CentOS 7 uses systemd /usr/bin/systemctl stop salt-$fname > /dev/null 2>&1 @@ -4895,36 +4510,24 @@ install_centos_restart_daemons() { } install_centos_testing_deps() { - # DGM debug - set -v - set -x install_centos_stable_deps || return 1 return 0 } install_centos_testing() { - # DGM debug - set -v - set -x install_centos_stable || return 1 return 0 } install_centos_testing_post() { - # DGM debug - set -v - set -x install_centos_stable_post || return 1 return 0 } install_centos_check_services() { - # DGM debug - set -v - set -x for fname in api master minion syndic; do # Skip salt-api since the service should be opt-in and not necessarily started on boot @@ -4937,7 +4540,6 @@ install_centos_check_services() { if [ -f "/etc/init.d/salt-$fname" ]; then __check_services_sysvinit "salt-$fname" || return 1 - ## DGM elif [ -f /usr/bin/systemctl ]; then elif [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_TRUE ]; then __check_services_systemd "salt-$fname" || return 1 fi @@ -4955,513 +4557,342 @@ install_centos_check_services() { # RedHat Install Functions # install_red_hat_linux_stable_deps() { - # DGM debug - set -v - set -x install_centos_stable_deps || return 1 return 0 } install_red_hat_linux_git_deps() { - # DGM debug - set -v - set -x install_centos_git_deps || return 1 return 0 } install_red_hat_linux_onedir_deps() { - # DGM debug - set -v - set -x install_centos_onedir_deps || return 1 return 0 } install_red_hat_enterprise_stable_deps() { - # DGM debug - set -v - set -x install_red_hat_linux_stable_deps || return 1 return 0 } install_red_hat_enterprise_git_deps() { - # DGM debug - set -v - set -x install_red_hat_linux_git_deps || return 1 return 0 } install_red_hat_enterprise_onedir_deps() { - # DGM debug - set -v - set -x install_red_hat_linux_onedir_deps || return 1 return 0 } install_red_hat_enterprise_linux_stable_deps() { - # DGM debug - set -v - set -x install_red_hat_linux_stable_deps || return 1 return 0 } install_red_hat_enterprise_linux_git_deps() { - # DGM debug - set -v - set -x install_red_hat_linux_git_deps || return 1 return 0 } install_red_hat_enterprise_linux_onedir_deps() { - # DGM debug - set -v - set -x install_red_hat_linux_onedir_deps || return 1 return 0 } install_red_hat_enterprise_server_stable_deps() { - # DGM debug - set -v - set -x install_red_hat_linux_stable_deps || return 1 return 0 } install_red_hat_enterprise_server_git_deps() { - # DGM debug - set -v - set -x install_red_hat_linux_git_deps || return 1 return 0 } install_red_hat_enterprise_server_onedir_deps() { - # DGM debug - set -v - set -x install_red_hat_linux_onedir_deps || return 1 return 0 } install_red_hat_enterprise_workstation_stable_deps() { - # DGM debug - set -v - set -x install_red_hat_linux_stable_deps || return 1 return 0 } install_red_hat_enterprise_workstation_git_deps() { - # DGM debug - set -v - set -x install_red_hat_linux_git_deps || return 1 return 0 } install_red_hat_enterprise_workstation_onedir_deps() { - # DGM debug - set -v - set -x install_red_hat_linux_timat_deps || return 1 return 0 } install_red_hat_linux_stable() { - # DGM debug - set -v - set -x install_centos_stable || return 1 return 0 } install_red_hat_linux_git() { - # DGM debug - set -v - set -x install_centos_git || return 1 return 0 } install_red_hat_linux_onedir() { - # DGM debug - set -v - set -x install_centos_onedir || return 1 return 0 } install_red_hat_enterprise_stable() { - # DGM debug - set -v - set -x install_red_hat_linux_stable || return 1 return 0 } install_red_hat_enterprise_git() { - # DGM debug - set -v - set -x install_red_hat_linux_git || return 1 return 0 } install_red_hat_enterprise_onedir() { - # DGM debug - set -v - set -x install_red_hat_linux_onedir || return 1 return 0 } install_red_hat_enterprise_linux_stable() { - # DGM debug - set -v - set -x install_red_hat_linux_stable || return 1 return 0 } install_red_hat_enterprise_linux_git() { - # DGM debug - set -v - set -x install_red_hat_linux_git || return 1 return 0 } install_red_hat_enterprise_linux_onedir() { - # DGM debug - set -v - set -x install_red_hat_linux_onedir || return 1 return 0 } install_red_hat_enterprise_server_stable() { - # DGM debug - set -v - set -x install_red_hat_linux_stable || return 1 return 0 } install_red_hat_enterprise_server_git() { - # DGM debug - set -v - set -x install_red_hat_linux_git || return 1 return 0 } install_red_hat_enterprise_server_onedir() { - # DGM debug - set -v - set -x install_red_hat_linux_onedir || return 1 return 0 } install_red_hat_enterprise_workstation_stable() { - # DGM debug - set -v - set -x install_red_hat_linux_stable || return 1 return 0 } install_red_hat_enterprise_workstation_git() { - # DGM debug - set -v - set -x install_red_hat_linux_git || return 1 return 0 } install_red_hat_enterprise_workstation_onedir() { - # DGM debug - set -v - set -x install_red_hat_linux_onedir || return 1 return 0 } install_red_hat_linux_stable_post() { - # DGM debug - set -v - set -x install_centos_stable_post || return 1 return 0 } install_red_hat_linux_restart_daemons() { - # DGM debug - set -v - set -x install_centos_restart_daemons || return 1 return 0 } install_red_hat_linux_git_post() { - # DGM debug - set -v - set -x install_centos_git_post || return 1 return 0 } install_red_hat_enterprise_stable_post() { - # DGM debug - set -v - set -x install_red_hat_linux_stable_post || return 1 return 0 } install_red_hat_enterprise_restart_daemons() { - # DGM debug - set -v - set -x install_red_hat_linux_restart_daemons || return 1 return 0 } install_red_hat_enterprise_git_post() { - # DGM debug - set -v - set -x install_red_hat_linux_git_post || return 1 return 0 } install_red_hat_enterprise_linux_stable_post() { - # DGM debug - set -v - set -x install_red_hat_linux_stable_post || return 1 return 0 } install_red_hat_enterprise_linux_restart_daemons() { - # DGM debug - set -v - set -x install_red_hat_linux_restart_daemons || return 1 return 0 } install_red_hat_enterprise_linux_git_post() { - # DGM debug - set -v - set -x install_red_hat_linux_git_post || return 1 return 0 } install_red_hat_enterprise_server_stable_post() { - # DGM debug - set -v - set -x install_red_hat_linux_stable_post || return 1 return 0 } install_red_hat_enterprise_server_restart_daemons() { - # DGM debug - set -v - set -x install_red_hat_linux_restart_daemons || return 1 return 0 } install_red_hat_enterprise_server_git_post() { - # DGM debug - set -v - set -x install_red_hat_linux_git_post || return 1 return 0 } install_red_hat_enterprise_workstation_stable_post() { - # DGM debug - set -v - set -x install_red_hat_linux_stable_post || return 1 return 0 } install_red_hat_enterprise_workstation_restart_daemons() { - # DGM debug - set -v - set -x install_red_hat_linux_restart_daemons || return 1 return 0 } install_red_hat_enterprise_workstation_git_post() { - # DGM debug - set -v - set -x install_red_hat_linux_git_post || return 1 return 0 } install_red_hat_linux_testing_deps() { - # DGM debug - set -v - set -x install_centos_testing_deps || return 1 return 0 } install_red_hat_linux_testing() { - # DGM debug - set -v - set -x install_centos_testing || return 1 return 0 } install_red_hat_linux_testing_post() { - # DGM debug - set -v - set -x install_centos_testing_post || return 1 return 0 } install_red_hat_enterprise_testing_deps() { - # DGM debug - set -v - set -x install_centos_testing_deps || return 1 return 0 } install_red_hat_enterprise_testing() { - # DGM debug - set -v - set -x install_centos_testing || return 1 return 0 } install_red_hat_enterprise_testing_post() { - # DGM debug - set -v - set -x install_centos_testing_post || return 1 return 0 } install_red_hat_enterprise_server_testing_deps() { - # DGM debug - set -v - set -x install_centos_testing_deps || return 1 return 0 } install_red_hat_enterprise_server_testing() { - # DGM debug - set -v - set -x install_centos_testing || return 1 return 0 } install_red_hat_enterprise_server_testing_post() { - # DGM debug - set -v - set -x install_centos_testing_post || return 1 return 0 } install_red_hat_enterprise_workstation_testing_deps() { - # DGM debug - set -v - set -x install_centos_testing_deps || return 1 return 0 } install_red_hat_enterprise_workstation_testing() { - # DGM debug - set -v - set -x install_centos_testing || return 1 return 0 } install_red_hat_enterprise_workstation_testing_post() { - # DGM debug - set -v - set -x install_centos_testing_post || return 1 return 0 @@ -5476,9 +4907,6 @@ install_red_hat_enterprise_workstation_testing_post() { # Oracle Linux Install Functions # install_oracle_linux_stable_deps() { - # DGM debug - set -v - set -x # Install Oracle's EPEL. if [ "${_EPEL_REPOS_INSTALLED}" -eq $BS_FALSE ]; then @@ -6084,12 +5512,6 @@ install_amazon_linux_ami_2_deps() { return 1 fi - ## DGM if [ "$ITYPE" = "stable" ]; then - ## DGM REPO_REV="$STABLE_REV" - ## DGM else - ## DGM REPO_REV="latest" - ## DGM fi - # We need to install yum-utils before doing anything else when installing on # Amazon Linux ECS-optimized images. See issue #974. __yum_install_noinput yum-utils @@ -6099,33 +5521,7 @@ install_amazon_linux_ami_2_deps() { yum -y update || return 1 fi -## DGM if [ $_DISABLE_REPOS -eq $BS_FALSE ] || [ "$_CUSTOM_REPO_URL" != "null" ]; then -## DGM __YUM_REPO_FILENAME="salt.repo" -## DGM PY_PKG_VER=3 -## DGM __PY_VERSION_REPO="py3" -## DGM repo_label="saltstack-py3-repo" -## DGM repo_name="SaltStack Python 3 repo for Amazon Linux 2" -## DGM -## DGM base_url="$HTTP_VAL://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/amazon/2/\$basearch/$repo_rev/" -## DGM gpg_key="${base_url}SALT-PROJECT-GPG-PUBKEY-2023.gpg" -## DGM -## DGM # This should prob be refactored to use __install_saltstack_rhel_onedir_repository() -## DGM # With args passed in to do the right thing. Reformatted to be more like the -## DGM # amazon linux yum file. -## DGM if [ ! -s "/etc/yum.repos.d/${__YUM_REPO_FILENAME}" ]; then -## DGM cat <<_eof > "/etc/yum.repos.d/${__YUM_REPO_FILENAME}" -## DGM [$repo_label] -## DGM name=$repo_name -## DGM failovermethod=priority -## DGM priority=10 -## DGM gpgcheck=1 -## DGM gpgkey=$gpg_key -## DGM baseurl=$base_url -## DGM _eof -## DGM fi - if [ $_DISABLE_REPOS -eq $BS_FALSE ] || [ "$_CUSTOM_REPO_URL" != "null" ]; then - ## DGM YUM_REPO_FILE="/etc/yum.repos.d/salt.repo" if [ ! -s "${YUM_REPO_FILE}" ]; then FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" @@ -6159,12 +5555,6 @@ install_amazon_linux_ami_2_onedir_deps() { return 1 fi - ## DGM if [ "$ITYPE" = "onedir" ]; then - ## DGM REPO_REV="$ONEDIR_REV" - ## DGM else - ## DGM REPO_REV="latest" - ## DGM fi - # We need to install yum-utils before doing anything else when installing on # Amazon Linux ECS-optimized images. See issue #974. __yum_install_noinput yum-utils @@ -6174,37 +5564,7 @@ install_amazon_linux_ami_2_onedir_deps() { yum -y update || return 1 fi -## DGM if [ $_DISABLE_REPOS -eq $BS_FALSE ] || [ "$_CUSTOM_REPO_URL" != "null" ]; then -## DGM __YUM_REPO_FILENAME="salt.repo" -## DGM __PY_VERSION_REPO="py3" -## DGM PY_PKG_VER=3 -## DGM repo_label="saltstack-py3-repo" -## DGM repo_name="SaltStack Python 3 repo for Amazon Linux 2" -## DGM fi -## DGM -## DGM base_url="$HTTP_VAL://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/amazon/2/\$basearch/$repo_rev/" -## DGM if [ "${ONEDIR_REV}" = "nightly" ] ; then -## DGM base_url="$HTTP_VAL://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/amazon/2/\$basearch/" -## DGM fi -## DGM -## DGM gpg_key="${base_url}SALT-PROJECT-GPG-PUBKEY-2023.pub" -## DGM -## DGM # With args passed in to do the right thing. Reformatted to be more like the -## DGM # amazon linux yum file. -## DGM if [ ! -s "/etc/yum.repos.d/${__YUM_REPO_FILENAME}" ]; then -## DGM cat <<_eof > "/etc/yum.repos.d/${__YUM_REPO_FILENAME}" -## DGM [$repo_label] -## DGM name=$repo_name -## DGM failovermethod=priority -## DGM priority=10 -## DGM gpgcheck=1 -## DGM gpgkey=$gpg_key -## DGM baseurl=$base_url -## DGM _eof -## DGM fi - if [ $_DISABLE_REPOS -eq $BS_FALSE ] || [ "$_CUSTOM_REPO_URL" != "null" ]; then - ## DGM YUM_REPO_FILE="/etc/yum.repos.d/salt.repo" if [ ! -s "${YUM_REPO_FILE}" ]; then FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" @@ -6321,11 +5681,6 @@ install_amazon_linux_ami_2023_git_deps() { } install_amazon_linux_ami_2023_onedir_deps() { - ## DGM if [ "$ITYPE" = "onedir" ]; then - ## DGM REPO_REV="$ONEDIR_REV" - ## DGM else - ## DGM REPO_REV="latest" - ## DGM fi # We need to install yum-utils before doing anything else when installing on # Amazon Linux ECS-optimized images. See issue #974. @@ -6336,37 +5691,7 @@ install_amazon_linux_ami_2023_onedir_deps() { yum -y update || return 1 fi -## DGM if [ "$_DISABLE_REPOS" -eq $BS_FALSE ] || [ "$_CUSTOM_REPO_URL" != "null" ]; then -## DGM __YUM_REPO_FILENAME="salt.repo" -## DGM __PY_VERSION_REPO="py3" -## DGM PY_PKG_VER=3 -## DGM repo_label="saltstack-py3-repo" -## DGM repo_name="SaltStack Python 3 repo for Amazon Linux 2023" -## DGM -## DGM base_url="$HTTP_VAL://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/amazon/2023/\$basearch/$repo_rev/" -## DGM if [ "${ONEDIR_REV}" = "nightly" ] ; then -## DGM base_url="$HTTP_VAL://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/amazon/2023/\$basearch/" -## DGM fi -## DGM -## DGM gpg_key="${base_url}SALT-PROJECT-GPG-PUBKEY-2023.pub" -## DGM -## DGM # This should prob be refactored to use __install_saltstack_rhel_onedir_repository() -## DGM # With args passed in to do the right thing. Reformatted to be more like the -## DGM # amazon linux yum file. -## DGM if [ ! -s "/etc/yum.repos.d/${__YUM_REPO_FILENAME}" ]; then -## DGM cat <<_eof > "/etc/yum.repos.d/${__YUM_REPO_FILENAME}" -## DGM [$repo_label] -## DGM name=$repo_name -## DGM failovermethod=priority -## DGM priority=10 -## DGM gpgcheck=1 -## DGM gpgkey=$gpg_key -## DGM baseurl=$base_url -## DGM _eof -## DGM fi -## DGM fi if [ $_DISABLE_REPOS -eq $BS_FALSE ] || [ "$_CUSTOM_REPO_URL" != "null" ]; then - ## DGM YUM_REPO_FILE="/etc/yum.repos.d/salt.repo" if [ ! -s "${YUM_REPO_FILE}" ]; then FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" @@ -6579,7 +5904,6 @@ install_arch_linux_post() { # Skip salt-api since the service should be opt-in and not necessarily started on boot [ $fname = "api" ] && continue - ## DGM if [ -f /usr/bin/systemctl ]; then if [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_TRUE ]; then # Using systemd /usr/bin/systemctl is-enabled salt-$fname.service > /dev/null 2>&1 || ( @@ -6611,7 +5935,6 @@ install_arch_linux_git_post() { _SERVICE_DIR="${_SALT_GIT_CHECKOUT_DIR}/pkg/rpm" fi - ## DGM if [ -f /usr/bin/systemctl ]; then if [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_TRUE ]; then __copyfile "${_SERVICE_DIR}/salt-${fname}.service" "/lib/systemd/system/salt-${fname}.service" @@ -6645,7 +5968,6 @@ install_arch_linux_restart_daemons() { [ $fname = "minion" ] && [ "$_INSTALL_MINION" -eq $BS_FALSE ] && continue [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue - ## DGM if [ -f /usr/bin/systemctl ]; then if [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_TRUE ]; then /usr/bin/systemctl stop salt-$fname.service > /dev/null 2>&1 /usr/bin/systemctl start salt-$fname.service && continue @@ -6662,7 +5984,6 @@ install_arch_linux_restart_daemons() { } install_arch_check_services() { - ## DGM if [ ! -f /usr/bin/systemctl ]; then if [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_TRUE ]; then # Not running systemd!? Don't check! return 0 @@ -6712,25 +6033,7 @@ __install_saltstack_photon_onedir_repository() { return 1 fi - ## DGM if [ "$ITYPE" = "stable" ]; then - ## DGM REPO_REV="$ONEDIR_REV" - ## DGM else - ## DGM REPO_REV="latest" - ## DGM fi - - ## DGM __PY_VERSION_REPO="py3" - ## DGM YUM_REPO_FILE="/etc/yum.repos.d/salt.repo" - if [ ! -s "$YUM_REPO_FILE" ] || [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]; then - ## DGM ## salt repo 4 & 5 have issues, need the Major version dot Zero, eg: 4.0, 5.0 - ## DGM FETCH_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/photon/${DISTRO_MAJOR_VERSION}.0/${CPU_ARCH_L}/${ONEDIR_REV}" - ## DGM if [ "${ONEDIR_REV}" = "nightly" ] ; then - ## DGM FETCH_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_NIGHTLY_DIR}/${__PY_VERSION_REPO}/photon/${DISTRO_MAJOR_VERSION}.0/${CPU_ARCH_L}/" - ## DGM fi - ## DGM __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}.repo" - ## DGM GPG_KEY="SALT-PROJECT-GPG-PUBKEY-2023.pub" - ## DGM __rpm_import_gpg "${FETCH_URL}/${GPG_KEY}" || return 1 - FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" if [ "$ONEDIR_REV" != "latest" ]; then @@ -7243,7 +6546,6 @@ install_opensuse_stable_post() { [ $fname = "minion" ] && [ "$_INSTALL_MINION" -eq $BS_FALSE ] && continue [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue - ## DGM if [ -f /bin/systemctl ] || [ -f /usr/bin/systemctl ]; then if [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_TRUE ]; then systemctl is-enabled salt-$fname.service || (systemctl preset salt-$fname.service && systemctl enable salt-$fname.service) sleep 1 @@ -7267,7 +6569,6 @@ install_opensuse_git_post() { [ $fname = "minion" ] && [ "$_INSTALL_MINION" -eq $BS_FALSE ] && continue [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue - ## DGM if command -v systemctl; then if [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_TRUE ]; then use_usr_lib=$BS_FALSE @@ -7320,7 +6621,6 @@ install_opensuse_restart_daemons() { [ $fname = "minion" ] && [ "$_INSTALL_MINION" -eq $BS_FALSE ] && continue [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue - ## DGM if [ -f /bin/systemctl ]; then if [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_TRUE ]; then systemctl stop salt-$fname > /dev/null 2>&1 systemctl start salt-$fname.service && continue @@ -7337,7 +6637,6 @@ install_opensuse_restart_daemons() { } install_opensuse_check_services() { - ## DGM if [ ! -f /bin/systemctl ]; then if [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_TRUE ]; then # Not running systemd!? Don't check! return 0 @@ -7924,41 +7223,22 @@ daemons_running_voidlinux() { # OS X / Darwin Install Functions # -## DGM __parse_repo_json_python() { -## DGM -## DGM # Using latest, grab the right -## DGM # version from the repo.json -## DGM _JSON_VERSION=$(python - <<-EOF -## DGM import json, urllib.request -## DGM ## DGM url = "https://repo.saltproject.io/salt/py3/macos/repo.json" ## DGM note the use of repo.json here TBD handling this -## DGM url = "https://packages.broadcom.com/ui/repos/tree/General/saltproject-generic/macos" -## DGM response = urllib.request.urlopen(url) -## DGM -## DGM ## DGM No json file to process here TBD handling it -## DGM data = json.loads(response.read()) -## DGM version = data["${_ONEDIR_REV}"][list(data["${_ONEDIR_REV}"])[0]]['version'] -## DGM print(version) -## DGM EOF -## DGM ) -## DGM echo "${_JSON_VERSION}" -## DGM } - __macosx_get_packagesite_onedir_latest() { - # DGM debug + ## DGM debug set -v set -x echodebug "Find latest MacOS release from repository" # get dir listing from url, sort and pick highest - macos_versions_tmpf=$(mktemp) + macos_versions_tmpdir=$(mktemp -d) curr_pwd=$(pwd) - cd ${macos_versions_tmpf} || return 1 + cd ${macos_versions_tmpdir} || return 1 wget -r -np -nH --exclude-directories=onedir,relenv,windows -x -l 1 "$SALT_MACOS_PKGDIR_URL/" # shellcheck disable=SC2010 LATEST_VERSION=$(ls artifactory/saltproject-generic/macos/ | grep -v 'index.html' | sort -V -u | tail -n 1) cd ${curr_pwd} || return "${LATEST_VERSION}" - rm -fR ${macos_versions_tmpf} + rm -fR ${macos_versions_tmpdir} echodebug "latest MacOS release from repository found ${LATEST_VERSION}" return "${LATEST_VERSION}" @@ -7966,9 +7246,6 @@ __macosx_get_packagesite_onedir_latest() { __macosx_get_packagesite_onedir() { - # DGM debug - set -v - set -x if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -ne 3 ]; then echoerror "Python version is no longer supported, only Python 3" @@ -7977,22 +7254,6 @@ __macosx_get_packagesite_onedir() { DARWIN_ARCH=${CPU_ARCH_L} -## DGM if [ "$(echo "$_ONEDIR_REV" | grep -E '^(latest)$')" != "" ]; then -## DGM ## DGM TBD what to do here -## DGM _PKG_VERSION=$(__parse_repo_json_python) -## DGM elif [ "$(echo "$_ONEDIR_REV" | grep -E '^([3-9][0-9]{3}(\.[0-9]*))')" != "" ]; then -## DGM _PKG_VERSION=$_ONEDIR_REV -## DGM else -## DGM ## DGM TBD what to do here -## DGM _PKG_VERSION=$(__parse_repo_json_python) -## DGM fi -## DGM -## DGM ## DGM PKG="salt-${_PKG_VERSION}-${__PY_VERSION_REPO}-${DARWIN_ARCH}.pkg" -## DGM ## DGM SALTPKGCONFURL="https://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/macos/${ONEDIR_REV}/${PKG}" -## DGM _ONEDIR_TYPE="saltproject-generic" -## DGM PKG="salt-${ONEDIR_REV}-py3-${DARWIN_ARCH}.pkg" -## DGM SALTPKGCONFURL="https://${_REPO_URL}/${_ONEDIR_TYPE}/macos/${ONEDIR_REV}/${PKG}" - _ONEDIR_TYPE="saltproject-generic" SALT_MACOS_PKGDIR_URL="https://${_REPO_URL}/${_ONEDIR_TYPE}/macos" if [ "$(echo "$_ONEDIR_REV" | grep -E '^(latest)$')" != "" ]; then @@ -8009,36 +7270,24 @@ __macosx_get_packagesite_onedir() { } __configure_macosx_pkg_details_onedir() { - # DGM debug - set -v - set -x __macosx_get_packagesite_onedir || return 1 return 0 } install_macosx_stable_deps() { - # DGM debug - set -v - set -x __configure_macosx_pkg_details_onedir || return 1 return 0 } install_macosx_onedir_deps() { - # DGM debug - set -v - set -x __configure_macosx_pkg_details_onedir || return 1 return 0 } install_macosx_git_deps() { - # DGM debug - set -v - set -x install_macosx_stable_deps || return 1 @@ -8065,9 +7314,6 @@ install_macosx_git_deps() { } install_macosx_stable() { - # DGM debug - set -v - set -x install_macosx_stable_deps || return 1 @@ -8079,9 +7325,6 @@ install_macosx_stable() { } install_macosx_onedir() { - # DGM debug - set -v - set -x install_macosx_onedir_deps || return 1 @@ -8093,9 +7336,6 @@ install_macosx_onedir() { } install_macosx_git() { - # DGM debug - set -v - set -x if [ -n "$_PY_EXE" ]; then @@ -8110,9 +7350,6 @@ install_macosx_git() { } install_macosx_stable_post() { - # DGM debug - set -v - set -x if [ ! -f /etc/paths.d/salt ]; then print "%s\n" "/opt/salt/bin" "/usr/local/sbin" > /etc/paths.d/salt @@ -8129,27 +7366,18 @@ install_macosx_stable_post() { } install_macosx_onedir_post() { - # DGM debug - set -v - set -x install_macosx_stable_post || return 1 return 0 } install_macosx_git_post() { - # DGM debug - set -v - set -x install_macosx_stable_post || return 1 return 0 } install_macosx_restart_daemons() { - # DGM debug - set -v - set -x [ "$_START_DAEMONS" -eq $BS_FALSE ] && return @@ -8176,9 +7404,6 @@ install_macosx_restart_daemons() { # the -c options is passed. # config_salt() { - # DGM debug - set -v - set -x # If the configuration directory is not passed, return [ "$_TEMP_CONFIG_DIR" = "null" ] && return @@ -8341,9 +7566,6 @@ config_salt() { # as long as the -k option is passed. # preseed_master() { - # DGM debug - set -v - set -x # Create the PKI directory @@ -8379,9 +7601,6 @@ preseed_master() { # This function checks if all of the installed daemons are running or not. # daemons_running_onedir() { - # DGM debug - set -v - set -x [ "$_START_DAEMONS" -eq $BS_FALSE ] && return 0 @@ -8420,9 +7639,6 @@ daemons_running_onedir() { # This function checks if all of the installed daemons are running or not. # daemons_running() { - # DGM debug - set -v - set -x [ "$_START_DAEMONS" -eq $BS_FALSE ] && return 0 diff --git a/salt-quick-start.sh b/salt-quick-start.sh index 9be9e7afe..d4727ed27 100755 --- a/salt-quick-start.sh +++ b/salt-quick-start.sh @@ -1,7 +1,6 @@ #!/bin/sh __ScriptName="salt-quick-start.sh" -## DGM SALT_REPO_URL="https://repo.saltproject.io/salt/py3/onedir" SALT_REPO_URL="https://packages.broadcom.com/artifactory/salt-project-generic/onedir" _COLORS=${QS_COLORS:-$(tput colors 2>/dev/null || echo 0)} @@ -90,9 +89,6 @@ if [[ "$_LOCAL" == "1" && "$_FULL" == "1" ]]; then fi __parse_repo_json_jq() { - ## DGM TBD how to handle no repo.json - ## DGM _JSON_FILE="${SALT_REPO_URL}/repo.json" - ## DGM _JSON_VERSION=$(curl -s ${_JSON_FILE} | jq -sr ".[].latest[] | select(.os == \"$1\") | select(.arch == \"$2\").version") # $1 is OS_NAME # $2 is ARCH @@ -162,7 +158,6 @@ fi __parse_repo_json_jq ${OS_NAME} ${CPU_ARCH_L} FILE="salt-${_JSON_VERSION}-onedir-${OS_NAME_L}-${CPU_ARCH_L}.tar.xz" -## DGM URL="${SALT_REPO_URL}/latest/${FILE}" URL="${SALT_REPO_URL}/${_JSON_VERSION}/${FILE}" if [[ ! -f ${FILE} ]]; then diff --git a/tests/conftest.py b/tests/conftest.py index 3779d377d..74ded3b2d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -19,7 +19,6 @@ def target_salt_version(): target_salt = ".".join( [ item - ## DGM for item in os.environ["KITCHEN_SUITE"].split("-") for item in os.environ.get("KITCHEN_SUITE", "").split("-") if item not in bootstrap_types ] diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index 00da09587..8d0b41467 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -26,7 +26,6 @@ def run_salt_call(cmd): """ cmd.append("--out=json") result = subprocess.run(cmd, capture_output=True, text=True) - print(f"DGM run_salt_call, cmd '{cmd}', result '{result}'", flush=True) if result.stdout: json_data = json.loads(result.stdout) return json_data["local"] diff --git a/tests/requirements.txt b/tests/requirements.txt index 64223821a..e079f8a60 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,7 +1 @@ -## DGM pytest-testinfra -## DGM paramiko -## DGM requests-ntlm==1.1.0; sys.platform == 'win32' -## DGM pywinrm; sys.platform == 'win32' -## DGM six>=1.10.0 - pytest From 081590b816c156c21e28b249351917be58b917d9 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 6 Nov 2024 18:12:40 -0700 Subject: [PATCH 025/173] Enabled MacOS bootstrap in CI/CD and a little more debug output for MacOS --- .github/workflows/test-macos.yml | 8 ++++---- bootstrap-salt.sh | 5 +++++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index d07416d3d..504f8b3e7 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -46,11 +46,11 @@ jobs: run: | pip install -U pytest - ## TBD Disabled until CI/CD is functional - ## - name: Bootstrap Salt - ## run: | - ## sudo sh -x ./bootstrap-salt.sh + - name: Bootstrap Salt + run: | + sudo sh -x ./bootstrap-salt.sh + ## TBD Disabled until CI/CD is functional ## - name: Test Bootstrap ## run: | ## pytest --cache-clear -v -s -ra --log-cli-level=debug tests/integration/ diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 47e596a3a..735079f92 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -7246,6 +7246,11 @@ __macosx_get_packagesite_onedir_latest() { __macosx_get_packagesite_onedir() { + ## DGM debug + set -v + set -x + + echodebug "Get package site for onedir from repository" if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -ne 3 ]; then echoerror "Python version is no longer supported, only Python 3" From e5f0e78c674dafc6b4306723ef61b8c6bf687a9b Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 6 Nov 2024 18:28:46 -0700 Subject: [PATCH 026/173] Attempting to get string latest for MacOS --- bootstrap-salt.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 735079f92..921b23d94 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -7241,7 +7241,9 @@ __macosx_get_packagesite_onedir_latest() { rm -fR ${macos_versions_tmpdir} echodebug "latest MacOS release from repository found ${LATEST_VERSION}" - return "${LATEST_VERSION}" + # shellcheck disable=SC2116 + LATEST_VERSION_RET=$(echo "${LATEST_VERSION_RET}") + return "${LATEST_VERSION_RET}" } From 86bc98ad4de19b39e64edc7f70c1526ee9c3a649 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 6 Nov 2024 21:29:44 -0700 Subject: [PATCH 027/173] Updated method of getting latest MacOS version for Salt --- bootstrap-salt.sh | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 921b23d94..ab211e539 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -7223,6 +7223,10 @@ daemons_running_voidlinux() { # OS X / Darwin Install Functions # +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: __macosx_get_packagesite_onedir_latest +# DESCRIPTION: Set _PKG_VERSION to the latest for MacOS +#---------------------------------------------------------------------------------------------------------------------- __macosx_get_packagesite_onedir_latest() { ## DGM debug set -v @@ -7236,14 +7240,12 @@ __macosx_get_packagesite_onedir_latest() { cd ${macos_versions_tmpdir} || return 1 wget -r -np -nH --exclude-directories=onedir,relenv,windows -x -l 1 "$SALT_MACOS_PKGDIR_URL/" # shellcheck disable=SC2010 - LATEST_VERSION=$(ls artifactory/saltproject-generic/macos/ | grep -v 'index.html' | sort -V -u | tail -n 1) - cd ${curr_pwd} || return "${LATEST_VERSION}" + _PKG_VERSION=$(ls artifactory/saltproject-generic/macos/ | grep -v 'index.html' | sort -V -u | tail -n 1) + cd ${curr_pwd} || return "${_PKG_VERSION}" rm -fR ${macos_versions_tmpdir} - echodebug "latest MacOS release from repository found ${LATEST_VERSION}" - # shellcheck disable=SC2116 - LATEST_VERSION_RET=$(echo "${LATEST_VERSION_RET}") - return "${LATEST_VERSION_RET}" + echodebug "latest MacOS release from repository found ${_PKG_VERSION}" + } @@ -7260,11 +7262,12 @@ __macosx_get_packagesite_onedir() { fi DARWIN_ARCH=${CPU_ARCH_L} + _PKG_VERSION="" _ONEDIR_TYPE="saltproject-generic" SALT_MACOS_PKGDIR_URL="https://${_REPO_URL}/${_ONEDIR_TYPE}/macos" if [ "$(echo "$_ONEDIR_REV" | grep -E '^(latest)$')" != "" ]; then - _PKG_VERSION=$(__macosx_get_packagesite_onedir_latest) + __macosx_get_packagesite_onedir_latest elif [ "$(echo "$_ONEDIR_REV" | grep -E '^([3-9][0-9]{3}(\.[0-9]*))')" != "" ]; then _PKG_VERSION=$_ONEDIR_REV else From 505e796387135cf1c4027ba1c346bc8b2fd9142c Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 6 Nov 2024 22:17:10 -0700 Subject: [PATCH 028/173] Updated fetching MacOS Salt URL for package --- bootstrap-salt.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index ab211e539..697c30b7f 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -7271,11 +7271,12 @@ __macosx_get_packagesite_onedir() { elif [ "$(echo "$_ONEDIR_REV" | grep -E '^([3-9][0-9]{3}(\.[0-9]*))')" != "" ]; then _PKG_VERSION=$_ONEDIR_REV else - _PKG_VERSION=$(__macosx_get_packagesite_onedir_latest) + __macosx_get_packagesite_onedir_latest fi PKG="salt-${_PKG_VERSION}-py3-${DARWIN_ARCH}.pkg" - SALTPKGCONFURL="${SALT_MACOS_PKGDIR_URL}/${ONEDIR_REV}/${PKG}" + SALTPKGCONFURL="${SALT_MACOS_PKGDIR_URL}/${_PKG_VERSION}/${PKG}" + } From 4a6228abc460e29ab86557f7d6499a83e28d747c Mon Sep 17 00:00:00 2001 From: David Murphy Date: Thu, 7 Nov 2024 07:08:45 -0700 Subject: [PATCH 029/173] Removed MacOs debug, and updated Photon to allow for no config-manager with tdnf --- bootstrap-salt.sh | 51 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 39 insertions(+), 12 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 697c30b7f..d52214fd4 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -6034,20 +6034,53 @@ __install_saltstack_photon_onedir_repository() { fi if [ ! -s "$YUM_REPO_FILE" ] || [ "$_FORCE_OVERWRITE" -eq $BS_TRUE ]; then - FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" - __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" + ## Photon tdnf doesn't support config-manager + ## FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" + ## __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" + # shellcheck disable=SC2129 if [ "$ONEDIR_REV" != "latest" ]; then # 3006.x is default REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) if [ "$REPO_REV_MAJOR" -eq "3007" ]; then # Enable the Salt 3007 STS repo - dnf config-manager --set-disable salt-repo-* - dnf config-manager --set-enabled salt-repo-3007-sts + ## tdnf config-manager --set-disable salt-repo-* + ## tdnf config-manager --set-enabled salt-repo-3007-sts + echo "[salt-repo-3007-sts]" > "${YUM_REPO_FILE}" + echo "name=Salt Repo for Salt v3007 STS" >> "${YUM_REPO_FILE}" + echo "baseurl=https://packages.broadcom.com/artifactory/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "exclude=*3006* *3008* *3009* *3010*" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://packages.broadcom.com/artifactory/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" + else + # Salt 3006 repo + echo "[salt-repo-3006-lts]" > "${YUM_REPO_FILE}" + echo "name=Salt Repo for Salt v3006 LTS" >> "${YUM_REPO_FILE}" + echo "baseurl=https://packages.broadcom.com/artifactory/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "exclude=*3007* *3008* *3009* *3010*" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://packages.broadcom.com/artifactory/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" fi else # Enable the Salt LATEST repo - dnf config-manager --set-disable salt-repo-* - dnf config-manager --set-enabled salt-repo-latest + ## tdnf config-manager --set-disable salt-repo-* + ## tdnf config-manager --set-enabled salt-repo-latest + echo "[salt-repo-latest]" > "${YUM_REPO_FILE}" + echo "name=Salt Repo for Salt LATEST release" >> "${YUM_REPO_FILE}" + echo "baseurl=https://packages.broadcom.com/artifactory/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://packages.broadcom.com/artifactory/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" fi tdnf makecache || return 1 elif [ "$ONEDIR_REV" != "latest" ]; then @@ -7228,9 +7261,6 @@ daemons_running_voidlinux() { # DESCRIPTION: Set _PKG_VERSION to the latest for MacOS #---------------------------------------------------------------------------------------------------------------------- __macosx_get_packagesite_onedir_latest() { - ## DGM debug - set -v - set -x echodebug "Find latest MacOS release from repository" @@ -7250,9 +7280,6 @@ __macosx_get_packagesite_onedir_latest() { __macosx_get_packagesite_onedir() { - ## DGM debug - set -v - set -x echodebug "Get package site for onedir from repository" From aef73e4b566d36c5f9b9fd5fa12501587cdd375e Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 1 Nov 2024 14:57:44 -0600 Subject: [PATCH 030/173] Update windows bootstrap for new repo --- bootstrap-salt.ps1 | 294 +++++++++++++++------------------------------ 1 file changed, 99 insertions(+), 195 deletions(-) diff --git a/bootstrap-salt.ps1 b/bootstrap-salt.ps1 index eccbb3526..9ab42d528 100644 --- a/bootstrap-salt.ps1 +++ b/bootstrap-salt.ps1 @@ -74,15 +74,6 @@ param( #Name or IP of the master server. Installer defaults to "salt". [String]$Master = "not-specified", - [Parameter(Mandatory=$false, ValueFromPipeline=$True)] - [Alias("r")] - # URL to the windows packages. Will look for a file named repo.json at the - # root of the URL. This file is used to determine the name and location of - # the installer in the repo. If repo.json is not found, it will look for the - # file under the minor directory. - # Default is "https://repo.saltproject.io/salt/py3/windows" - [String]$RepoUrl = "https://repo.saltproject.io/salt/py3/windows", - [Parameter(Mandatory=$false, ValueFromPipeline=$True)] [Alias("c")] # Vagrant only @@ -303,14 +294,26 @@ if (!(Get-IsAdministrator)) { #=============================================================================== # Change RepoUrl for older versions #=============================================================================== -$defaultUrl = "https://repo.saltproject.io/salt/py3/windows" -$oldRepoUrl = "https://repo.saltproject.io/windows" $majorVersion = Get-MajorVersion -Version $Version -if ( [Uri]($RepoUrl).AbsoluteUri -eq $defaultUrl ) { - # No customURL passed, let's check for a pre 3006 version - if ($majorVersion -lt "3006") { - # This is an older version, use the old URL - $RepoUrl = $oldRepoUrl +if ($majorVersion -lt "3006") { + # This is an older version, use the old URL + Write-Host "Versions older than 3006 are not available" -ForegroundColor Red + exit 1 +} + +#=============================================================================== +# Declare variables +#=============================================================================== +$ConfDir = "$RootDir\conf" +$PkiDir = "$ConfDir\pki\minion" +$RootDir = "$env:ProgramData\Salt Project\Salt" +$ApiUrl = "https://packages.broadcom.com/artifactory/api/storage/saltproject-generic/windows" + +# Check for existing installation where RootDir is stored in the registry +$SaltRegKey = "HKLM:\SOFTWARE\Salt Project\Salt" +if (Test-Path -Path $SaltRegKey) { + if ($null -ne (Get-ItemProperty $SaltRegKey).root_dir) { + $RootDir = (Get-ItemProperty $SaltRegKey).root_dir } } @@ -324,7 +327,9 @@ Write-Verbose "version: $Version" Write-Verbose "runservice: $RunService" Write-Verbose "master: $Master" Write-Verbose "minion: $Minion" -Write-Verbose "repourl: $RepoUrl" +Write-Verbose "apiurl: $ApiUrl" +Write-Verbose "ConfDir: $ConfDir" +Write-Verbose "RootDir: $RootDir" if ($RunService.ToLower() -eq "true") { Write-Verbose "Windows service will be set to run" @@ -339,34 +344,11 @@ if ($RunService.ToLower() -eq "true") { } #=============================================================================== -# Ensure Directories are present, copy Vagrant Configs if found +# Copy Vagrant Files to their proper location. #=============================================================================== $ConfiguredAnything = $False -# Detect older version of Salt to determing default RootDir -if ($majorVersion -lt 3004) { - $RootDir = "$env:SystemDrive`:\salt" -} else { - $RootDir = "$env:ProgramData\Salt Project\Salt" -} - -# Check for existing installation where RootDir is stored in the registry -$SaltRegKey = "HKLM:\SOFTWARE\Salt Project\Salt" -if (Test-Path -Path $SaltRegKey) { - if ($null -ne (Get-ItemProperty $SaltRegKey).root_dir) { - $RootDir = (Get-ItemProperty $SaltRegKey).root_dir - } -} - -$ConfDir = "$RootDir\conf" -$PkiDir = "$ConfDir\pki\minion" -Write-Verbose "ConfDir: $ConfDir" - -#=============================================================================== -# Copy Vagrant Files to their proper location. -#=============================================================================== - # Vagrant files will be placed in C:\tmp # Check if minion keys have been uploaded, copy to correct location if (Test-Path C:\tmp\minion.pem) { @@ -413,146 +395,64 @@ if ([IntPtr]::Size -eq 4) { } #=============================================================================== -# Get file name to download +# Getting version information from the repo #=============================================================================== -$saltFileName = "" -$saltVersion = "" -$saltSha512= "" -$saltFileUrl = "" -# Look for a repo.json file -try { - Write-Verbose "Looking for $RepoUrl/repo.json" - $response = Invoke-WebRequest "$RepoUrl/repo.json" ` - -DisableKeepAlive ` - -UseBasicParsing ` - -Method Head - if ( $response.StatusCode -eq "200" ) { - Write-Verbose "Found $RepoUrl/repo.json" - # This URL contains a repo.json file, let's use it - $use_repo_json = $true - } else { - Write-Verbose "Did not find $RepoUrl/repo.json" - # No repo.json file found at the default location - $use_repo_json = $false - } -} catch { - Write-Verbose "There was an error looking up $RepoUrl/repo.json" - Write-Verbose "ERROR: $_" - $use_repo_json = $false +Write-Verbose "Getting version information from Artifactory" +$response = Invoke-WebRequest $ApiUrl +# Convert the output to a powershell object +$psobj = $response.ToString() | ConvertFrom-Json + +# Filter the object for folders +$filtered = $psobj.children | Where-Object -Property folder -EQ $true + +# Get each uri and add it to the list of versions +$versions = [System.Collections.ArrayList]@() +$filtered | Select-Object -Property uri | ForEach-Object { + $versions.Add($_.uri.Trim("/")) | Out-Null } -if ( $use_repo_json ) { - # We will use the json file to get the name of the installer - $enc = [System.Text.Encoding]::UTF8 - try { - Write-Verbose "Downloading $RepoUrl/repo.json" - $response = Invoke-WebRequest -Uri "$RepoUrl/repo.json" -UseBasicParsing - if ($response.Content.GetType().Name -eq "Byte[]") { - $psobj = $enc.GetString($response.Content) | ConvertFrom-Json - } else { - $psobj = $response.Content | ConvertFrom-Json - } - $hash = Convert-PSObjectToHashtable $psobj - } catch { - Write-Verbose "repo.json not found at: $RepoUrl" - Write-Host "ERROR: $_" - $hash = @{} - } - $searchVersion = $Version.ToLower() - if ( $hash.Contains($searchVersion)) { - Write-Verbose "Found $searchVersion in $RepoUrl/repo.json" - foreach ($item in $hash.($searchVersion).Keys) { - if ( $item.ToLower().EndsWith(".exe") ) { - if ( $item.ToLower().Contains($arch.ToLower()) ) { - $saltFileName = $hash.($searchVersion).($item).name - $saltVersion = $hash.($searchVersion).($item).version - $saltSha512 = $hash.($searchVersion).($item).SHA512 - } - } - } - } else { - try { - Write-Verbose "Searching for $searchVersion in $RepoUrl/minor/repo.json" - $response = Invoke-WebRequest -Uri "$RepoUrl/minor/repo.json" -UseBasicParsing - if ($response.Content.GetType().Name -eq "Byte[]") { - $psobj = $enc.GetString($response.Content) | ConvertFrom-Json - } else { - $psobj = $response.Content | ConvertFrom-Json - } - $hash = Convert-PSObjectToHashtable $psobj - } catch { - Write-Verbose "repo.json not found at: $RepoUrl/minor/repo.json" - Write-Verbose "ERROR: $_" - $hash = @{} - } - if ( $hash.Contains($searchVersion)) { - Write-Verbose "Found $searchVersion in $RepoUrl/minor/repo.json" - foreach ($item in $hash.($searchVersion).Keys) { - if ( $item.ToLower().EndsWith(".exe") ) { - if ( $item.ToLower().Contains($arch.ToLower()) ) { - $saltFileName = $hash.($searchVersion).($item).name - $saltVersion = $hash.($searchVersion).($item).version - $saltSha512 = $hash.($searchVersion).($item).SHA512 - } - } - } - } else { - Write-Verbose "Version not found in $RepoUrl/minor/repo.json" - } - } +# last one in the list is the latest +$latest = $versions | Select-Object -Last 1 + +Write-Verbose "Available versions:" +$versions | ForEach-Object { + Write-Verbose $_ } +Write-Verbose "Latest version: $latest" -if ( $saltFileName -and $saltVersion -and $saltSha512 ) { +if ( $Version -EQ "latest") { + $Version = $latest +} + +#=============================================================================== +# Validate passed version +#=============================================================================== +if ( $versions -notcontains $Version ) { + Write-Host "Version $Version is not available" -ForegroundColor Red + Write-Host "Available versions are:" -ForegroundColor Yellow + $versions | ForEach-Object { Write-Host $_ -ForegroundColor Yellow } + exit 1 +} + +#=============================================================================== +# Get file name to download +#=============================================================================== +$saltFileName = "Salt-Minion-$Version-Py3-$arch-Setup.exe" +$response = Invoke-WebRequest "$ApiUrl/$Version/$saltFileName" +$psobj = $response.ToString() | ConvertFrom-Json +$saltFileUrl = $psobj.downloadUri +$saltSha256 = $psobj.checksums.sha256 + +if ( $saltFileName -and $saltVersion -and $saltSha256) { Write-Verbose "Found Name, Version, and Sha" } else { # We will guess the name of the installer - Write-Verbose "Failed to get Name, Version, and Sha from repo.json" + Write-Verbose "Failed to get Name, Version, and Sha from Artifactory API" Write-Verbose "We'll try to find the file in standard paths" $saltFileName = "Salt-Minion-$Version-Py3-$arch-Setup.exe" $saltVersion = $Version } -Write-Verbose "Creating list of urls using the following:" -Write-Verbose "RepoUrl: $RepoUrl" -Write-Verbose "Version: $saltVersion" -Write-Verbose "File Name: $saltFileName" -$urls = $(@($RepoUrl, $saltVersion, $saltFileName) -join "/"), - $(@($RepoUrl, "minor", $saltVersion, $saltFileName) -join "/"), - $(@($RepoUrl, $saltFileName) -join "/"), - $(@($oldRepoUrl, $saltFileName) -join "/") - -$saltFileUrl = $null - -foreach ($url in $urls) { - try { - Write-Verbose "Looking for installer at: $url" - $response = Invoke-WebRequest "$url" ` - -DisableKeepAlive ` - -UseBasicParsing ` - -Method Head - if ( $response.StatusCode -eq "200" ) { - Write-Verbose "Found installer" - # This URL contains a repo.json file, let's use it - $saltFileUrl = $url - break - } else { - Write-Verbose "Installer not found: $url" - } - } catch { - Write-Verbose "ERROR: $url" - } -} - -if ( !$saltFileUrl ) { - Write-Host "Could not find an installer:" - Write-Verbose "Here are the urls searched:" - foreach ($url in $urls) { - Write-Verbose $url - } - exit 1 -} - - #=============================================================================== # Download minion setup file #=============================================================================== @@ -561,6 +461,7 @@ Write-Host " Bootstrapping Salt Minion" -ForegroundColor Green Write-Host " - version: $Version" Write-Host " - file name: $saltFileName" Write-Host " - file url: $saltFileUrl" +Write-Host " - file hash: $saltSha256" Write-Host " - master: $Master" Write-Host " - minion id: $Minion" Write-Host " - start service: $RunService" @@ -580,15 +481,16 @@ if ( Test-Path -Path $localFile ) { Write-Host "Success" -ForegroundColor Green } else { Write-Host "Failed" -ForegroundColor Red + exit 1 } -if ( $saltSha512 ) { - $localSha512 = (Get-FileHash -Path $localFile -Algorithm SHA512).Hash +if ( $saltSha256 ) { + $localSha256 = (Get-FileHash -Path $localFile -Algorithm SHA256).Hash Write-Host "Comparing Hash: " -NoNewline Write-Verbose "" - Write-Verbose "Local Hash: $localSha512" - Write-Verbose "Remote Hash: $saltSha512" - if ( $localSha512 -eq $saltSha512 ) { + Write-Verbose "Local Hash: $localSha256" + Write-Verbose "Remote Hash: $saltSha256" + if ( $localSha256 -eq $saltSha256 ) { Write-Host "Success" -ForegroundColor Green } else { Write-Host "Failed" -ForegroundColor Red @@ -626,7 +528,7 @@ $process | Wait-Process -Timeout 300 -ErrorAction SilentlyContinue $process.Refresh() if ( !$process.HasExited ) { - Write-Host "Timedout" -ForegroundColor Yellow + Write-Host "Installer Timeout" -ForegroundColor Yellow Write-Host "Killing hung installer: " -NoNewline $process | Stop-Process $process.Refresh() @@ -636,8 +538,6 @@ if ( !$process.HasExited ) { Write-Host "Failed" -ForegroundColor Red exit 1 } - - Write-Host "Checking installed service: " -NoNewline } # Wait for salt-minion service to be registered to verify successful @@ -658,7 +558,7 @@ while ( ! $service ) { # probably went wrong and user intervention is required - report # failure. Write-Host "Failed" -ForegroundColor Red - Write-Host "Timed out waiting for the salt-minion service to be installed" + Write-Host "Timeout waiting for the salt-minion service to be installed" exit 1 } } @@ -676,27 +576,31 @@ if( $RunService ) { # We'll try for 2 minutes, sometimes the minion takes that long to start as # it compiles python code for the first time $max_tries = 60 - while ( $service.Status -ne "Running" ) { - if ( $service.Status -eq "Stopped" ) { - Start-Service -Name "salt-minion" -ErrorAction SilentlyContinue - } - Start-Sleep -Seconds 2 - Write-Verbose "Checking the service status" - $service.Refresh() - if ( $service.Status -eq "Running" ) { - Write-Host "Success" -ForegroundColor Green - } else { - if ( $tries -le $max_tries ) { - $tries += 1 + if ( $service.Status -ne "Running" ) { + while ( $service.Status -ne "Running" ) { + if ( $service.Status -eq "Stopped" ) { + Start-Service -Name "salt-minion" -ErrorAction SilentlyContinue + } + Start-Sleep -Seconds 2 + Write-Verbose "Checking the service status" + $service.Refresh() + if ( $service.Status -eq "Running" ) { + Write-Host "Success" -ForegroundColor Green } else { - # If the salt-minion service is still not running, something - # probably went wrong and user intervention is required - report - # failure. - Write-Host "Failed" -ForegroundColor Red - Write-Host "Timed out waiting for the salt-minion service to start" - exit 1 + if ( $tries -le $max_tries ) { + $tries += 1 + } else { + # If the salt-minion service is still not running, something + # probably went wrong and user intervention is required - report + # failure. + Write-Host "Failed" -ForegroundColor Red + Write-Host "Timed out waiting for the salt-minion service to start" + exit 1 + } } } + } else { + Write-Host "Success" -ForegroundColor Green } } else { # Set the service to manual start From de5f8d4292425d5f6cf163b803210fa7654f9e5f Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 4 Nov 2024 13:18:55 -0700 Subject: [PATCH 031/173] Update tests on Windows --- .github/workflows/ci.yml | 17 ---------- .github/workflows/templates/generate.py | 2 -- .github/workflows/test-windows.yml | 1 - kitchen.windows.yml | 42 ------------------------- tests/integration/test_installation.py | 19 ++++++----- tests/requirements.txt | 1 + 6 files changed, 10 insertions(+), 72 deletions(-) delete mode 100644 kitchen.windows.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fc4b306f0..e55ea4e9b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -144,22 +144,6 @@ jobs: instances: '["stable-3006", "stable-3006-8", "stable-3007", "stable-3007-1", "latest"]' - - windows-2019: - name: Windows 2019 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-windows.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: windows-2019 - display-name: Windows 2019 - timeout: 20 - runs-on: windows-2019 - instances: '["stable-3006", "stable-3006-8", "stable-3007", "stable-3007-1", "latest"]' - - windows-2022: name: Windows 2022 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' @@ -480,7 +464,6 @@ jobs: - generate-actions-workflow - macos-12 - macos-13 - - windows-2019 - windows-2022 - almalinux-8 - almalinux-9 diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index 0fb618158..f285da8c4 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -32,7 +32,6 @@ "ubuntu-2404", ] WINDOWS = [ - "windows-2019", "windows-2022", ] @@ -265,7 +264,6 @@ "ubuntu-2404": "Ubuntu 24.04", "macos-12": "macOS 12", "macos-13": "macOS 13", - "windows-2019": "Windows 2019", "windows-2022": "Windows 2022", } diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index 4bc98f4d4..719e34f8b 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -41,7 +41,6 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Set up Python 3.10 uses: actions/setup-python@v5 with: diff --git a/kitchen.windows.yml b/kitchen.windows.yml deleted file mode 100644 index a2b88e2e3..000000000 --- a/kitchen.windows.yml +++ /dev/null @@ -1,42 +0,0 @@ ---- -driver: - name: proxy - host: localhost - reset_command: "exit 0" - port: 5985 - username: kitchen - password: Password1 - -provisioner: - salt_bootstrap_url: D:/a/salt-bootstrap/salt-bootstrap/bootstrap-salt.ps1 - salt_bootstrap_options: -Version %s -Verbose - init_environment: '' - -platforms: - - name: windows-2022 - - name: windows-2019 - -suites: - - name: stable-3006 - provisioner: - salt_version: 3006 - salt_call_command: c:\Program Files\Salt Project\Salt\salt-call.exe - - name: stable-3006-8 - provisioner: - salt_version: 3006.8 - salt_call_command: c:\Program Files\Salt Project\Salt\salt-call.exe - - name: stable-3007 - provisioner: - salt_version: 3007 - salt_call_command: c:\Program Files\Salt Project\Salt\salt-call.exe - - name: stable-3007-1 - provisioner: - salt_version: 3007.1 - salt_call_command: c:\Program Files\Salt Project\Salt\salt-call.exe - - name: latest - provisioner: - salt_version: latest - salt_call_command: c:\Program Files\Salt Project\Salt\salt-call.exe - -verifier: - command: pytest --cache-clear -v -s -ra --log-cli-level=debug -k "not test_ping" tests/integration/ diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index 8d0b41467..f8427b675 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -1,12 +1,11 @@ import json import logging +import os import platform import subprocess import pytest -## from Demos.win32cred_demo import target - log = logging.getLogger(__name__) @@ -26,19 +25,17 @@ def run_salt_call(cmd): """ cmd.append("--out=json") result = subprocess.run(cmd, capture_output=True, text=True) - if result.stdout: - json_data = json.loads(result.stdout) - return json_data["local"] - return None + json_data = json.loads(result.stdout) + return json_data["local"] -def test_ping(): +def test_ping(path): cmd = ["salt-call", "--local", "test.ping"] result = run_salt_call(cmd) assert result == True -def test_target_python_version(target_python_version): +def test_target_python_version(path, target_python_version): cmd = ["salt-call", "--local", "grains.item", "pythonversion", "--timeout=120"] result = run_salt_call(cmd) # Returns: {'pythonversion': [3, 10, 11, 'final', 0]} @@ -46,8 +43,10 @@ def test_target_python_version(target_python_version): assert py_maj_ver == target_python_version -def test_target_salt_version(target_salt_version): +def test_target_salt_version(path, target_salt_version): + if not target_salt_version: + pytest.skip(f"No target version specified") cmd = ["salt-call", "--local", "grains.item", "saltversion", "--timeout=120"] result = run_salt_call(cmd) # Returns: {'saltversion': '3006.9+217.g53cfa53040'} - assert result["saltversion"] == target_salt_version + assert result["saltversion"] == target_salt_version \ No newline at end of file diff --git a/tests/requirements.txt b/tests/requirements.txt index e079f8a60..16d3b7789 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1 +1,2 @@ pytest +requests \ No newline at end of file From 2fa863a9e6b6930c6cc33fa66c4f3773a9481348 Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 4 Nov 2024 13:20:32 -0700 Subject: [PATCH 032/173] Add -UseBasicParsing switch --- bootstrap-salt.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bootstrap-salt.ps1 b/bootstrap-salt.ps1 index 9ab42d528..a20b67078 100644 --- a/bootstrap-salt.ps1 +++ b/bootstrap-salt.ps1 @@ -398,7 +398,7 @@ if ([IntPtr]::Size -eq 4) { # Getting version information from the repo #=============================================================================== Write-Verbose "Getting version information from Artifactory" -$response = Invoke-WebRequest $ApiUrl +$response = Invoke-WebRequest $ApiUrl -UseBasicParsing # Convert the output to a powershell object $psobj = $response.ToString() | ConvertFrom-Json @@ -438,7 +438,7 @@ if ( $versions -notcontains $Version ) { # Get file name to download #=============================================================================== $saltFileName = "Salt-Minion-$Version-Py3-$arch-Setup.exe" -$response = Invoke-WebRequest "$ApiUrl/$Version/$saltFileName" +$response = Invoke-WebRequest "$ApiUrl/$Version/$saltFileName" -UseBasicParsing $psobj = $response.ToString() | ConvertFrom-Json $saltFileUrl = $psobj.downloadUri $saltSha256 = $psobj.checksums.sha256 From f09afd7ceec873dc85d3560afc97eb0b157e5603 Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 4 Nov 2024 13:25:15 -0700 Subject: [PATCH 033/173] Generate workflow --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e55ea4e9b..91019ecf8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -144,6 +144,7 @@ jobs: instances: '["stable-3006", "stable-3006-8", "stable-3007", "stable-3007-1", "latest"]' + windows-2022: name: Windows 2022 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' From d013827c0608fd89a8ee6c88ed63688331de846d Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 4 Nov 2024 13:34:22 -0700 Subject: [PATCH 034/173] Refresh environment variables --- .github/workflows/test-windows.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index 719e34f8b..fc906dd3f 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -63,6 +63,13 @@ jobs: Write-Output "Environment Variables" Get-ChildItem Env: + - name: Refresh Environment + run: | + foreach ($level in "Machine", "User") { + $vars = [Environment]::GetEnvironmentVariables($level).GetEnumerator() + $vars | ForEach-Object { $_ } | Set-Content -Path { "Env:$( $_.Name )" } + } + - name: Test Bootstrap run: | pytest --cache-clear -v -s -ra --log-cli-level=debug tests/integration/ From f36c2c06a324d8100207cd3c6d81c2e9b2a3d391 Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 4 Nov 2024 14:19:03 -0700 Subject: [PATCH 035/173] Trim what we're testing on --- .github/workflows/ci.yml | 318 ------------------------ .github/workflows/templates/generate.py | 219 +--------------- .github/workflows/test-windows.yml | 7 +- kitchen.macos.yml | 42 ---- kitchen.yml | 138 ---------- 5 files changed, 8 insertions(+), 716 deletions(-) delete mode 100644 kitchen.macos.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 91019ecf8..39f042d7e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -114,37 +114,6 @@ jobs: - macos-12: - name: macOS 12 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-macos.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: macos-12 - display-name: macOS 12 - timeout: 20 - runs-on: macos-12 - instances: '["stable-3006", "stable-3006-8", "stable-3007", "stable-3007-1", "latest"]' - - - macos-13: - name: macOS 13 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-macos.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: macos-13 - display-name: macOS 13 - timeout: 20 - runs-on: macos-13 - instances: '["stable-3006", "stable-3006-8", "stable-3007", "stable-3007-1", "latest"]' - - - windows-2022: name: Windows 2022 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' @@ -161,244 +130,6 @@ jobs: - almalinux-8: - name: AlmaLinux 8 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: almalinux-8 - display-name: AlmaLinux 8 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' - - - almalinux-9: - name: AlmaLinux 9 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: almalinux-9 - display-name: AlmaLinux 9 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - - - amazon-2: - name: Amazon 2 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: amazon-2 - display-name: Amazon 2 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' - - - arch: - name: Arch - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: arch - display-name: Arch - timeout: 20 - instances: '["git-master", "latest", "default"]' - - - centos-stream9: - name: CentOS Stream 9 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: centos-stream9 - display-name: CentOS Stream 9 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - - - debian-11: - name: Debian 11 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: debian-11 - display-name: Debian 11 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - - - debian-12: - name: Debian 12 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: debian-12 - display-name: Debian 12 - timeout: 20 - instances: '["stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - - - fedora-39: - name: Fedora 39 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: fedora-39 - display-name: Fedora 39 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "latest", "default"]' - - - fedora-40: - name: Fedora 40 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: fedora-40 - display-name: Fedora 40 - timeout: 20 - instances: '["stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - - - gentoo: - name: Gentoo - if: github.event_name == 'push' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: gentoo - display-name: Gentoo - timeout: 90 - instances: '["git-master"]' - - - gentoo-systemd: - name: Gentoo (systemd) - if: github.event_name == 'push' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: gentoo-systemd - display-name: Gentoo (systemd) - timeout: 90 - instances: '["git-master"]' - - - opensuse-15: - name: Opensuse 15 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: opensuse-15 - display-name: Opensuse 15 - timeout: 20 - instances: '["latest", "default"]' - - - oraclelinux-8: - name: Oracle Linux 8 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: oraclelinux-8 - display-name: Oracle Linux 8 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' - - - oraclelinux-9: - name: Oracle Linux 9 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: oraclelinux-9 - display-name: Oracle Linux 9 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - - - photon-4: - name: Photon OS 4 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: photon-4 - display-name: Photon OS 4 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "latest", "default"]' - - - photon-5: - name: Photon OS 5 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: photon-5 - display-name: Photon OS 5 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "latest", "default"]' - - - rockylinux-8: - name: Rocky Linux 8 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: rockylinux-8 - display-name: Rocky Linux 8 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' - - rockylinux-9: name: Rocky Linux 9 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' @@ -413,34 +144,6 @@ jobs: instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - ubuntu-2004: - name: Ubuntu 20.04 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: ubuntu-2004 - display-name: Ubuntu 20.04 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - - - ubuntu-2204: - name: Ubuntu 22.04 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: ubuntu-2204 - display-name: Ubuntu 22.04 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - - ubuntu-2404: name: Ubuntu 24.04 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' @@ -463,29 +166,8 @@ jobs: needs: - lint - generate-actions-workflow - - macos-12 - - macos-13 - windows-2022 - - almalinux-8 - - almalinux-9 - - amazon-2 - - arch - - centos-stream9 - - debian-11 - - debian-12 - - fedora-39 - - fedora-40 - - gentoo - - gentoo-systemd - - opensuse-15 - - oraclelinux-8 - - oraclelinux-9 - - photon-4 - - photon-5 - - rockylinux-8 - rockylinux-9 - - ubuntu-2004 - - ubuntu-2204 - ubuntu-2404 if: always() steps: diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index f285da8c4..3d842dd4f 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -6,171 +6,46 @@ os.chdir(os.path.abspath(os.path.dirname(__file__))) -## "amazon-2023", -## "opensuse-tumbleweed", LINUX_DISTROS = [ - "almalinux-8", - "almalinux-9", - "amazon-2", - "arch", - "centos-stream9", - "debian-11", - "debian-12", - "fedora-39", - "fedora-40", - "gentoo", - "gentoo-systemd", - "opensuse-15", - "oraclelinux-8", - "oraclelinux-9", - "photon-4", - "photon-5", - "rockylinux-8", "rockylinux-9", - "ubuntu-2004", - "ubuntu-2204", "ubuntu-2404", ] WINDOWS = [ "windows-2022", ] -OSX = [ - "macos-12", - "macos-13", -] - - -## "amazon-2023", -## "opensuse-tumbleweed", STABLE_DISTROS = [ - "almalinux-8", - "almalinux-9", - "amazon-2", - "arch", - "centos-stream9", - "debian-11", - "debian-12", - "fedora-39", - "fedora-40", - "opensuse-15", - "oraclelinux-8", - "oraclelinux-9", - "photon-4", - "photon-5", - "rockylinux-8", "rockylinux-9", - "ubuntu-2004", - "ubuntu-2204", "ubuntu-2404", ] -## "amazon-2023", ONEDIR_DISTROS = [ - "almalinux-8", - "almalinux-9", - "amazon-2", - "centos-stream9", - "debian-11", - "debian-12", - "fedora-39", - "fedora-40", - "oraclelinux-8", - "oraclelinux-9", - "photon-4", - "photon-5", - "rockylinux-8", "rockylinux-9", - "ubuntu-2004", - "ubuntu-2204", "ubuntu-2404", ] ONEDIR_RC_DISTROS = [ - "almalinux-9", - "amazon-2", - "centos-stream9", - "debian-12", - "oraclelinux-9", - "photon-4", - "photon-5", "rockylinux-9", "ubuntu-2404", ] -## "opensuse-tumbleweed", BLACKLIST_3006 = [ - "arch", - "debian-12", - "fedora-40", - "gentoo", - "gentoo-systemd", - "opensuse-15", "ubuntu-2404", ] -## "opensuse-tumbleweed", -BLACKLIST_3007 = [ - "arch", - "fedora-39", - "gentoo", - "gentoo-systemd", - "opensuse-15", - "photon-4", - "photon-5", -] +BLACKLIST_3007 = [] -## "amazon-2023", -## "opensuse-tumbleweed", BLACKLIST_GIT_3006 = [ - "almalinux-9", - "amazon-2", - "arch", - "centos-stream9", - "debian-11", - "debian-12", - "fedora-40", - "gentoo", - "gentoo-systemd", - "opensuse-15", - "oraclelinux-9", - "photon-4", - "photon-5", "rockylinux-9", - "ubuntu-2004", - "ubuntu-2204", "ubuntu-2404", ] -## "amazon-2023", -## "opensuse-tumbleweed", BLACKLIST_GIT_3007 = [ - "almalinux-9", - "amazon-2", - "arch", - "centos-stream9", - "debian-11", - "debian-12", - "fedora-39", - "fedora-40", - "gentoo", - "gentoo-systemd", - "opensuse-15", - "oraclelinux-9", - "photon-4", - "photon-5", "rockylinux-9", - "ubuntu-2004", - "ubuntu-2204", "ubuntu-2404", ] -BLACKLIST_GIT_MASTER = [ - "amazon-2", - "fedora-39", - "photon-4", - "photon-5", -] +BLACKLIST_GIT_MASTER = [] SALT_VERSIONS = [ "3006", @@ -225,57 +100,19 @@ # SetuptoolsDeprecationWarning: setup.py install is deprecated. # Use build and pip and other standards-based tools. # -GIT_DISTRO_BLACKLIST = [ - "almalinux-8", - "fedora-39", - "opensuse-15", - "oraclelinux-8", - "rockylinux-8", -] +GIT_DISTRO_BLACKLIST = [] -LATEST_PKG_BLACKLIST = [ - "gentoo", - "gentoo-systemd", -] +LATEST_PKG_BLACKLIST = [] -## "amazon-2023": "Amazon 2023", -## "opensuse-tumbleweed": "Opensuse Tumbleweed", DISTRO_DISPLAY_NAMES = { - "almalinux-8": "AlmaLinux 8", - "almalinux-9": "AlmaLinux 9", - "amazon-2": "Amazon 2", - "arch": "Arch", - "centos-stream9": "CentOS Stream 9", - "debian-11": "Debian 11", - "debian-12": "Debian 12", - "fedora-39": "Fedora 39", - "fedora-40": "Fedora 40", - "gentoo": "Gentoo", - "gentoo-systemd": "Gentoo (systemd)", - "opensuse-15": "Opensuse 15", - "oraclelinux-8": "Oracle Linux 8", - "oraclelinux-9": "Oracle Linux 9", - "photon-4": "Photon OS 4", - "photon-5": "Photon OS 5", - "rockylinux-8": "Rocky Linux 8", "rockylinux-9": "Rocky Linux 9", - "ubuntu-2004": "Ubuntu 20.04", - "ubuntu-2204": "Ubuntu 22.04", "ubuntu-2404": "Ubuntu 24.04", - "macos-12": "macOS 12", - "macos-13": "macOS 13", "windows-2022": "Windows 2022", } TIMEOUT_DEFAULT = 20 -TIMEOUT_OVERRIDES = { - "gentoo": 90, - "gentoo-systemd": 90, -} -VERSION_ONLY_OVERRIDES = [ - "gentoo", - "gentoo-systemd", -] +TIMEOUT_OVERRIDES = {} +VERSION_ONLY_OVERRIDES = [] TEMPLATE = """ {distro}: @@ -296,50 +133,6 @@ def generate_test_jobs(): test_jobs = "" needs = ["lint", "generate-actions-workflow"] - test_jobs += "\n" - for distro in OSX: - test_jobs += "\n" - runs_on = distro - runs_on = f"\n runs-on: {runs_on}" - ifcheck = "\n if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true'" - uses = "./.github/workflows/test-macos.yml" - instances = [] - timeout_minutes = ( - TIMEOUT_OVERRIDES[distro] - if distro in TIMEOUT_OVERRIDES - else TIMEOUT_DEFAULT - ) - - for salt_version in SALT_VERSIONS: - if salt_version == "latest": - instances.append(salt_version) - continue - - for bootstrap_type in ["stable"]: - if bootstrap_type == "stable": - if salt_version in MAC_STABLE_VERSION_BLACKLIST: - continue - - test_target = f"{bootstrap_type}-{salt_version}" - instances.append(test_target) - - for bootstrap_type in ["default"]: - if distro not in STABLE_DISTROS: - continue - instances.append(bootstrap_type) - - if instances: - needs.append(distro) - test_jobs += TEMPLATE.format( - distro=distro, - runs_on=runs_on, - uses=uses, - ifcheck=ifcheck, - instances=json.dumps(instances), - display_name=DISTRO_DISPLAY_NAMES[distro], - timeout_minutes=timeout_minutes, - ) - test_jobs += "\n" for distro in WINDOWS: test_jobs += "\n" diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index fc906dd3f..ecc332c51 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -63,12 +63,9 @@ jobs: Write-Output "Environment Variables" Get-ChildItem Env: - - name: Refresh Environment + - name: Add Salt to Path run: | - foreach ($level in "Machine", "User") { - $vars = [Environment]::GetEnvironmentVariables($level).GetEnumerator() - $vars | ForEach-Object { $_ } | Set-Content -Path { "Env:$( $_.Name )" } - } + $env:Path = "$env:Path;C:\Program Files\Salt Project\Salt" - name: Test Bootstrap run: | diff --git a/kitchen.macos.yml b/kitchen.macos.yml deleted file mode 100644 index 616a489e0..000000000 --- a/kitchen.macos.yml +++ /dev/null @@ -1,42 +0,0 @@ ---- -driver: - name: exec - -provisioner: - sudo: true - salt_bootstrap_options: -MP stable %s - init_environment: | - echo 'auto_accept: true' > /tmp/auto-accept-keys.conf - sudo mkdir -p /etc/salt/master.d - sudo mv /tmp/auto-accept-keys.conf /etc/salt/master.d/auto-accept-keys.conf - brew install coreutils - sh -c 't=$(gshuf -i 1-15 -n 1); echo Sleeping $t seconds; sleep $t' - -platforms: - - name: macos-12 - - name: macos-13 - -suites: - - name: stable-3006 - provisioner: - salt_version: 3006 - salt_call_command: /opt/salt/salt-call - - name: stable-3006-8 - provisioner: - salt_version: 3006.8 - salt_call_command: /opt/salt/salt-call - - name: stable-3007 - provisioner: - salt_version: 3007 - salt_call_command: /opt/salt/salt-call - - name: stable-3007-1 - provisioner: - salt_version: 3007.1 - salt_call_command: /opt/salt/salt-call - - name: latest - provisioner: - salt_version: latest - salt_call_command: /opt/salt/salt-call - -verifier: - command: pytest --cache-clear -v -s -ra --log-cli-level=debug -k "not test_ping" tests/integration/ diff --git a/kitchen.yml b/kitchen.yml index 62cc73808..a08927578 100644 --- a/kitchen.yml +++ b/kitchen.yml @@ -29,136 +29,13 @@ provisioner: sh -c 't=$(shuf -i 1-15 -n 1); echo Sleeping $t seconds; sleep $t' -## - name: amazon-2023 -## driver: -## image: amazonlinux:2023 -## provision_command: -## - yum -y install --allowerasing procps-ng curl -## - echo "PubkeyAcceptedAlgorithms +ssh-rsa" | tee -a /etc/ssh/sshd_config - -## - name: opensuse-tumbleweed -## driver: -## image: opensuse/tumbleweed:latest -## provision_command: -## - *opensuse_provision_command_01 -## - *opensuse_provision_command_02 -## - *opensuse_provision_command_03 -## - echo "PubkeyAcceptedAlgorithms +ssh-rsa" | tee -a /etc/ssh/sshd_config - platforms: - - name: almalinux-9 - driver: - provision_command: - - dnf -y install crypto-policies-scripts procps-ng - - update-crypto-policies --set DEFAULT:SHA1 - - echo "PubkeyAcceptedAlgorithms +ssh-rsa" | tee -a /etc/ssh/sshd_config - - name: almalinux-8 - - name: amazon-2 - driver: - image: amazonlinux:2 - platform: rhel - provision_command: - - yum -y install procps-ng - - name: arch - driver: - image: archlinux/archlinux - provision_command: - - pacman -Syu --noconfirm --needed systemd grep awk procps which - - systemctl enable sshd - - echo "PubkeyAcceptedAlgorithms +ssh-rsa" | tee -a /etc/ssh/sshd_config - - name: centos-stream9 - driver: - platform: centosstream - image: quay.io/centos/centos:stream9 - provision_command: - - dnf -y install crypto-policies-scripts procps-ng - - update-crypto-policies --set DEFAULT:SHA1 - - echo "PubkeyAcceptedAlgorithms +ssh-rsa" | tee -a /etc/ssh/sshd_config - - name: debian-11 - driver: - image: debian:bullseye - run_command: /lib/systemd/systemd - - name: debian-12 - driver: - image: debian:bookworm - run_command: /lib/systemd/systemd - provision_command: - - echo "PubkeyAcceptedAlgorithms +ssh-rsa" | tee -a /etc/ssh/sshd_config - - name: fedora-39 - driver: - provision_command: &fedora_provision_command - - dnf -y install procps-ng crypto-policies-scripts - - update-crypto-policies --set LEGACY - - name: fedora-40 - driver: - provision_command: *fedora_provision_command - - name: gentoo - driver: - image: gentoo/stage3:latest - run_command: /sbin/init - provision_command: - - rc-update add sshd default - - echo "PubkeyAcceptedAlgorithms +ssh-rsa" | tee -a /etc/ssh/sshd_config - - name: gentoo-systemd - driver: - image: gentoo/stage3:systemd - run_command: /lib/systemd/systemd - provision_command: - - systemctl enable sshd.service - - echo "PubkeyAcceptedAlgorithms +ssh-rsa" | tee -a /etc/ssh/sshd_config - - name: opensuse-15 - driver: - image: opensuse/leap:15.4 - provision_command: - - &opensuse_provision_command_01 zypper --non-interactive install --auto-agree-with-licenses dbus-1 - - &opensuse_provision_command_02 zypper --non-interactive install --auto-agree-with-licenses sudo openssh which curl systemd - - &opensuse_provision_command_03 systemctl enable sshd.service - name: rockylinux-9 - driver: - platform: centosstream - run_command: /usr/lib/systemd/systemd - provision_command: - - dnf -y install crypto-policies-scripts procps-ng - - update-crypto-policies --set DEFAULT:SHA1 - - echo "PubkeyAcceptedAlgorithms +ssh-rsa" | tee -a /etc/ssh/sshd_config - - name: oraclelinux-9 - driver: - run_command: /usr/lib/systemd/systemd - provision_command: - - dnf -y install crypto-policies-scripts procps-ng - - update-crypto-policies --set DEFAULT:SHA1 - - echo "PubkeyAcceptedAlgorithms +ssh-rsa" | tee -a /etc/ssh/sshd_config - - name: rockylinux-8 - - name: oraclelinux-8 - name: ubuntu-24.04 driver: run_command: /lib/systemd/systemd provision_command: - echo "PubkeyAcceptedAlgorithms +ssh-rsa" | tee -a /etc/ssh/sshd_config - - name: ubuntu-22.04 - driver: - run_command: /lib/systemd/systemd - provision_command: - - echo "PubkeyAcceptedAlgorithms +ssh-rsa" | tee -a /etc/ssh/sshd_config - - name: ubuntu-20.04 - driver: - run_command: /lib/systemd/systemd - - name: photon-4 - driver: - image: photon:4.0 - provision_command: - - tdnf -y install rpm procps-ng coreutils gawk systemd - - echo "PubkeyAcceptedKeyTypes +ssh-rsa" | tee -a /etc/ssh/sshd_config - - sed -ie 's/PermitRootLogin no/PermitRootLogin yes/' /etc/ssh/sshd_config - - systemctl enable sshd.service - - name: photon-5 - driver: - image: photon:5.0 - provision_command: - - tdnf -y install rpm procps-ng coreutils gawk systemd - - echo "PubkeyAcceptedKeyTypes +ssh-rsa" | tee -a /etc/ssh/sshd_config - - sed -ie 's/PermitRootLogin no/PermitRootLogin yes/' /etc/ssh/sshd_config - - systemctl enable sshd.service suites: - name: git-3006 @@ -168,9 +45,6 @@ suites: excludes: - opensuse-15 - opensuse-tumbleweed - - debian-11 - - debian-12 - - arch - gentoo - gentoo-systemd - name: git-3006x @@ -180,9 +54,6 @@ suites: excludes: - opensuse-15 - opensuse-tumbleweed - - debian-11 - - debian-12 - - arch - gentoo - gentoo-systemd - name: git-3007 @@ -192,8 +63,6 @@ suites: excludes: - opensuse-15 - opensuse-tumbleweed - - debian-11 - - arch - gentoo - gentoo-systemd - name: git-3007x @@ -203,8 +72,6 @@ suites: excludes: - opensuse-15 - opensuse-tumbleweed - - debian-11 - - arch - gentoo - gentoo-systemd - name: stable-3006 @@ -214,7 +81,6 @@ suites: excludes: - opensuse-15 - opensuse-tumbleweed - - arch - name: stable-3006-8 provisioner: salt_version: 3006.8 @@ -222,7 +88,6 @@ suites: excludes: - opensuse-15 - opensuse-tumbleweed - - arch - name: stable-3007 provisioner: salt_version: 3007 @@ -230,7 +95,6 @@ suites: excludes: - opensuse-15 - opensuse-tumbleweed - - arch - name: stable-3007-1 provisioner: salt_version: 3007.1 @@ -238,7 +102,6 @@ suites: excludes: - opensuse-15 - opensuse-tumbleweed - - arch - name: git-master provisioner: salt_version: master @@ -279,7 +142,6 @@ suites: salt_version: 3008.0rc1 salt_bootstrap_options: -R staging.repo.saltproject.io -MP onedir_rc %s excludes: - - arch - gentoo - opensuse-15 - opensuse-tumbleweed From 7c350c4f48ef613d8655bd4ec556828059e507c7 Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 4 Nov 2024 14:29:27 -0700 Subject: [PATCH 036/173] Don't start windows service in tests --- .github/workflows/test-windows.yml | 29 ++++++++++++++--------------- bootstrap-salt.ps1 | 19 +++++++------------ 2 files changed, 21 insertions(+), 27 deletions(-) diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index ecc332c51..aa7f315e2 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -48,24 +48,23 @@ jobs: - name: Install Pytest run: | - pip install -U pytest + pip install -r tests\requirements.txt - - name: Bootstrap Salt - run: | - . ./bootstrap-salt.ps1 -RunService $false - - - name: Set Environment Variables + - name: Get Version run: | - $env:Path = "$env:Path;C:\Program Files\Salt Project\Salt" - $env:Instance = ${{ matrix.instance }} - Write-Output "Path:" - Write-Output $env:Path - Write-Output "Environment Variables" - Get-ChildItem Env: + # We need to get the version here and make it an environment variable + # It is used to install via bootstrap and in the test + # The version is in the instance name + $instance = "${{ matrix.instance }}" + $version = $instance -split "-",2 + if ( $version.Count -gt 1 ) { + $version = $version[1].Replace("-", ".") + } + Write-Output "SaltVersion=$version" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - - name: Add Salt to Path + - name: Bootstrap Salt run: | - $env:Path = "$env:Path;C:\Program Files\Salt Project\Salt" + . .\bootstrap-salt.ps1 -RunService $false -Version $env:SaltVersion - name: Test Bootstrap run: | @@ -82,4 +81,4 @@ jobs: uses: actions/upload-artifact@v4 with: name: exitstatus-${{ github.job }}-${{ matrix.instance }}-${{ inputs.distro-slug }} - path: exitstatus/ + path: exitstatus/ \ No newline at end of file diff --git a/bootstrap-salt.ps1 b/bootstrap-salt.ps1 index a20b67078..1859abc91 100644 --- a/bootstrap-salt.ps1 +++ b/bootstrap-salt.ps1 @@ -18,7 +18,7 @@ Specifies a particular version of the installer. .EXAMPLE - ./bootstrap-salt.ps1 -RunService false + ./bootstrap-salt.ps1 -RunService $false Specifies the salt-minion service to stop and be set to manual. Useful for testing locally from the command line with the --local switch @@ -28,7 +28,7 @@ installer values of host name for the minion id and "salt" for the master. .EXAMPLE - ./bootstrap-salt.ps1 -Minion minion-box -Master master-box -Version 3006.7 -RunService false + ./bootstrap-salt.ps1 -Minion minion-box -Master master-box -Version 3006.7 -RunService $false Specifies all the optional parameters in no particular order. .NOTES @@ -56,12 +56,11 @@ param( [String]$Version = "latest", [Parameter(Mandatory=$false, ValueFromPipeline=$True)] - [ValidateSet("true","false")] [Alias("s")] - # Boolean flag to start or stop the minion service. True will start the - # minion service. False will stop the minion service and set it to "manual". + # Boolean flag to start or stop the minion service. $true will start the + # minion service. $false will stop the minion service and set it to "manual". # The installer starts it by default. - [String]$RunService = "true", + [Bool]$RunService = $true, [Parameter(Mandatory=$false, ValueFromPipeline=$True)] [Alias("m")] @@ -331,16 +330,12 @@ Write-Verbose "apiurl: $ApiUrl" Write-Verbose "ConfDir: $ConfDir" Write-Verbose "RootDir: $RootDir" -if ($RunService.ToLower() -eq "true") { +if ($RunService) { Write-Verbose "Windows service will be set to run" [bool]$RunService = $True -} elseif ($RunService.ToLower() -eq "false") { +} else { Write-Verbose "Windows service will be stopped and set to manual" [bool]$RunService = $False -} else { - # Param passed in wasn't clear so defaulting to true. - Write-Verbose "Windows service defaulting to run automatically" - [bool]$RunService = $True } #=============================================================================== From f1a5e97125c3b557f404806730cacfc3055db3d0 Mon Sep 17 00:00:00 2001 From: twangboy Date: Mon, 4 Nov 2024 14:39:59 -0700 Subject: [PATCH 037/173] Get version from instance name --- .github/workflows/test-windows.yml | 12 +++ .gitignore | 6 -- Gemfile | 11 -- Vagrantfile | 119 ---------------------- kitchen.yml | 158 ----------------------------- tests/conftest.py | 13 +-- tests/integration/__init__.py | 1 - 7 files changed, 13 insertions(+), 307 deletions(-) delete mode 100644 Gemfile delete mode 100644 Vagrantfile delete mode 100644 kitchen.yml delete mode 100644 tests/integration/__init__.py diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index aa7f315e2..7a11264d8 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -62,6 +62,18 @@ jobs: } Write-Output "SaltVersion=$version" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + - name: Get Version + run: | + # We need to get the version here and make it an environment variable + # It is used to install via bootstrap and in the test + # The version is in the instance name + $instance = "${{ matrix.instance }}" + $version = $instance -split "-",2 + if ( $version.Count -gt 1 ) { + $version = $version[1].Replace("-", ".") + } + Write-Output "SaltVersion=$version" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append + - name: Bootstrap Salt run: | . .\bootstrap-salt.ps1 -RunService $false -Version $env:SaltVersion diff --git a/.gitignore b/.gitignore index f1af41c03..f21787dbe 100644 --- a/.gitignore +++ b/.gitignore @@ -8,9 +8,3 @@ venv # Pycharm .idea - -# test-kitchen -.kitchen.local.yml -kitchen.local.yml -.kitchen/ -.bundle/ diff --git a/Gemfile b/Gemfile deleted file mode 100644 index a70a9da93..000000000 --- a/Gemfile +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -source "https://rubygems.org" - -gem 'test-kitchen', '>= 3.2.2' -gem 'kitchen-salt', '>= 0.7.2' -gem 'kitchen-docker', :git => 'https://github.com/test-kitchen/kitchen-docker.git', :branch => 'main' - -group :vagrant do - gem 'kitchen-vagrant' -end diff --git a/Vagrantfile b/Vagrantfile deleted file mode 100644 index 4a8a57f57..000000000 --- a/Vagrantfile +++ /dev/null @@ -1,119 +0,0 @@ -# -*- mode: ruby -*- -# vi: set ft=ruby : - -# Vagrantfile API/syntax version. Don't touch unless you know what you're doing! -VAGRANTFILE_API_VERSION = "2" - -Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| - # All Vagrant configuration is done here. The most common configuration - # options are documented and commented below. For a complete reference, - # please see the online documentation at vagrantup.com. - - # Every Vagrant virtual environment requires a box to build off of. - config.vm.box = "ubuntu/focal64" - - # The url from where the 'config.vm.box' box will be fetched if it - # doesn't already exist on the user's system. - # config.vm.box_url = "http://domain.com/path/to/above.box" - - # Create a forwarded port mapping which allows access to a specific port - # within the machine from a port on the host machine. In the example below, - # accessing "localhost:8080" will access port 80 on the guest machine. - # config.vm.network :forwarded_port, guest: 80, host: 8080 - - # Create a private network, which allows host-only access to the machine - # using a specific IP. - # config.vm.network :private_network, ip: "192.168.33.10" - - # Create a public network, which generally matched to bridged network. - # Bridged networks make the machine appear as another physical device on - # your network. - # config.vm.network :public_network - - # If true, then any SSH connections made will enable agent forwarding. - # Default value: false - # config.ssh.forward_agent = true - - # Share an additional folder to the guest VM. The first argument is - # the path on the host to the actual folder. The second argument is - # the path on the guest to mount the folder. And the optional third - # argument is a set of non-required options. - config.vm.synced_folder ".", "/salt_bootstrap" - - config.vm.provision "shell", path: "bootstrap-salt.sh" - # Provider-specific configuration so you can fine-tune various - # backing providers for Vagrant. These expose provider-specific options. - # Example for VirtualBox: - # - # config.vm.provider :virtualbox do |vb| - # # Don't boot with headless mode - # vb.gui = true - # - # # Use VBoxManage to customize the VM. For example to change memory: - # vb.customize ["modifyvm", :id, "--memory", "1024"] - # end - # - # View the documentation for the provider you're using for more - # information on available options. - - # Enable provisioning with Puppet stand alone. Puppet manifests - # are contained in a directory path relative to this Vagrantfile. - # You will need to create the manifests directory and a manifest in - # the file precise64.pp in the manifests_path directory. - # - # An example Puppet manifest to provision the message of the day: - # - # # group { "puppet": - # # ensure => "present", - # # } - # # - # # File { owner => 0, group => 0, mode => 0644 } - # # - # # file { '/etc/motd': - # # content => "Welcome to your Vagrant-built virtual machine! - # # Managed by Puppet.\n" - # # } - # - # config.vm.provision :puppet do |puppet| - # puppet.manifests_path = "manifests" - # puppet.manifest_file = "site.pp" - # end - - # Enable provisioning with chef solo, specifying a cookbooks path, roles - # path, and data_bags path (all relative to this Vagrantfile), and adding - # some recipes and/or roles. - # - # config.vm.provision :chef_solo do |chef| - # chef.cookbooks_path = "../my-recipes/cookbooks" - # chef.roles_path = "../my-recipes/roles" - # chef.data_bags_path = "../my-recipes/data_bags" - # chef.add_recipe "mysql" - # chef.add_role "web" - # - # # You may also specify custom JSON attributes: - # chef.json = { :mysql_password => "foo" } - # end - - # Enable provisioning with chef server, specifying the chef server URL, - # and the path to the validation key (relative to this Vagrantfile). - # - # The Opscode Platform uses HTTPS. Substitute your organization for - # ORGNAME in the URL and validation key. - # - # If you have your own Chef Server, use the appropriate URL, which may be - # HTTP instead of HTTPS depending on your configuration. Also change the - # validation key to validation.pem. - # - # config.vm.provision :chef_client do |chef| - # chef.chef_server_url = "https://api.opscode.com/organizations/ORGNAME" - # chef.validation_key_path = "ORGNAME-validator.pem" - # end - # - # If you're using the Opscode platform, your validator client is - # ORGNAME-validator, replacing ORGNAME with your organization name. - # - # If you have your own Chef Server, the default validation client name is - # chef-validator, unless you changed the configuration. - # - # chef.validation_client_name = "ORGNAME-validator" -end diff --git a/kitchen.yml b/kitchen.yml deleted file mode 100644 index a08927578..000000000 --- a/kitchen.yml +++ /dev/null @@ -1,158 +0,0 @@ ---- -driver: - name: docker - use_sudo: false - hostname: salt - privileged: true - username: root - cap_add: - - sys_admin - disable_upstart: false - use_internal_docker_network: false - run_command: /usr/lib/systemd/systemd - -provisioner: - name: salt_solo - salt_install: bootstrap - salt_bootstrap_url: bootstrap-salt.sh - salt_bootstrap_options: -MPfq git %s - install_after_init_environment: true - log_level: info - sudo: false - require_chef: false - formula: tests - run_salt_call: false - init_environment: | - echo 'auto_accept: true' > /tmp/auto-accept-keys.conf - mkdir -p /etc/salt/master.d - mv /tmp/auto-accept-keys.conf /etc/salt/master.d/auto-accept-keys.conf - sh -c 't=$(shuf -i 1-15 -n 1); echo Sleeping $t seconds; sleep $t' - - -platforms: - - name: rockylinux-9 - - name: ubuntu-24.04 - driver: - run_command: /lib/systemd/systemd - provision_command: - - echo "PubkeyAcceptedAlgorithms +ssh-rsa" | tee -a /etc/ssh/sshd_config - -suites: - - name: git-3006 - provisioner: - salt_version: v3006 - salt_bootstrap_options: -x python3 -MPfq git %s - excludes: - - opensuse-15 - - opensuse-tumbleweed - - gentoo - - gentoo-systemd - - name: git-3006x - provisioner: - salt_version: 3006.x - salt_bootstrap_options: -x python3 -MPfq git %s - excludes: - - opensuse-15 - - opensuse-tumbleweed - - gentoo - - gentoo-systemd - - name: git-3007 - provisioner: - salt_version: v3007 - salt_bootstrap_options: -x python3 -MPfq git %s - excludes: - - opensuse-15 - - opensuse-tumbleweed - - gentoo - - gentoo-systemd - - name: git-3007x - provisioner: - salt_version: 3007.x - salt_bootstrap_options: -x python3 -MPfq git %s - excludes: - - opensuse-15 - - opensuse-tumbleweed - - gentoo - - gentoo-systemd - - name: stable-3006 - provisioner: - salt_version: 3006 - salt_bootstrap_options: -x python3 -MP stable %s - excludes: - - opensuse-15 - - opensuse-tumbleweed - - name: stable-3006-8 - provisioner: - salt_version: 3006.8 - salt_bootstrap_options: -x python3 -MP stable %s - excludes: - - opensuse-15 - - opensuse-tumbleweed - - name: stable-3007 - provisioner: - salt_version: 3007 - salt_bootstrap_options: -x python3 -MP stable %s - excludes: - - opensuse-15 - - opensuse-tumbleweed - - name: stable-3007-1 - provisioner: - salt_version: 3007.1 - salt_bootstrap_options: -x python3 -MP stable %s - excludes: - - opensuse-15 - - opensuse-tumbleweed - - name: git-master - provisioner: - salt_version: master - salt_bootstrap_options: -x python3 -MPfq -D git %s - - - name: latest - provisioner: - salt_version: latest - salt_bootstrap_options: -MP stable %s - - - name: default - provisioner: - salt_version: latest - salt_bootstrap_options: -MP - - - name: onedir-nightly - provisioner: - salt_version: nightly - salt_bootstrap_options: -MP onedir %s - - - name: onedir-latest - provisioner: - salt_version: latest - salt_bootstrap_options: -MP onedir %s - - - name: onedir-3006 - provisioner: - salt_version: 3006 - salt_bootstrap_options: -MP onedir %s - - - name: onedir-3007 - provisioner: - salt_version: 3007 - salt_bootstrap_options: -MP onedir %s - - - name: onedir-rc-3008-0rc1 - provisioner: - salt_version: 3008.0rc1 - salt_bootstrap_options: -R staging.repo.saltproject.io -MP onedir_rc %s - excludes: - - gentoo - - opensuse-15 - - opensuse-tumbleweed - - ubuntu-2004 - - ubuntu-2204 - - - name: quickstart - provisioner: - salt_bootstrap_options: -Q - -verifier: - name: shell - remote_exec: false - command: pytest --cache-clear -v -s -ra --log-cli-level=info tests/integration/ diff --git a/tests/conftest.py b/tests/conftest.py index 74ded3b2d..b0ca0af75 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,18 +13,7 @@ def target_python_version(): @pytest.fixture(scope="session") def target_salt_version(): - bootstrap_types = ("git", "stable", "onedir", "onedir_rc") - - # filter out any bootstrap types and then join - target_salt = ".".join( - [ - item - for item in os.environ.get("KITCHEN_SUITE", "").split("-") - if item not in bootstrap_types - ] - ) - - # target_salt = os.environ["KITCHEN_SUITE"].split("-", 1)[-1].replace("-", ".") + target_salt = os.environ.get("SaltVersion", "") if target_salt.startswith("v"): target_salt = target_salt[1:] diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py deleted file mode 100644 index 40a96afc6..000000000 --- a/tests/integration/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# -*- coding: utf-8 -*- From 8686ca3a39afcec33cf9a2dc861309d990a39fd9 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 5 Nov 2024 09:11:03 -0700 Subject: [PATCH 038/173] Parse versions from artifactory --- bootstrap-salt.ps1 | 35 ++++++++++++++++++++++------------- tests/conftest.py | 25 ++++++++++++++++++++++--- 2 files changed, 44 insertions(+), 16 deletions(-) diff --git a/bootstrap-salt.ps1 b/bootstrap-salt.ps1 index 1859abc91..1223fba66 100644 --- a/bootstrap-salt.ps1 +++ b/bootstrap-salt.ps1 @@ -401,32 +401,41 @@ $psobj = $response.ToString() | ConvertFrom-Json $filtered = $psobj.children | Where-Object -Property folder -EQ $true # Get each uri and add it to the list of versions -$versions = [System.Collections.ArrayList]@() +$available_versions = [System.Collections.ArrayList]@() $filtered | Select-Object -Property uri | ForEach-Object { - $versions.Add($_.uri.Trim("/")) | Out-Null + $available_versions.Add($_.uri.Trim("/")) | Out-Null } -# last one in the list is the latest -$latest = $versions | Select-Object -Last 1 - -Write-Verbose "Available versions:" -$versions | ForEach-Object { - Write-Verbose $_ +# Create a versions table, similar to repo.json +# This will have the latest version available, the latest version available for +# each major version, and every version available. This makes the version +# lookup logic easier. You can view the contents of the versions table by +# passing the -Verbose command +$latest = $available_versions | Select-Object -Last 1 +$versions_table = [ordered]@{"latest"=$latest} + +$available_versions | ForEach-Object { + $versions_table[$(Get-MajorVersion $_)] = $_ + $versions_table[$_.ToLower()] = $_.ToLower() } -Write-Verbose "Latest version: $latest" -if ( $Version -EQ "latest") { - $Version = $latest +Write-Verbose "Available versions:" +$available_versions | ForEach-Object { + Write-Verbose "- $_" } +Write-Verbose "Versions Table:" +$versions_table | Sort-Object Name | Out-String | Write-Verbose #=============================================================================== # Validate passed version #=============================================================================== -if ( $versions -notcontains $Version ) { +if ( $versions_table -notcontains $Version.ToLower() ) { Write-Host "Version $Version is not available" -ForegroundColor Red Write-Host "Available versions are:" -ForegroundColor Yellow - $versions | ForEach-Object { Write-Host $_ -ForegroundColor Yellow } + $available_versions | ForEach-Object { Write-Host "- $_" -ForegroundColor Yellow } exit 1 +} else { + $Version = $versions_table[$Version.ToLower()] } #=============================================================================== diff --git a/tests/conftest.py b/tests/conftest.py index b0ca0af75..154b257c0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,9 +1,12 @@ -import logging +import json import os import pytest +import requests -log = logging.getLogger(__name__) +API_URL = ( + "https://packages.broadcom.com/artifactory/api/storage/saltproject-generic/windows" +) @pytest.fixture(scope="session") @@ -13,10 +16,26 @@ def target_python_version(): @pytest.fixture(scope="session") def target_salt_version(): + target_salt = os.environ.get("SaltVersion", "") + html_response = requests.get(API_URL) + content = json.loads(html_response.text) + folders = content["children"] + versions = {} + for folder in folders: + if folder["folder"]: + version = folder["uri"].strip("/") + versions[version] = version + # We're trying to get the latest major version and latest overall + maj_version = version.split(".")[0] + versions[maj_version] = version + versions["latest"] = version + if target_salt.startswith("v"): target_salt = target_salt[1:] + if target_salt not in versions: + pytest.skip(f"Invalid testing version: {target_salt}") if target_salt in ("default", "latest", "master", "nightly"): pytest.skip("Don't have a specific salt version to test against") - return target_salt + return versions[target_salt] From d018ef7ebb0c7a3c16c47b747fbfdef855045bf6 Mon Sep 17 00:00:00 2001 From: twangboy Date: Tue, 5 Nov 2024 09:37:56 -0700 Subject: [PATCH 039/173] Lookup version correctly --- bootstrap-salt.ps1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bootstrap-salt.ps1 b/bootstrap-salt.ps1 index 1223fba66..3faaabd32 100644 --- a/bootstrap-salt.ps1 +++ b/bootstrap-salt.ps1 @@ -429,13 +429,13 @@ $versions_table | Sort-Object Name | Out-String | Write-Verbose #=============================================================================== # Validate passed version #=============================================================================== -if ( $versions_table -notcontains $Version.ToLower() ) { +if ( $versions_table.Contains($Version.ToLower()) ) { + $Version = $versions_table[$Version.ToLower()] +} else { Write-Host "Version $Version is not available" -ForegroundColor Red Write-Host "Available versions are:" -ForegroundColor Yellow $available_versions | ForEach-Object { Write-Host "- $_" -ForegroundColor Yellow } exit 1 -} else { - $Version = $versions_table[$Version.ToLower()] } #=============================================================================== From 1156b28cb2d64a0123ba99b488c6d406c387425e Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 6 Nov 2024 14:58:41 -0700 Subject: [PATCH 040/173] Add back RepoUrl for air-gap scenarios --- bootstrap-salt.ps1 | 139 ++++++++++++++++++++++++++------------------- 1 file changed, 80 insertions(+), 59 deletions(-) diff --git a/bootstrap-salt.ps1 b/bootstrap-salt.ps1 index 3faaabd32..781bda545 100644 --- a/bootstrap-salt.ps1 +++ b/bootstrap-salt.ps1 @@ -73,6 +73,15 @@ param( #Name or IP of the master server. Installer defaults to "salt". [String]$Master = "not-specified", + [Parameter(Mandatory=$false, ValueFromPipeline=$True)] + [Alias("r")] + # URL to the windows packages. Will look for a file named repo.json at the + # root of the URL. This file is used to determine the name and location of + # the installer in the repo. If repo.json is not found, it will look for the + # file under the minor directory. + # Default is "https://packages.broadcom.com/artifactory/saltproject-generic/windows/" + [String]$RepoUrl = "https://packages.broadcom.com/artifactory/saltproject-generic/windows/", + [Parameter(Mandatory=$false, ValueFromPipeline=$True)] [Alias("c")] # Vagrant only @@ -291,7 +300,7 @@ if (!(Get-IsAdministrator)) { } #=============================================================================== -# Change RepoUrl for older versions +# Check for older versions #=============================================================================== $majorVersion = Get-MajorVersion -Version $Version if ($majorVersion -lt "3006") { @@ -306,6 +315,7 @@ if ($majorVersion -lt "3006") { $ConfDir = "$RootDir\conf" $PkiDir = "$ConfDir\pki\minion" $RootDir = "$env:ProgramData\Salt Project\Salt" +$DfltUrl = "https://packages.broadcom.com/artifactory/saltproject-generic/windows/" $ApiUrl = "https://packages.broadcom.com/artifactory/api/storage/saltproject-generic/windows" # Check for existing installation where RootDir is stored in the registry @@ -326,6 +336,7 @@ Write-Verbose "version: $Version" Write-Verbose "runservice: $RunService" Write-Verbose "master: $Master" Write-Verbose "minion: $Minion" +Write-Verbose "repourl: $RepoUrl" Write-Verbose "apiurl: $ApiUrl" Write-Verbose "ConfDir: $ConfDir" Write-Verbose "RootDir: $RootDir" @@ -392,69 +403,79 @@ if ([IntPtr]::Size -eq 4) { #=============================================================================== # Getting version information from the repo #=============================================================================== -Write-Verbose "Getting version information from Artifactory" -$response = Invoke-WebRequest $ApiUrl -UseBasicParsing -# Convert the output to a powershell object -$psobj = $response.ToString() | ConvertFrom-Json - -# Filter the object for folders -$filtered = $psobj.children | Where-Object -Property folder -EQ $true - -# Get each uri and add it to the list of versions -$available_versions = [System.Collections.ArrayList]@() -$filtered | Select-Object -Property uri | ForEach-Object { - $available_versions.Add($_.uri.Trim("/")) | Out-Null -} +if ( $RepoUrl -eq $DfltUrl ) { + Write-Verbose "Getting version information from Artifactory" + $response = Invoke-WebRequest $ApiUrl -UseBasicParsing + # Convert the output to a powershell object + $psobj = $response.ToString() | ConvertFrom-Json -# Create a versions table, similar to repo.json -# This will have the latest version available, the latest version available for -# each major version, and every version available. This makes the version -# lookup logic easier. You can view the contents of the versions table by -# passing the -Verbose command -$latest = $available_versions | Select-Object -Last 1 -$versions_table = [ordered]@{"latest"=$latest} - -$available_versions | ForEach-Object { - $versions_table[$(Get-MajorVersion $_)] = $_ - $versions_table[$_.ToLower()] = $_.ToLower() -} + # Filter the object for folders + $filtered = $psobj.children | Where-Object -Property folder -EQ $true -Write-Verbose "Available versions:" -$available_versions | ForEach-Object { - Write-Verbose "- $_" -} -Write-Verbose "Versions Table:" -$versions_table | Sort-Object Name | Out-String | Write-Verbose + # Get each uri and add it to the list of versions + $available_versions = [System.Collections.ArrayList]@() + $filtered | Select-Object -Property uri | ForEach-Object { + $available_versions.Add($_.uri.Trim("/")) | Out-Null + } -#=============================================================================== -# Validate passed version -#=============================================================================== -if ( $versions_table.Contains($Version.ToLower()) ) { - $Version = $versions_table[$Version.ToLower()] -} else { - Write-Host "Version $Version is not available" -ForegroundColor Red - Write-Host "Available versions are:" -ForegroundColor Yellow - $available_versions | ForEach-Object { Write-Host "- $_" -ForegroundColor Yellow } - exit 1 -} + # Create a versions table, similar to repo.json + # This will have the latest version available, the latest version available for + # each major version, and every version available. This makes the version + # lookup logic easier. You can view the contents of the versions table by + # passing the -Verbose command + $latest = $available_versions | Select-Object -Last 1 + $versions_table = [ordered]@{"latest"=$latest} + + $available_versions | ForEach-Object { + $versions_table[$(Get-MajorVersion $_)] = $_ + $versions_table[$_.ToLower()] = $_.ToLower() + } -#=============================================================================== -# Get file name to download -#=============================================================================== -$saltFileName = "Salt-Minion-$Version-Py3-$arch-Setup.exe" -$response = Invoke-WebRequest "$ApiUrl/$Version/$saltFileName" -UseBasicParsing -$psobj = $response.ToString() | ConvertFrom-Json -$saltFileUrl = $psobj.downloadUri -$saltSha256 = $psobj.checksums.sha256 - -if ( $saltFileName -and $saltVersion -and $saltSha256) { - Write-Verbose "Found Name, Version, and Sha" + Write-Verbose "Available versions:" + $available_versions | ForEach-Object { + Write-Verbose "- $_" + } + Write-Verbose "Versions Table:" + $versions_table | Sort-Object Name | Out-String | Write-Verbose + + #=============================================================================== + # Validate passed version + #=============================================================================== + if ( $versions_table.Contains($Version.ToLower()) ) { + $Version = $versions_table[$Version.ToLower()] + } else { + Write-Host "Version $Version is not available" -ForegroundColor Red + Write-Host "Available versions are:" -ForegroundColor Yellow + $available_versions | ForEach-Object { Write-Host "- $_" -ForegroundColor Yellow } + exit 1 + } + + #=============================================================================== + # Get file url and sha256 + #=============================================================================== + $saltFileName = "Salt-Minion-$Version-Py3-$arch-Setup.exe" + $response = Invoke-WebRequest "$ApiUrl/$Version/$saltFileName" -UseBasicParsing + $psobj = $response.ToString() | ConvertFrom-Json + $saltFileUrl = $psobj.downloadUri + $saltSha256 = $psobj.checksums.sha256 + + if ( $saltFileName -and $saltVersion -and $saltSha256) { + Write-Verbose "Found Name, Version, and Sha" + } else { + # We will guess the name of the installer + Write-Verbose "Failed to get Name, Version, and Sha from Artifactory API" + Write-Verbose "We'll try to find the file in standard paths" + $saltFileName = "Salt-Minion-$Version-Py3-$arch-Setup.exe" + $saltVersion = $Version + } } else { - # We will guess the name of the installer - Write-Verbose "Failed to get Name, Version, and Sha from Artifactory API" - Write-Verbose "We'll try to find the file in standard paths" + # If we're using a custom RepoUrl, we're going to assum that the binary is + # in the reoot of the RepoUrl/Version. We will not check the sha on custom + # repos $saltFileName = "Salt-Minion-$Version-Py3-$arch-Setup.exe" + $saltFileUrl = "$RepoUrl/$Version/$saltFileName" $saltVersion = $Version + $saltSha256 = "" } #=============================================================================== @@ -478,8 +499,8 @@ Write-Verbose "" Write-Verbose "Salt File URL: $saltFileUrl" Write-Verbose "Local File: $localFile" -$webclient = New-Object System.Net.WebClient -$webclient.DownloadFile($saltFileUrl, $localFile) +if ( Test-Path -Path $localFile ) {Remove-Item -Path $localFile -Force} +Invoke-WebRequest -Uri $saltFileUrl -OutFile $localFile if ( Test-Path -Path $localFile ) { Write-Host "Success" -ForegroundColor Green From 94b61ec940e35cb2680a90abef53b76789d6ac01 Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 7 Nov 2024 10:45:20 -0700 Subject: [PATCH 041/173] Add back other OS's for testing --- .github/workflows/ci.yml | 273 ++++++++++++++++++++++++ .github/workflows/templates/generate.py | 172 ++++++++++++++- tests/requirements.txt | 2 +- 3 files changed, 443 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 39f042d7e..eebfa5191 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -114,6 +114,37 @@ jobs: + macos-12: + name: macOS 12 + if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' + uses: ./.github/workflows/test-macos.yml + needs: + - lint + - generate-actions-workflow + with: + distro-slug: macos-12 + display-name: macOS 12 + timeout: 20 + runs-on: macos-12 + instances: '["stable-3006", "stable-3006-8", "stable-3007", "stable-3007-1", "latest"]' + + + macos-13: + name: macOS 13 + if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' + uses: ./.github/workflows/test-macos.yml + needs: + - lint + - generate-actions-workflow + with: + distro-slug: macos-13 + display-name: macOS 13 + timeout: 20 + runs-on: macos-13 + instances: '["stable-3006", "stable-3006-8", "stable-3007", "stable-3007-1", "latest"]' + + + windows-2022: name: Windows 2022 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' @@ -130,6 +161,202 @@ jobs: + almalinux-8: + name: AlmaLinux 8 + if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' + uses: ./.github/workflows/test-linux.yml + needs: + - lint + - generate-actions-workflow + with: + distro-slug: almalinux-8 + display-name: AlmaLinux 8 + timeout: 20 + instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' + + + almalinux-9: + name: AlmaLinux 9 + if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' + uses: ./.github/workflows/test-linux.yml + needs: + - lint + - generate-actions-workflow + with: + distro-slug: almalinux-9 + display-name: AlmaLinux 9 + timeout: 20 + instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' + + + amazon-2: + name: Amazon 2 + if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' + uses: ./.github/workflows/test-linux.yml + needs: + - lint + - generate-actions-workflow + with: + distro-slug: amazon-2 + display-name: Amazon 2 + timeout: 20 + instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' + + + centos-stream9: + name: CentOS Stream 9 + if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' + uses: ./.github/workflows/test-linux.yml + needs: + - lint + - generate-actions-workflow + with: + distro-slug: centos-stream9 + display-name: CentOS Stream 9 + timeout: 20 + instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' + + + debian-11: + name: Debian 11 + if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' + uses: ./.github/workflows/test-linux.yml + needs: + - lint + - generate-actions-workflow + with: + distro-slug: debian-11 + display-name: Debian 11 + timeout: 20 + instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' + + + debian-12: + name: Debian 12 + if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' + uses: ./.github/workflows/test-linux.yml + needs: + - lint + - generate-actions-workflow + with: + distro-slug: debian-12 + display-name: Debian 12 + timeout: 20 + instances: '["stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' + + + fedora-39: + name: Fedora 39 + if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' + uses: ./.github/workflows/test-linux.yml + needs: + - lint + - generate-actions-workflow + with: + distro-slug: fedora-39 + display-name: Fedora 39 + timeout: 20 + instances: '["stable-3006", "onedir-3006", "stable-3006-8", "latest", "default"]' + + + fedora-40: + name: Fedora 40 + if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' + uses: ./.github/workflows/test-linux.yml + needs: + - lint + - generate-actions-workflow + with: + distro-slug: fedora-40 + display-name: Fedora 40 + timeout: 20 + instances: '["stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' + + + opensuse-15: + name: Opensuse 15 + if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' + uses: ./.github/workflows/test-linux.yml + needs: + - lint + - generate-actions-workflow + with: + distro-slug: opensuse-15 + display-name: Opensuse 15 + timeout: 20 + instances: '["latest", "default"]' + + + oraclelinux-8: + name: Oracle Linux 8 + if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' + uses: ./.github/workflows/test-linux.yml + needs: + - lint + - generate-actions-workflow + with: + distro-slug: oraclelinux-8 + display-name: Oracle Linux 8 + timeout: 20 + instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' + + + oraclelinux-9: + name: Oracle Linux 9 + if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' + uses: ./.github/workflows/test-linux.yml + needs: + - lint + - generate-actions-workflow + with: + distro-slug: oraclelinux-9 + display-name: Oracle Linux 9 + timeout: 20 + instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' + + + photon-4: + name: Photon OS 4 + if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' + uses: ./.github/workflows/test-linux.yml + needs: + - lint + - generate-actions-workflow + with: + distro-slug: photon-4 + display-name: Photon OS 4 + timeout: 20 + instances: '["stable-3006", "onedir-3006", "stable-3006-8", "latest", "default"]' + + + photon-5: + name: Photon OS 5 + if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' + uses: ./.github/workflows/test-linux.yml + needs: + - lint + - generate-actions-workflow + with: + distro-slug: photon-5 + display-name: Photon OS 5 + timeout: 20 + instances: '["stable-3006", "onedir-3006", "stable-3006-8", "latest", "default"]' + + + rockylinux-8: + name: Rocky Linux 8 + if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' + uses: ./.github/workflows/test-linux.yml + needs: + - lint + - generate-actions-workflow + with: + distro-slug: rockylinux-8 + display-name: Rocky Linux 8 + timeout: 20 + instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' + + rockylinux-9: name: Rocky Linux 9 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' @@ -144,6 +371,34 @@ jobs: instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' + ubuntu-2004: + name: Ubuntu 20.04 + if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' + uses: ./.github/workflows/test-linux.yml + needs: + - lint + - generate-actions-workflow + with: + distro-slug: ubuntu-2004 + display-name: Ubuntu 20.04 + timeout: 20 + instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' + + + ubuntu-2204: + name: Ubuntu 22.04 + if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' + uses: ./.github/workflows/test-linux.yml + needs: + - lint + - generate-actions-workflow + with: + distro-slug: ubuntu-2204 + display-name: Ubuntu 22.04 + timeout: 20 + instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' + + ubuntu-2404: name: Ubuntu 24.04 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' @@ -166,8 +421,26 @@ jobs: needs: - lint - generate-actions-workflow + - macos-12 + - macos-13 - windows-2022 + - almalinux-8 + - almalinux-9 + - amazon-2 + - centos-stream9 + - debian-11 + - debian-12 + - fedora-39 + - fedora-40 + - opensuse-15 + - oraclelinux-8 + - oraclelinux-9 + - photon-4 + - photon-5 + - rockylinux-8 - rockylinux-9 + - ubuntu-2004 + - ubuntu-2204 - ubuntu-2404 if: always() steps: diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index 3d842dd4f..537d1da5e 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -7,45 +7,143 @@ os.chdir(os.path.abspath(os.path.dirname(__file__))) LINUX_DISTROS = [ + "almalinux-8", + "almalinux-9", + "amazon-2", + "centos-stream9", + "debian-11", + "debian-12", + "fedora-39", + "fedora-40", + "opensuse-15", + "oraclelinux-8", + "oraclelinux-9", + "photon-4", + "photon-5", + "rockylinux-8", "rockylinux-9", + "ubuntu-2004", + "ubuntu-2204", "ubuntu-2404", ] + WINDOWS = [ "windows-2022", ] +OSX = [ + "macos-12", + "macos-13", +] + STABLE_DISTROS = [ + "almalinux-8", + "almalinux-9", + "amazon-2", + "centos-stream9", + "debian-11", + "debian-12", + "fedora-39", + "fedora-40", + "opensuse-15", + "oraclelinux-8", + "oraclelinux-9", + "photon-4", + "photon-5", + "rockylinux-8", "rockylinux-9", + "ubuntu-2004", + "ubuntu-2204", "ubuntu-2404", ] ONEDIR_DISTROS = [ + "almalinux-8", + "almalinux-9", + "amazon-2", + "centos-stream9", + "debian-11", + "debian-12", + "fedora-39", + "fedora-40", + "oraclelinux-8", + "oraclelinux-9", + "photon-4", + "photon-5", + "rockylinux-8", "rockylinux-9", + "ubuntu-2004", + "ubuntu-2204", "ubuntu-2404", ] ONEDIR_RC_DISTROS = [ + "almalinux-9", + "amazon-2", + "centos-stream9", + "debian-12", + "oraclelinux-9", + "photon-4", + "photon-5", "rockylinux-9", "ubuntu-2404", ] BLACKLIST_3006 = [ + "debian-12", + "fedora-40", + "opensuse-15", "ubuntu-2404", ] -BLACKLIST_3007 = [] +BLACKLIST_3007 = [ + "fedora-39", + "opensuse-15", + "photon-4", + "photon-5", +] BLACKLIST_GIT_3006 = [ + "almalinux-9", + "amazon-2", + "centos-stream9", + "debian-11", + "debian-12", + "fedora-40", + "opensuse-15", + "oraclelinux-9", + "photon-4", + "photon-5", "rockylinux-9", + "ubuntu-2004", + "ubuntu-2204", "ubuntu-2404", ] BLACKLIST_GIT_3007 = [ + "almalinux-9", + "amazon-2", + "centos-stream9", + "debian-11", + "debian-12", + "fedora-39", + "fedora-40", + "opensuse-15", + "oraclelinux-9", + "photon-4", + "photon-5", "rockylinux-9", + "ubuntu-2004", + "ubuntu-2204", "ubuntu-2404", ] -BLACKLIST_GIT_MASTER = [] +BLACKLIST_GIT_MASTER = [ + "amazon-2", + "fedora-39", + "photon-4", + "photon-5", +] SALT_VERSIONS = [ "3006", @@ -100,13 +198,37 @@ # SetuptoolsDeprecationWarning: setup.py install is deprecated. # Use build and pip and other standards-based tools. # -GIT_DISTRO_BLACKLIST = [] +GIT_DISTRO_BLACKLIST = [ + "almalinux-8", + "fedora-39", + "opensuse-15", + "oraclelinux-8", + "rockylinux-8", +] LATEST_PKG_BLACKLIST = [] DISTRO_DISPLAY_NAMES = { + "almalinux-8": "AlmaLinux 8", + "almalinux-9": "AlmaLinux 9", + "amazon-2": "Amazon 2", + "centos-stream9": "CentOS Stream 9", + "debian-11": "Debian 11", + "debian-12": "Debian 12", + "fedora-39": "Fedora 39", + "fedora-40": "Fedora 40", + "opensuse-15": "Opensuse 15", + "oraclelinux-8": "Oracle Linux 8", + "oraclelinux-9": "Oracle Linux 9", + "photon-4": "Photon OS 4", + "photon-5": "Photon OS 5", + "rockylinux-8": "Rocky Linux 8", "rockylinux-9": "Rocky Linux 9", + "ubuntu-2004": "Ubuntu 20.04", + "ubuntu-2204": "Ubuntu 22.04", "ubuntu-2404": "Ubuntu 24.04", + "macos-12": "macOS 12", + "macos-13": "macOS 13", "windows-2022": "Windows 2022", } @@ -133,6 +255,50 @@ def generate_test_jobs(): test_jobs = "" needs = ["lint", "generate-actions-workflow"] + test_jobs += "\n" + for distro in OSX: + test_jobs += "\n" + runs_on = distro + runs_on = f"\n runs-on: {runs_on}" + ifcheck = "\n if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true'" + uses = "./.github/workflows/test-macos.yml" + instances = [] + timeout_minutes = ( + TIMEOUT_OVERRIDES[distro] + if distro in TIMEOUT_OVERRIDES + else TIMEOUT_DEFAULT + ) + + for salt_version in SALT_VERSIONS: + if salt_version == "latest": + instances.append(salt_version) + continue + + for bootstrap_type in ["stable"]: + if bootstrap_type == "stable": + if salt_version in MAC_STABLE_VERSION_BLACKLIST: + continue + + test_target = f"{bootstrap_type}-{salt_version}" + instances.append(test_target) + + for bootstrap_type in ["default"]: + if distro not in STABLE_DISTROS: + continue + instances.append(bootstrap_type) + + if instances: + needs.append(distro) + test_jobs += TEMPLATE.format( + distro=distro, + runs_on=runs_on, + uses=uses, + ifcheck=ifcheck, + instances=json.dumps(instances), + display_name=DISTRO_DISPLAY_NAMES[distro], + timeout_minutes=timeout_minutes, + ) + test_jobs += "\n" for distro in WINDOWS: test_jobs += "\n" diff --git a/tests/requirements.txt b/tests/requirements.txt index 16d3b7789..547de5c5b 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,2 +1,2 @@ pytest -requests \ No newline at end of file +requests From 6414095b6096d42f473e4a42398584f6d707b3d1 Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 7 Nov 2024 10:47:05 -0700 Subject: [PATCH 042/173] Remove duplicate job step from rebase --- .github/workflows/test-windows.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index 7a11264d8..aa7f315e2 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -62,18 +62,6 @@ jobs: } Write-Output "SaltVersion=$version" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - - name: Get Version - run: | - # We need to get the version here and make it an environment variable - # It is used to install via bootstrap and in the test - # The version is in the instance name - $instance = "${{ matrix.instance }}" - $version = $instance -split "-",2 - if ( $version.Count -gt 1 ) { - $version = $version[1].Replace("-", ".") - } - Write-Output "SaltVersion=$version" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append - - name: Bootstrap Salt run: | . .\bootstrap-salt.ps1 -RunService $false -Version $env:SaltVersion From f56ce5c866bbac75d24fab462815a964d00923a9 Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 7 Nov 2024 10:47:48 -0700 Subject: [PATCH 043/173] Add missing newline --- .github/workflows/test-windows.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index aa7f315e2..e4d4dde34 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -81,4 +81,4 @@ jobs: uses: actions/upload-artifact@v4 with: name: exitstatus-${{ github.job }}-${{ matrix.instance }}-${{ inputs.distro-slug }} - path: exitstatus/ \ No newline at end of file + path: exitstatus/ From 3316de34f12ff2fe7a1e4f83cb6557aed9be1e68 Mon Sep 17 00:00:00 2001 From: twangboy Date: Thu, 7 Nov 2024 10:51:47 -0700 Subject: [PATCH 044/173] Update docs for RepoUrl --- bootstrap-salt.ps1 | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/bootstrap-salt.ps1 b/bootstrap-salt.ps1 index 781bda545..fc862ab7f 100644 --- a/bootstrap-salt.ps1 +++ b/bootstrap-salt.ps1 @@ -75,10 +75,9 @@ param( [Parameter(Mandatory=$false, ValueFromPipeline=$True)] [Alias("r")] - # URL to the windows packages. Will look for a file named repo.json at the - # root of the URL. This file is used to determine the name and location of - # the installer in the repo. If repo.json is not found, it will look for the - # file under the minor directory. + # URL to the windows packages. Will look for the installer at the root of + # the URL/Version. Place a folder for each version of Salt in this directory + # and place the installer binary for each version in its folder. # Default is "https://packages.broadcom.com/artifactory/saltproject-generic/windows/" [String]$RepoUrl = "https://packages.broadcom.com/artifactory/saltproject-generic/windows/", From 2a4e6a3552a74d56e595ed66c76c21c27a55b25a Mon Sep 17 00:00:00 2001 From: David Murphy Date: Thu, 7 Nov 2024 11:31:30 -0700 Subject: [PATCH 045/173] Disabling signing of commits on release updates --- .github/workflows/release.yml | 263 +++++++++++++++++----------------- bootstrap-salt.sh | 4 +- 2 files changed, 135 insertions(+), 132 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index aeb362715..9744f33b0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -74,35 +74,35 @@ jobs: python3 -m pip install -r requirements/release.txt pre-commit install --install-hooks - - name: Setup GnuPG - run: | - sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg - GNUPGHOME="$(mktemp -d -p /run/gpg)" - echo "GNUPGHOME=${GNUPGHOME}" >> "$GITHUB_ENV" - cat < "${GNUPGHOME}/gpg.conf" - batch - no-tty - pinentry-mode loopback - EOF - - - name: Get Secrets - id: get-secrets - env: - SECRETS_KEY: ${{ secrets.SECRETS_KEY }} - run: | - SECRETS_KEY_FILE=$(mktemp /tmp/output.XXXXXXXXXX) - echo "$SECRETS_KEY" > "$SECRETS_KEY_FILE" - aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ - --query SecretString --output text | jq .default_key -r | base64 -d \ - | gpg --passphrase-file "${SECRETS_KEY_FILE}" -d - \ - | gpg --import - - sync - aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ - --query SecretString --output text| jq .default_passphrase -r | base64 -d \ - | gpg --passphrase-file "${SECRETS_KEY_FILE}" -o "${GNUPGHOME}/passphrase" -d - - sync - rm "$SECRETS_KEY_FILE" - echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf" + ## - name: Setup GnuPG + ## run: | + ## sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg + ## GNUPGHOME="$(mktemp -d -p /run/gpg)" + ## echo "GNUPGHOME=${GNUPGHOME}" >> "$GITHUB_ENV" + ## cat < "${GNUPGHOME}/gpg.conf" + ## batch + ## no-tty + ## pinentry-mode loopback + ## EOF + + ## - name: Get Secrets + ## id: get-secrets + ## env: + ## SECRETS_KEY: ${{ secrets.SECRETS_KEY }} + ## run: | + ## SECRETS_KEY_FILE=$(mktemp /tmp/output.XXXXXXXXXX) + ## echo "$SECRETS_KEY" > "$SECRETS_KEY_FILE" + ## aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ + ## --query SecretString --output text | jq .default_key -r | base64 -d \ + ## | gpg --passphrase-file "${SECRETS_KEY_FILE}" -d - \ + ## | gpg --import - + ## sync + ## aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ + ## --query SecretString --output text| jq .default_passphrase -r | base64 -d \ + ## | gpg --passphrase-file "${SECRETS_KEY_FILE}" -o "${GNUPGHOME}/passphrase" -d - + ## sync + ## rm "$SECRETS_KEY_FILE" + ## echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf" - name: Configure Git shell: bash @@ -110,8 +110,9 @@ jobs: git config --global --add safe.directory "$(pwd)" git config --global user.name "Salt Project Packaging" git config --global user.email saltproject-packaging@vmware.com - git config --global user.signingkey 64CBBC8173D76B3F - git config --global commit.gpgsign true + git config --global commit.gpgsign false + ## git config --global user.signingkey 64CBBC8173D76B3F + ## git config --global commit.gpgsign true - name: Update Repository id: update-repo @@ -166,35 +167,35 @@ jobs: ssh-key: ${{ secrets.SALT_BOOTSTRAP_RELEASE_KEY }} fetch-depth: 0 - - name: Setup GnuPG - run: | - sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg - GNUPGHOME="$(mktemp -d -p /run/gpg)" - echo "GNUPGHOME=${GNUPGHOME}" >> "$GITHUB_ENV" - cat < "${GNUPGHOME}/gpg.conf" - batch - no-tty - pinentry-mode loopback - EOF - - - name: Get Secrets - id: get-secrets - env: - SECRETS_KEY: ${{ secrets.SECRETS_KEY }} - run: | - SECRETS_KEY_FILE=$(mktemp /tmp/output.XXXXXXXXXX) - echo "$SECRETS_KEY" > "$SECRETS_KEY_FILE" - aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ - --query SecretString --output text | jq .default_key -r | base64 -d \ - | gpg --passphrase-file "${SECRETS_KEY_FILE}" -d - \ - | gpg --import - - sync - aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ - --query SecretString --output text| jq .default_passphrase -r | base64 -d \ - | gpg --passphrase-file "${SECRETS_KEY_FILE}" -o "${GNUPGHOME}/passphrase" -d - - sync - rm "$SECRETS_KEY_FILE" - echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf" + ## - name: Setup GnuPG + ## run: | + ## sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg + ## GNUPGHOME="$(mktemp -d -p /run/gpg)" + ## echo "GNUPGHOME=${GNUPGHOME}" >> "$GITHUB_ENV" + ## cat < "${GNUPGHOME}/gpg.conf" + ## batch + ## no-tty + ## pinentry-mode loopback + ## EOF + + ## - name: Get Secrets + ## id: get-secrets + ## env: + ## SECRETS_KEY: ${{ secrets.SECRETS_KEY }} + ## run: | + ## SECRETS_KEY_FILE=$(mktemp /tmp/output.XXXXXXXXXX) + ## echo "$SECRETS_KEY" > "$SECRETS_KEY_FILE" + ## aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ + ## --query SecretString --output text | jq .default_key -r | base64 -d \ + ## | gpg --passphrase-file "${SECRETS_KEY_FILE}" -d - \ + ## | gpg --import - + ## sync + ## aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ + ## --query SecretString --output text| jq .default_passphrase -r | base64 -d \ + ## | gpg --passphrase-file "${SECRETS_KEY_FILE}" -o "${GNUPGHOME}/passphrase" -d - + ## sync + ## rm "$SECRETS_KEY_FILE" + ## echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf" - name: Configure Git shell: bash @@ -202,8 +203,9 @@ jobs: git config --global --add safe.directory "$(pwd)" git config --global user.name "Salt Project Packaging" git config --global user.email saltproject-packaging@vmware.com - git config --global user.signingkey 64CBBC8173D76B3F - git config --global commit.gpgsign true + git config --global commit.gpgsign false + ## git config --global user.signingkey 64CBBC8173D76B3F + ## git config --global commit.gpgsign true - name: Download Release Details uses: actions/download-artifact@v4 @@ -317,43 +319,43 @@ jobs: SPB_ENVIRONMENT=$(curl -sS -f -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/spb:environment) echo "SPB_ENVIRONMENT=$SPB_ENVIRONMENT" >> "$GITHUB_ENV" - - name: Setup GnuPG - run: | - sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg - GNUPGHOME="$(mktemp -d -p /run/gpg)" - echo "GNUPGHOME=${GNUPGHOME}" >> "$GITHUB_ENV" - cat < "${GNUPGHOME}/gpg.conf" - batch - no-tty - pinentry-mode loopback - EOF - - - name: Get Secrets - id: get-secrets - env: - SECRETS_KEY: ${{ secrets.SECRETS_KEY }} - run: | - SECRETS_KEY_FILE=$(mktemp /tmp/output.XXXXXXXXXX) - echo "$SECRETS_KEY" > "$SECRETS_KEY_FILE" - aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ - --query SecretString --output text | jq .default_key -r | base64 -d \ - | gpg --passphrase-file "${SECRETS_KEY_FILE}" -d - \ - | gpg --import - - sync - aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ - --query SecretString --output text| jq .default_passphrase -r | base64 -d \ - | gpg --passphrase-file "${SECRETS_KEY_FILE}" -o "${GNUPGHOME}/passphrase" -d - - sync - rm "$SECRETS_KEY_FILE" - echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf" - - - name: Install Requirements - run: | - python3 -m pip install -r requirements/release.txt - - - name: Upload Stable Release to S3 - run: | - tools release s3-publish --key-id 64CBBC8173D76B3F stable + ## - name: Setup GnuPG + ## run: | + ## sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg + ## GNUPGHOME="$(mktemp -d -p /run/gpg)" + ## echo "GNUPGHOME=${GNUPGHOME}" >> "$GITHUB_ENV" + ## cat < "${GNUPGHOME}/gpg.conf" + ## batch + ## no-tty + ## pinentry-mode loopback + ## EOF + + ## - name: Get Secrets + ## id: get-secrets + ## env: + ## SECRETS_KEY: ${{ secrets.SECRETS_KEY }} + ## run: | + ## SECRETS_KEY_FILE=$(mktemp /tmp/output.XXXXXXXXXX) + ## echo "$SECRETS_KEY" > "$SECRETS_KEY_FILE" + ## aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ + ## --query SecretString --output text | jq .default_key -r | base64 -d \ + ## | gpg --passphrase-file "${SECRETS_KEY_FILE}" -d - \ + ## | gpg --import - + ## sync + ## aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ + ## --query SecretString --output text| jq .default_passphrase -r | base64 -d \ + ## | gpg --passphrase-file "${SECRETS_KEY_FILE}" -o "${GNUPGHOME}/passphrase" -d - + ## sync + ## rm "$SECRETS_KEY_FILE" + ## echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf" + + ## - name: Install Requirements + ## run: | + ## python3 -m pip install -r requirements/release.txt + + ## - name: Upload Stable Release to S3 + ## run: | + ## tools release s3-publish --key-id 64CBBC8173D76B3F stable update-develop-checksums: name: Update Release Checksums on Develop @@ -386,35 +388,35 @@ jobs: repository: ${{ github.repository }} ssh-key: ${{ secrets.SALT_BOOTSTRAP_RELEASE_KEY }} - - name: Setup GnuPG - run: | - sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg - GNUPGHOME="$(mktemp -d -p /run/gpg)" - echo "GNUPGHOME=${GNUPGHOME}" >> "$GITHUB_ENV" - cat < "${GNUPGHOME}/gpg.conf" - batch - no-tty - pinentry-mode loopback - EOF - - - name: Get Secrets - id: get-secrets - env: - SECRETS_KEY: ${{ secrets.SECRETS_KEY }} - run: | - SECRETS_KEY_FILE=$(mktemp /tmp/output.XXXXXXXXXX) - echo "$SECRETS_KEY" > "$SECRETS_KEY_FILE" - aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ - --query SecretString --output text | jq .default_key -r | base64 -d \ - | gpg --passphrase-file "${SECRETS_KEY_FILE}" -d - \ - | gpg --import - - sync - aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ - --query SecretString --output text| jq .default_passphrase -r | base64 -d \ - | gpg --passphrase-file "${SECRETS_KEY_FILE}" -o "${GNUPGHOME}/passphrase" -d - - sync - rm "$SECRETS_KEY_FILE" - echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf" + ## - name: Setup GnuPG + ## run: | + ## sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg + ## GNUPGHOME="$(mktemp -d -p /run/gpg)" + ## echo "GNUPGHOME=${GNUPGHOME}" >> "$GITHUB_ENV" + ## cat < "${GNUPGHOME}/gpg.conf" + ## batch + ## no-tty + ## pinentry-mode loopback + ## EOF + + ## - name: Get Secrets + ## id: get-secrets + ## env: + ## SECRETS_KEY: ${{ secrets.SECRETS_KEY }} + ## run: | + ## SECRETS_KEY_FILE=$(mktemp /tmp/output.XXXXXXXXXX) + ## echo "$SECRETS_KEY" > "$SECRETS_KEY_FILE" + ## aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ + ## --query SecretString --output text | jq .default_key -r | base64 -d \ + ## | gpg --passphrase-file "${SECRETS_KEY_FILE}" -d - \ + ## | gpg --import - + ## sync + ## aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ + ## --query SecretString --output text| jq .default_passphrase -r | base64 -d \ + ## | gpg --passphrase-file "${SECRETS_KEY_FILE}" -o "${GNUPGHOME}/passphrase" -d - + ## sync + ## rm "$SECRETS_KEY_FILE" + ## echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf" - name: Configure Git shell: bash @@ -422,8 +424,9 @@ jobs: git config --global --add safe.directory "$(pwd)" git config --global user.name "Salt Project Packaging" git config --global user.email saltproject-packaging@vmware.com - git config --global user.signingkey 64CBBC8173D76B3F - git config --global commit.gpgsign true + git config --global commit.gpgsign false + ## git config --global user.signingkey 64CBBC8173D76B3F + ## git config --global commit.gpgsign true - name: Update Latest Release on README run: | diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index d52214fd4..faf8a0375 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -657,7 +657,7 @@ elif [ "$ITYPE" = "stable" ]; then ITYPE="onedir" shift else - echo "Unknown stable version: $1 (valid: 3006, 3007, latest)" + echo "Unknown stable version: $1 (valid: 3006, 3007, latest), versions older than 3006 are not available" exit 1 fi fi @@ -676,7 +676,7 @@ elif [ "$ITYPE" = "onedir" ]; then STABLE_REV="$1" shift else - echo "Unknown onedir version: $1 (valid: 3006, 3007, latest.)" + echo "Unknown onedir version: $1 (valid: 3006, 3007, latest), versions older than 3006 are not available" exit 1 fi fi From ee72db1f2a6ca32dc555c5e6bf4f838b85546535 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Thu, 7 Nov 2024 14:13:39 -0700 Subject: [PATCH 046/173] Changed release runners and sheel script date --- .github/workflows/release.yml | 147 ++++++++++++++++++---------------- bootstrap-salt.sh | 2 +- 2 files changed, 77 insertions(+), 72 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9744f33b0..a62d35a4e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -52,9 +52,10 @@ jobs: update-develop: name: Update CHANGELOG.md and bootstrap-salt.sh runs-on: - - self-hosted - - linux - - repo-release + - ubuntu-latest + ## - self-hosted + ## - linux + ## - repo-release permissions: contents: write # To be able to publish the release environment: release @@ -151,9 +152,10 @@ jobs: merge-develop-into-stable: name: Merge develop into stable runs-on: - - self-hosted - - linux - - repo-release + - ubuntu-latest + ## - self-hosted + ## - linux + ## - repo-release needs: - update-develop environment: release @@ -249,8 +251,9 @@ jobs: publish-release: name: Create GitHub Release runs-on: - - self-hosted - - linux + - ubuntu-latest + ## - self-hosted + ## - linux needs: - merge-develop-into-stable environment: release @@ -296,73 +299,75 @@ jobs: name: release-details failOnError: false - update-s3-bucket: - name: Update S3 Bucket - runs-on: - - self-hosted - - linux - - repo-release - needs: - - publish-release - environment: release - - steps: - - uses: actions/checkout@v4 - with: - ref: stable - repository: ${{ github.repository }} - ssh-key: ${{ secrets.SALT_BOOTSTRAP_RELEASE_KEY }} - - - name: Get Salt Project GitHub Actions Bot Environment - run: | - TOKEN=$(curl -sS -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 30") - SPB_ENVIRONMENT=$(curl -sS -f -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/spb:environment) - echo "SPB_ENVIRONMENT=$SPB_ENVIRONMENT" >> "$GITHUB_ENV" - - ## - name: Setup GnuPG - ## run: | - ## sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg - ## GNUPGHOME="$(mktemp -d -p /run/gpg)" - ## echo "GNUPGHOME=${GNUPGHOME}" >> "$GITHUB_ENV" - ## cat < "${GNUPGHOME}/gpg.conf" - ## batch - ## no-tty - ## pinentry-mode loopback - ## EOF - - ## - name: Get Secrets - ## id: get-secrets - ## env: - ## SECRETS_KEY: ${{ secrets.SECRETS_KEY }} - ## run: | - ## SECRETS_KEY_FILE=$(mktemp /tmp/output.XXXXXXXXXX) - ## echo "$SECRETS_KEY" > "$SECRETS_KEY_FILE" - ## aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ - ## --query SecretString --output text | jq .default_key -r | base64 -d \ - ## | gpg --passphrase-file "${SECRETS_KEY_FILE}" -d - \ - ## | gpg --import - - ## sync - ## aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ - ## --query SecretString --output text| jq .default_passphrase -r | base64 -d \ - ## | gpg --passphrase-file "${SECRETS_KEY_FILE}" -o "${GNUPGHOME}/passphrase" -d - - ## sync - ## rm "$SECRETS_KEY_FILE" - ## echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf" - - ## - name: Install Requirements - ## run: | - ## python3 -m pip install -r requirements/release.txt - - ## - name: Upload Stable Release to S3 - ## run: | - ## tools release s3-publish --key-id 64CBBC8173D76B3F stable + ## update-s3-bucket: + ## name: Update S3 Bucket + ## runs-on: + ## - ubuntu-latest + ## ## - self-hosted + ## ## - linux + ## ## - repo-release + ## needs: + ## - publish-release + ## environment: release + + ## steps: + ## - uses: actions/checkout@v4 + ## with: + ## ref: stable + ## repository: ${{ github.repository }} + ## ssh-key: ${{ secrets.SALT_BOOTSTRAP_RELEASE_KEY }} + + ## - name: Get Salt Project GitHub Actions Bot Environment + ## run: | + ## TOKEN=$(curl -sS -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 30") + ## SPB_ENVIRONMENT=$(curl -sS -f -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/spb:environment) + ## echo "SPB_ENVIRONMENT=$SPB_ENVIRONMENT" >> "$GITHUB_ENV" + + ## ## - name: Setup GnuPG + ## ## run: | + ## ## sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg + ## ## GNUPGHOME="$(mktemp -d -p /run/gpg)" + ## ## echo "GNUPGHOME=${GNUPGHOME}" >> "$GITHUB_ENV" + ## ## cat < "${GNUPGHOME}/gpg.conf" + ## ## batch + ## ## no-tty + ## ## pinentry-mode loopback + ## ## EOF + + ## ## - name: Get Secrets + ## ## id: get-secrets + ## ## env: + ## ## SECRETS_KEY: ${{ secrets.SECRETS_KEY }} + ## ## run: | + ## ## SECRETS_KEY_FILE=$(mktemp /tmp/output.XXXXXXXXXX) + ## ## echo "$SECRETS_KEY" > "$SECRETS_KEY_FILE" + ## ## aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ + ## ## --query SecretString --output text | jq .default_key -r | base64 -d \ + ## ## | gpg --passphrase-file "${SECRETS_KEY_FILE}" -d - \ + ## ## | gpg --import - + ## ## sync + ## ## aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ + ## ## --query SecretString --output text| jq .default_passphrase -r | base64 -d \ + ## ## | gpg --passphrase-file "${SECRETS_KEY_FILE}" -o "${GNUPGHOME}/passphrase" -d - + ## ## sync + ## ## rm "$SECRETS_KEY_FILE" + ## ## echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf" + + ## ## - name: Install Requirements + ## ## run: | + ## ## python3 -m pip install -r requirements/release.txt + + ## ## - name: Upload Stable Release to S3 + ## ## run: | + ## ## tools release s3-publish --key-id 64CBBC8173D76B3F stable update-develop-checksums: name: Update Release Checksums on Develop runs-on: - - self-hosted - - linux - - repo-release + - ubuntu-latest + ## - self-hosted + ## - linux + ## - repo-release needs: - publish-release environment: release diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index faf8a0375..d4fa965e8 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -26,7 +26,7 @@ #====================================================================================================================== set -o nounset # Treat unset variables as an error -__ScriptVersion="2024.11.06" +__ScriptVersion="2024.11.07" __ScriptName="bootstrap-salt.sh" __ScriptFullName="$0" From 2fe253a9c08035dd86b7938e6d8084ab868f6a7c Mon Sep 17 00:00:00 2001 From: Salt Project Packaging Date: Thu, 7 Nov 2024 21:21:20 +0000 Subject: [PATCH 047/173] Update develop branch for the v2024.11.07 release --- CHANGELOG.md | 8 ++++++++ bootstrap-salt.ps1 | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 33ee81d93..ccc2ac03a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +# v2024.11.07 + +## What's Changed + +- Changed release runners and shell script date by @dmurphy18 in https://github.com/saltstack/salt-bootstrap/pull/2035 + +**Full Changelog**: https://github.com/saltstack/salt-bootstrap/compare/v2024.11.06...v2024.11.07 + # v2024.09.24 ## What's Changed diff --git a/bootstrap-salt.ps1 b/bootstrap-salt.ps1 index fc862ab7f..85223d371 100644 --- a/bootstrap-salt.ps1 +++ b/bootstrap-salt.ps1 @@ -108,7 +108,7 @@ if ($help) { exit 0 } -$__ScriptVersion = "2024.09.24" +$__ScriptVersion = "2024.11.07" $ScriptName = $myInvocation.MyCommand.Name # We'll check for the Version next, because it also has no requirements From ced8291c040a6b6d64af1eb558a754ca341809a5 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Thu, 7 Nov 2024 14:32:08 -0700 Subject: [PATCH 048/173] Forcing git tag to not sign --- .github/workflows/release.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a62d35a4e..d58e23f83 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -231,7 +231,8 @@ jobs: - name: Tag The ${{ needs.update-develop.outputs.release-version }} Release run: | - git tag -m "Release ${{ needs.update-develop.outputs.release-version }}" -as ${{ needs.update-develop.outputs.release-version }} + cat ~/.gitconfig + git tag --no-sign -m "Release ${{ needs.update-develop.outputs.release-version }}" -as ${{ needs.update-develop.outputs.release-version }} - name: Update bootstrap-salt.sh sha256sum's run: | From 549a47f94b5e828ffb36f5f9e712aafbbc9533ec Mon Sep 17 00:00:00 2001 From: Salt Project Packaging Date: Thu, 7 Nov 2024 21:35:23 +0000 Subject: [PATCH 049/173] Update develop branch for the v2024.11.07 release --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ccc2ac03a..3584ea4c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ ## What's Changed +- Changed release runners and shell script date by @dmurphy18 in https://github.com/saltstack/salt-bootstrap/pull/2035 +- Forcing git tag to not sign by @dmurphy18 in https://github.com/saltstack/salt-bootstrap/pull/2036 + +**Full Changelog**: https://github.com/saltstack/salt-bootstrap/compare/v2024.11.06...v2024.11.07 + +# v2024.11.07 + +## What's Changed + - Changed release runners and shell script date by @dmurphy18 in https://github.com/saltstack/salt-bootstrap/pull/2035 **Full Changelog**: https://github.com/saltstack/salt-bootstrap/compare/v2024.11.06...v2024.11.07 From cda77be3a3b685cc14441b338921f271faed5ff5 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Thu, 7 Nov 2024 14:38:49 -0700 Subject: [PATCH 050/173] Fixing git tag signing issue --- .github/workflows/release.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d58e23f83..935b0f70f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -231,8 +231,7 @@ jobs: - name: Tag The ${{ needs.update-develop.outputs.release-version }} Release run: | - cat ~/.gitconfig - git tag --no-sign -m "Release ${{ needs.update-develop.outputs.release-version }}" -as ${{ needs.update-develop.outputs.release-version }} + git tag --no-sign -m "Release ${{ needs.update-develop.outputs.release-version }}" -a ${{ needs.update-develop.outputs.release-version }} - name: Update bootstrap-salt.sh sha256sum's run: | From bf7943e78c735bd32448f6f3c0669504f0ec680e Mon Sep 17 00:00:00 2001 From: Salt Project Packaging Date: Thu, 7 Nov 2024 21:41:35 +0000 Subject: [PATCH 051/173] Update develop branch for the v2024.11.07 release --- CHANGELOG.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3584ea4c7..725022f71 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,16 @@ ## What's Changed +- Changed release runners and shell script date by @dmurphy18 in https://github.com/saltstack/salt-bootstrap/pull/2035 +- Forcing git tag to not sign by @dmurphy18 in https://github.com/saltstack/salt-bootstrap/pull/2036 +- Fixing git tag signing issue by @dmurphy18 in https://github.com/saltstack/salt-bootstrap/pull/2037 + +**Full Changelog**: https://github.com/saltstack/salt-bootstrap/compare/v2024.11.06...v2024.11.07 + +# v2024.11.07 + +## What's Changed + - Changed release runners and shell script date by @dmurphy18 in https://github.com/saltstack/salt-bootstrap/pull/2035 - Forcing git tag to not sign by @dmurphy18 in https://github.com/saltstack/salt-bootstrap/pull/2036 From b2545a20a78adad7067cf7edb49a58405731becb Mon Sep 17 00:00:00 2001 From: Salt Project Packaging Date: Thu, 7 Nov 2024 21:42:26 +0000 Subject: [PATCH 052/173] Update README.rst with 2024.11.07 release sha256sum --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index aa7609d33..2e896305b 100644 --- a/README.rst +++ b/README.rst @@ -37,6 +37,7 @@ sum** of the downloaded ``bootstrap-salt.sh`` file. The SHA256 sum of the ``bootstrap-salt.sh`` file, per release, is: +- 2024.11.07: ``70a9783649e129985563d1a86cf28b8984499643e62ae1dc47dc008bd204fcbb`` - 2024.09.24: ``88e4e4cad4b115a7b721dd9c21d5ee5df390b5b73b63de48f99399146f43f371`` - 2024.07.23: ``7212b6b497b5c3d2bf15bfe5301625ec7bc1bf3e2949cd47d8e2073614935bf8`` - 2024.07.18: ``92a74e7ff8a9032a7713c2b3955991d66aaca08a4eb9494ce3dd66b5044f6bc3`` From e6e5cd64e9fad70c0f0830e3ef926acfa34f5bdd Mon Sep 17 00:00:00 2001 From: twangboy Date: Fri, 15 Nov 2024 10:43:48 -0700 Subject: [PATCH 053/173] Make universal repo url --- bootstrap-salt.ps1 | 256 ++++++++++++++++++++++++++------------------- 1 file changed, 147 insertions(+), 109 deletions(-) diff --git a/bootstrap-salt.ps1 b/bootstrap-salt.ps1 index 85223d371..350ea84ab 100644 --- a/bootstrap-salt.ps1 +++ b/bootstrap-salt.ps1 @@ -48,7 +48,6 @@ [CmdletBinding()] param( [Parameter(Mandatory=$false, ValueFromPipeline=$True)] - [ValidatePattern('^(\d{4}(\.\d{1,2}){0,2}(\-\d{1})?)|(latest)$', Options=1)] [Alias("v")] # The version of the Salt minion to install. Default is "latest" which will # install the latest version of Salt minion available. Doesn't support @@ -153,34 +152,116 @@ function Get-MajorVersion { return ( $Version -split "\." )[0] } -function Convert-PSObjectToHashtable { - param ( - [Parameter(ValueFromPipeline)] - $InputObject - ) - if ($null -eq $InputObject) { return $null } - - $is_enum = $InputObject -is [System.Collections.IEnumerable] - $not_string = $InputObject -isnot [string] - if ($is_enum -and $not_string) { - $collection = @( - foreach ($object in $InputObject) { - Convert-PSObjectToHashtable $object - } - ) +function Get-AvailableVersions { + # Get available versions from a remote location specified in the Source + # Parameter + Write-Verbose "Getting version information from the repo" + Write-Verbose "base_url: $base_url" + + $available_versions = [System.Collections.ArrayList]@() + + if ( $base_url.StartsWith("http") -or $base_url.StartsWith("ftp") ) { + # We're dealing with HTTP, HTTPS, or FTP + $response = Invoke-WebRequest "$base_url" -UseBasicParsing + try { + $response = Invoke-WebRequest "$base_url" -UseBasicParsing + } catch { + Write-Host "Failed to get version information" -ForegroundColor Red + exit 1 + } - Write-Host -NoEnumerate $collection - } elseif ($InputObject -is [PSObject]) { - $hash = @{} + if ( $response.StatusCode -ne 200 ) { + Write-Host "There was an error getting version information" -ForegroundColor Red + Write-Host "Error: $($response.StatusCode)" -ForegroundColor red + exit 1 + } - foreach ($property in $InputObject.PSObject.Properties) { - $hash[$property.Name] = Convert-PSObjectToHashtable $property.Value + $response.links | ForEach-Object { + if ( $_.href.Length -gt 8) { + Write-Host "The content at this location is unexpected" -ForegroundColor Red + Write-Host "Should be a list of directories where the name is a version of Salt" -ForegroundColor Red + exit 1 + } } - $hash + # Getting available versions from response + Write-Verbose "Getting available versions from response" + $filtered = $response.Links | Where-Object -Property href -NE "../" + $filtered | Select-Object -Property href | ForEach-Object { + $available_versions.Add($_.href.Trim("/")) | Out-Null + } + } elseif ( $base_url.StartsWith("\\") -or $base_url -match "^[A-Za-z]:\\" ) { + # We're dealing with a local directory or SMB source + Get-ChildItem -Path $base_url -Directory | ForEach-Object { + $available_versions.Add($_.Name) | Out-Null + } } else { - $InputObject + Write-Host "Unknown Source Type" -ForegroundColor Red + Write-Host "Must be one of HTTP, HTTPS, FTP, SMB Share, Local Directory" -ForegroundColor Red + exit 1 + } + + Write-Verbose "Available versions:" + $available_versions | ForEach-Object { + Write-Verbose "- $_" + } + + # Get the latest version, should be the last in the list + Write-Verbose "Getting latest available version" + $latest = $available_versions | Select-Object -Last 1 + Write-Verbose "Latest available version: $latest" + + # Create a versions table + # This will have the latest version available, the latest version available + # for each major version, and every version available. This makes the + # version lookup logic easier. The contents of the versions table can be + # found by running -Verbose + Write-Verbose "Populating the versions table" + $versions_table = [ordered]@{"latest"=$latest} + $available_versions | ForEach-Object { + $versions_table[$(Get-MajorVersion $_)] = $_ + $versions_table[$_.ToLower()] = $_.ToLower() + } + + Write-Verbose "Versions Table:" + $versions_table | Sort-Object Name | Out-String | ForEach-Object { + Write-Verbose "$_" + } + + return $versions_table +} + +function Get-HashFromArtifactory { + # This function uses the artifactory API to get the SHA265 Hash for the file + # If Source is NOT artifactory, the sha will not be checked + [CmdletBinding()] + param( + [Parameter(Mandatory=$true)] + [String] $SaltVersion, + + [Parameter(Mandatory=$true)] + [String] $SaltFileName + ) + if ( $api_url ) { + $full_url = "$api_url/$SaltVersion/$SaltFileName" + Write-Verbose "Querying Artifactory API for hash:" + Write-Verbose $full_url + try { + $response = Invoke-RestMethod $full_url -UseBasicParsing + return $response.checksums.sha256 + } catch { + Write-Verbose "Artifactory API Not available or file not" + Write-Verbose "available at specified location" + Write-Verbose "Hash will not be checked" + return "" + } + Write-Verbose "No hash found for this file: $SaltFileName" + Write-Verbose "Hash will not be checked" + return "" } + Write-Verbose "No artifactory API defined" + Write-Verbose "Hash will not be checked" + return "" } function Get-FileHash { @@ -313,10 +394,8 @@ if ($majorVersion -lt "3006") { #=============================================================================== $ConfDir = "$RootDir\conf" $PkiDir = "$ConfDir\pki\minion" -$RootDir = "$env:ProgramData\Salt Project\Salt" -$DfltUrl = "https://packages.broadcom.com/artifactory/saltproject-generic/windows/" -$ApiUrl = "https://packages.broadcom.com/artifactory/api/storage/saltproject-generic/windows" +$RootDir = "$env:ProgramData\Salt Project\Salt" # Check for existing installation where RootDir is stored in the registry $SaltRegKey = "HKLM:\SOFTWARE\Salt Project\Salt" if (Test-Path -Path $SaltRegKey) { @@ -325,6 +404,18 @@ if (Test-Path -Path $SaltRegKey) { } } +# Get repo and api URLs. An artifactory URL will have "artifactory" in it +$domain, $target = $RepoUrl -split "/artifactory/" +if ( $target ) { + # Create $base_url and $api_url + $base_url = "$domain/artifactory/$target" + $api_url = "$domain/artifactory/api/storage/$target" +} else { + # This is a non-artifactory url, there is no api + $base_url = $domain + $api_url = "" +} + #=============================================================================== # Verify Parameters #=============================================================================== @@ -335,8 +426,8 @@ Write-Verbose "version: $Version" Write-Verbose "runservice: $RunService" Write-Verbose "master: $Master" Write-Verbose "minion: $Minion" -Write-Verbose "repourl: $RepoUrl" -Write-Verbose "apiurl: $ApiUrl" +Write-Verbose "repourl: $base_url" +Write-Verbose "apiurl: $api_url" Write-Verbose "ConfDir: $ConfDir" Write-Verbose "RootDir: $RootDir" @@ -393,90 +484,34 @@ if ( $ConfigureOnly ) { #=============================================================================== # Detect architecture #=============================================================================== -if ([IntPtr]::Size -eq 4) { - $arch = "x86" -} else { - $arch = "AMD64" -} +if ([IntPtr]::Size -eq 4) { $arch = "x86" } else { $arch = "AMD64" } #=============================================================================== # Getting version information from the repo #=============================================================================== -if ( $RepoUrl -eq $DfltUrl ) { - Write-Verbose "Getting version information from Artifactory" - $response = Invoke-WebRequest $ApiUrl -UseBasicParsing - # Convert the output to a powershell object - $psobj = $response.ToString() | ConvertFrom-Json - - # Filter the object for folders - $filtered = $psobj.children | Where-Object -Property folder -EQ $true - - # Get each uri and add it to the list of versions - $available_versions = [System.Collections.ArrayList]@() - $filtered | Select-Object -Property uri | ForEach-Object { - $available_versions.Add($_.uri.Trim("/")) | Out-Null - } +$versions = Get-AvailableVersions - # Create a versions table, similar to repo.json - # This will have the latest version available, the latest version available for - # each major version, and every version available. This makes the version - # lookup logic easier. You can view the contents of the versions table by - # passing the -Verbose command - $latest = $available_versions | Select-Object -Last 1 - $versions_table = [ordered]@{"latest"=$latest} - - $available_versions | ForEach-Object { - $versions_table[$(Get-MajorVersion $_)] = $_ - $versions_table[$_.ToLower()] = $_.ToLower() - } - - Write-Verbose "Available versions:" - $available_versions | ForEach-Object { - Write-Verbose "- $_" - } - Write-Verbose "Versions Table:" - $versions_table | Sort-Object Name | Out-String | Write-Verbose - - #=============================================================================== - # Validate passed version - #=============================================================================== - if ( $versions_table.Contains($Version.ToLower()) ) { - $Version = $versions_table[$Version.ToLower()] - } else { - Write-Host "Version $Version is not available" -ForegroundColor Red - Write-Host "Available versions are:" -ForegroundColor Yellow - $available_versions | ForEach-Object { Write-Host "- $_" -ForegroundColor Yellow } - exit 1 - } - - #=============================================================================== - # Get file url and sha256 - #=============================================================================== - $saltFileName = "Salt-Minion-$Version-Py3-$arch-Setup.exe" - $response = Invoke-WebRequest "$ApiUrl/$Version/$saltFileName" -UseBasicParsing - $psobj = $response.ToString() | ConvertFrom-Json - $saltFileUrl = $psobj.downloadUri - $saltSha256 = $psobj.checksums.sha256 - - if ( $saltFileName -and $saltVersion -and $saltSha256) { - Write-Verbose "Found Name, Version, and Sha" - } else { - # We will guess the name of the installer - Write-Verbose "Failed to get Name, Version, and Sha from Artifactory API" - Write-Verbose "We'll try to find the file in standard paths" - $saltFileName = "Salt-Minion-$Version-Py3-$arch-Setup.exe" - $saltVersion = $Version - } +#=============================================================================== +# Validate passed version +#=============================================================================== +Write-Verbose "Looking up version: $Version" +if ( $versions.Contains($Version.ToLower()) ) { + $Version = $versions[$Version.ToLower()] + Write-Verbose "Found version: $Version" } else { - # If we're using a custom RepoUrl, we're going to assum that the binary is - # in the reoot of the RepoUrl/Version. We will not check the sha on custom - # repos - $saltFileName = "Salt-Minion-$Version-Py3-$arch-Setup.exe" - $saltFileUrl = "$RepoUrl/$Version/$saltFileName" - $saltVersion = $Version - $saltSha256 = "" + Write-Host "Version $Version is not available" -ForegroundColor Red + Write-Host "Available versions are:" -ForegroundColor Yellow + $versions + exit 1 } +#=============================================================================== +# Get file url and sha256 +#=============================================================================== +$saltFileName = "Salt-Minion-$Version-Py3-$arch-Setup.exe" +$saltFileUrl = "$base_url/$Version/$saltFileName" +$saltSha256 = Get-HashFromArtifactory -SaltVersion $Version -SaltFileName $saltFileName + #=============================================================================== # Download minion setup file #=============================================================================== @@ -484,7 +519,7 @@ Write-Host "==================================================================== Write-Host " Bootstrapping Salt Minion" -ForegroundColor Green Write-Host " - version: $Version" Write-Host " - file name: $saltFileName" -Write-Host " - file url: $saltFileUrl" +Write-Host " - file url : $saltFileUrl" Write-Host " - file hash: $saltSha256" Write-Host " - master: $Master" Write-Host " - minion id: $Minion" @@ -498,9 +533,11 @@ Write-Verbose "" Write-Verbose "Salt File URL: $saltFileUrl" Write-Verbose "Local File: $localFile" -if ( Test-Path -Path $localFile ) {Remove-Item -Path $localFile -Force} -Invoke-WebRequest -Uri $saltFileUrl -OutFile $localFile +# Remove existing local file +if ( Test-Path -Path $localFile ) { Remove-Item -Path $localFile -Force } +# Download the file +Invoke-WebRequest -Uri $saltFileUrl -OutFile $localFile if ( Test-Path -Path $localFile ) { Write-Host "Success" -ForegroundColor Green } else { @@ -508,6 +545,7 @@ if ( Test-Path -Path $localFile ) { exit 1 } +# Compare the hash if there is a hash to compare if ( $saltSha256 ) { $localSha256 = (Get-FileHash -Path $localFile -Algorithm SHA256).Hash Write-Host "Comparing Hash: " -NoNewline @@ -546,13 +584,13 @@ $process = Start-Process $localFile ` -NoNewWindow -PassThru # Sometimes the installer hangs... we'll wait 5 minutes and then kill it -Write-Verbose "" Write-Verbose "Waiting for installer to finish" $process | Wait-Process -Timeout 300 -ErrorAction SilentlyContinue $process.Refresh() if ( !$process.HasExited ) { - Write-Host "Installer Timeout" -ForegroundColor Yellow + Write-Verbose "Installer Timeout" + Write-Host "" Write-Host "Killing hung installer: " -NoNewline $process | Stop-Process $process.Refresh() From a8b1804f18840d281d468f63cb019a67db552fac Mon Sep 17 00:00:00 2001 From: David Murphy Date: Thu, 7 Nov 2024 15:17:51 -0700 Subject: [PATCH 054/173] [WIP] initial testing for CI/CD clean up --- .github/workflows/ci.yml | 15 -- .github/workflows/nightly.yml | 58 -------- .github/workflows/release.yml | 181 +----------------------- .github/workflows/templates/generate.py | 8 -- .github/workflows/test-linux.yml | 15 +- .github/workflows/test-macos.yml | 7 +- 6 files changed, 15 insertions(+), 269 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index eebfa5191..91e8d2cc1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -245,20 +245,6 @@ jobs: instances: '["stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - fedora-39: - name: Fedora 39 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: fedora-39 - display-name: Fedora 39 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "latest", "default"]' - - fedora-40: name: Fedora 40 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' @@ -430,7 +416,6 @@ jobs: - centos-stream9 - debian-11 - debian-12 - - fedora-39 - fedora-40 - opensuse-15 - oraclelinux-8 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index bfdb47b13..c776a44cc 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -41,61 +41,3 @@ jobs: echo "${MSG}" >> "${GITHUB_STEP_SUMMARY}" echo "requirements-met=true" >> "${GITHUB_OUTPUT}" fi - - update-s3-bucket: - name: Update S3 Bucket - if: ${{ fromJSON(needs.workflow-requirements.outputs.requirements-met) }} - runs-on: - - self-hosted - - linux - - repo-release - needs: - - workflow-requirements - environment: release - - steps: - - uses: actions/checkout@v4 - - - name: Get Salt Project GitHub Actions Bot Environment - run: | - TOKEN=$(curl -sS -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 30") - SPB_ENVIRONMENT=$(curl -sS -f -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/spb:environment) - echo "SPB_ENVIRONMENT=$SPB_ENVIRONMENT" >> "$GITHUB_ENV" - - - name: Setup GnuPG - run: | - sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg - GNUPGHOME="$(mktemp -d -p /run/gpg)" - echo "GNUPGHOME=${GNUPGHOME}" >> "$GITHUB_ENV" - cat < "${GNUPGHOME}/gpg.conf" - batch - no-tty - pinentry-mode loopback - EOF - - - name: Get Secrets - id: get-secrets - env: - SECRETS_KEY: ${{ secrets.SECRETS_KEY }} - run: | - SECRETS_KEY_FILE=$(mktemp /tmp/output.XXXXXXXXXX) - echo "$SECRETS_KEY" > "$SECRETS_KEY_FILE" - aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ - --query SecretString --output text | jq .default_key -r | base64 -d \ - | gpg --passphrase-file "${SECRETS_KEY_FILE}" -d - \ - | gpg --import - - sync - aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ - --query SecretString --output text| jq .default_passphrase -r | base64 -d \ - | gpg --passphrase-file "${SECRETS_KEY_FILE}" -o "${GNUPGHOME}/passphrase" -d - - sync - rm "$SECRETS_KEY_FILE" - echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf" - - - name: Install Requirements - run: | - python3 -m pip install -r requirements/release.txt - - - name: Upload Develop to S3 - run: | - tools release s3-publish --key-id 64CBBC8173D76B3F develop diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 935b0f70f..f31d6c470 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -51,11 +51,7 @@ jobs: update-develop: name: Update CHANGELOG.md and bootstrap-salt.sh - runs-on: - - ubuntu-latest - ## - self-hosted - ## - linux - ## - repo-release + runs-on: ubuntu-latest permissions: contents: write # To be able to publish the release environment: release @@ -75,36 +71,6 @@ jobs: python3 -m pip install -r requirements/release.txt pre-commit install --install-hooks - ## - name: Setup GnuPG - ## run: | - ## sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg - ## GNUPGHOME="$(mktemp -d -p /run/gpg)" - ## echo "GNUPGHOME=${GNUPGHOME}" >> "$GITHUB_ENV" - ## cat < "${GNUPGHOME}/gpg.conf" - ## batch - ## no-tty - ## pinentry-mode loopback - ## EOF - - ## - name: Get Secrets - ## id: get-secrets - ## env: - ## SECRETS_KEY: ${{ secrets.SECRETS_KEY }} - ## run: | - ## SECRETS_KEY_FILE=$(mktemp /tmp/output.XXXXXXXXXX) - ## echo "$SECRETS_KEY" > "$SECRETS_KEY_FILE" - ## aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ - ## --query SecretString --output text | jq .default_key -r | base64 -d \ - ## | gpg --passphrase-file "${SECRETS_KEY_FILE}" -d - \ - ## | gpg --import - - ## sync - ## aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ - ## --query SecretString --output text| jq .default_passphrase -r | base64 -d \ - ## | gpg --passphrase-file "${SECRETS_KEY_FILE}" -o "${GNUPGHOME}/passphrase" -d - - ## sync - ## rm "$SECRETS_KEY_FILE" - ## echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf" - - name: Configure Git shell: bash run: | @@ -112,8 +78,6 @@ jobs: git config --global user.name "Salt Project Packaging" git config --global user.email saltproject-packaging@vmware.com git config --global commit.gpgsign false - ## git config --global user.signingkey 64CBBC8173D76B3F - ## git config --global commit.gpgsign true - name: Update Repository id: update-repo @@ -151,11 +115,7 @@ jobs: merge-develop-into-stable: name: Merge develop into stable - runs-on: - - ubuntu-latest - ## - self-hosted - ## - linux - ## - repo-release + runs-on: ubuntu-latest needs: - update-develop environment: release @@ -169,36 +129,6 @@ jobs: ssh-key: ${{ secrets.SALT_BOOTSTRAP_RELEASE_KEY }} fetch-depth: 0 - ## - name: Setup GnuPG - ## run: | - ## sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg - ## GNUPGHOME="$(mktemp -d -p /run/gpg)" - ## echo "GNUPGHOME=${GNUPGHOME}" >> "$GITHUB_ENV" - ## cat < "${GNUPGHOME}/gpg.conf" - ## batch - ## no-tty - ## pinentry-mode loopback - ## EOF - - ## - name: Get Secrets - ## id: get-secrets - ## env: - ## SECRETS_KEY: ${{ secrets.SECRETS_KEY }} - ## run: | - ## SECRETS_KEY_FILE=$(mktemp /tmp/output.XXXXXXXXXX) - ## echo "$SECRETS_KEY" > "$SECRETS_KEY_FILE" - ## aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ - ## --query SecretString --output text | jq .default_key -r | base64 -d \ - ## | gpg --passphrase-file "${SECRETS_KEY_FILE}" -d - \ - ## | gpg --import - - ## sync - ## aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ - ## --query SecretString --output text| jq .default_passphrase -r | base64 -d \ - ## | gpg --passphrase-file "${SECRETS_KEY_FILE}" -o "${GNUPGHOME}/passphrase" -d - - ## sync - ## rm "$SECRETS_KEY_FILE" - ## echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf" - - name: Configure Git shell: bash run: | @@ -206,8 +136,6 @@ jobs: git config --global user.name "Salt Project Packaging" git config --global user.email saltproject-packaging@vmware.com git config --global commit.gpgsign false - ## git config --global user.signingkey 64CBBC8173D76B3F - ## git config --global commit.gpgsign true - name: Download Release Details uses: actions/download-artifact@v4 @@ -250,10 +178,7 @@ jobs: publish-release: name: Create GitHub Release - runs-on: - - ubuntu-latest - ## - self-hosted - ## - linux + runs-on: ubuntu-latest needs: - merge-develop-into-stable environment: release @@ -299,75 +224,9 @@ jobs: name: release-details failOnError: false - ## update-s3-bucket: - ## name: Update S3 Bucket - ## runs-on: - ## - ubuntu-latest - ## ## - self-hosted - ## ## - linux - ## ## - repo-release - ## needs: - ## - publish-release - ## environment: release - - ## steps: - ## - uses: actions/checkout@v4 - ## with: - ## ref: stable - ## repository: ${{ github.repository }} - ## ssh-key: ${{ secrets.SALT_BOOTSTRAP_RELEASE_KEY }} - - ## - name: Get Salt Project GitHub Actions Bot Environment - ## run: | - ## TOKEN=$(curl -sS -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 30") - ## SPB_ENVIRONMENT=$(curl -sS -f -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/spb:environment) - ## echo "SPB_ENVIRONMENT=$SPB_ENVIRONMENT" >> "$GITHUB_ENV" - - ## ## - name: Setup GnuPG - ## ## run: | - ## ## sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg - ## ## GNUPGHOME="$(mktemp -d -p /run/gpg)" - ## ## echo "GNUPGHOME=${GNUPGHOME}" >> "$GITHUB_ENV" - ## ## cat < "${GNUPGHOME}/gpg.conf" - ## ## batch - ## ## no-tty - ## ## pinentry-mode loopback - ## ## EOF - - ## ## - name: Get Secrets - ## ## id: get-secrets - ## ## env: - ## ## SECRETS_KEY: ${{ secrets.SECRETS_KEY }} - ## ## run: | - ## ## SECRETS_KEY_FILE=$(mktemp /tmp/output.XXXXXXXXXX) - ## ## echo "$SECRETS_KEY" > "$SECRETS_KEY_FILE" - ## ## aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ - ## ## --query SecretString --output text | jq .default_key -r | base64 -d \ - ## ## | gpg --passphrase-file "${SECRETS_KEY_FILE}" -d - \ - ## ## | gpg --import - - ## ## sync - ## ## aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ - ## ## --query SecretString --output text| jq .default_passphrase -r | base64 -d \ - ## ## | gpg --passphrase-file "${SECRETS_KEY_FILE}" -o "${GNUPGHOME}/passphrase" -d - - ## ## sync - ## ## rm "$SECRETS_KEY_FILE" - ## ## echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf" - - ## ## - name: Install Requirements - ## ## run: | - ## ## python3 -m pip install -r requirements/release.txt - - ## ## - name: Upload Stable Release to S3 - ## ## run: | - ## ## tools release s3-publish --key-id 64CBBC8173D76B3F stable - update-develop-checksums: name: Update Release Checksums on Develop - runs-on: - - ubuntu-latest - ## - self-hosted - ## - linux - ## - repo-release + runs-on: ubuntu-latest needs: - publish-release environment: release @@ -393,36 +252,6 @@ jobs: repository: ${{ github.repository }} ssh-key: ${{ secrets.SALT_BOOTSTRAP_RELEASE_KEY }} - ## - name: Setup GnuPG - ## run: | - ## sudo install -d -m 0700 -o "$(id -u)" -g "$(id -g)" /run/gpg - ## GNUPGHOME="$(mktemp -d -p /run/gpg)" - ## echo "GNUPGHOME=${GNUPGHOME}" >> "$GITHUB_ENV" - ## cat < "${GNUPGHOME}/gpg.conf" - ## batch - ## no-tty - ## pinentry-mode loopback - ## EOF - - ## - name: Get Secrets - ## id: get-secrets - ## env: - ## SECRETS_KEY: ${{ secrets.SECRETS_KEY }} - ## run: | - ## SECRETS_KEY_FILE=$(mktemp /tmp/output.XXXXXXXXXX) - ## echo "$SECRETS_KEY" > "$SECRETS_KEY_FILE" - ## aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ - ## --query SecretString --output text | jq .default_key -r | base64 -d \ - ## | gpg --passphrase-file "${SECRETS_KEY_FILE}" -d - \ - ## | gpg --import - - ## sync - ## aws --region us-west-2 secretsmanager get-secret-value --secret-id /cmbu-saltstack/signing/repo-signing-keys-sha256-2023 \ - ## --query SecretString --output text| jq .default_passphrase -r | base64 -d \ - ## | gpg --passphrase-file "${SECRETS_KEY_FILE}" -o "${GNUPGHOME}/passphrase" -d - - ## sync - ## rm "$SECRETS_KEY_FILE" - ## echo "passphrase-file ${GNUPGHOME}/passphrase" >> "${GNUPGHOME}/gpg.conf" - - name: Configure Git shell: bash run: | @@ -430,8 +259,6 @@ jobs: git config --global user.name "Salt Project Packaging" git config --global user.email saltproject-packaging@vmware.com git config --global commit.gpgsign false - ## git config --global user.signingkey 64CBBC8173D76B3F - ## git config --global commit.gpgsign true - name: Update Latest Release on README run: | diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index 537d1da5e..f1a5f39e4 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -13,7 +13,6 @@ "centos-stream9", "debian-11", "debian-12", - "fedora-39", "fedora-40", "opensuse-15", "oraclelinux-8", @@ -43,7 +42,6 @@ "centos-stream9", "debian-11", "debian-12", - "fedora-39", "fedora-40", "opensuse-15", "oraclelinux-8", @@ -64,7 +62,6 @@ "centos-stream9", "debian-11", "debian-12", - "fedora-39", "fedora-40", "oraclelinux-8", "oraclelinux-9", @@ -97,7 +94,6 @@ ] BLACKLIST_3007 = [ - "fedora-39", "opensuse-15", "photon-4", "photon-5", @@ -126,7 +122,6 @@ "centos-stream9", "debian-11", "debian-12", - "fedora-39", "fedora-40", "opensuse-15", "oraclelinux-9", @@ -140,7 +135,6 @@ BLACKLIST_GIT_MASTER = [ "amazon-2", - "fedora-39", "photon-4", "photon-5", ] @@ -200,7 +194,6 @@ # GIT_DISTRO_BLACKLIST = [ "almalinux-8", - "fedora-39", "opensuse-15", "oraclelinux-8", "rockylinux-8", @@ -215,7 +208,6 @@ "centos-stream9": "CentOS Stream 9", "debian-11": "Debian 11", "debian-12": "Debian 12", - "fedora-39": "Fedora 39", "fedora-40": "Fedora 40", "opensuse-15": "Opensuse 15", "oraclelinux-8": "Oracle Linux 8", diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index d31ef758f..cfba976e8 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -45,14 +45,15 @@ jobs: run: | pip install -U pytest - ## TBD Disabled until CI/CD is functional - ## - name: Bootstrap Salt - ## run: | - ## sh -x ./bootstrap-salt.sh + - name: Bootstrap Salt + run: | + echo "matrix instance ,${{ matrix.instance }}," + echo "inputs.distro-slug ,${{ inputs.distro-slug }}," + sh -x ./bootstrap-salt.sh - ## - name: Test Bootstrap - ## run: | - ## pytest --cache-clear -v -s -ra --log-cli-level=debug tests/integration/ + - name: Test Bootstrap + run: | + pytest --cache-clear -v -s -ra --log-cli-level=debug tests/integration/ - name: Set Exit Status if: always() diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index 504f8b3e7..f4ad541f0 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -50,10 +50,9 @@ jobs: run: | sudo sh -x ./bootstrap-salt.sh - ## TBD Disabled until CI/CD is functional - ## - name: Test Bootstrap - ## run: | - ## pytest --cache-clear -v -s -ra --log-cli-level=debug tests/integration/ + - name: Test Bootstrap + run: | + pytest --cache-clear -v -s -ra --log-cli-level=debug tests/integration/ - name: Set Exit Status if: always() From 62040848a83ed4247f68b8451e7965771421844d Mon Sep 17 00:00:00 2001 From: David Murphy Date: Thu, 7 Nov 2024 15:48:33 -0700 Subject: [PATCH 055/173] Debugging linux tests --- .github/workflows/test-linux.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index cfba976e8..59177d75e 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -26,10 +26,9 @@ on: jobs: Test: name: ${{ matrix.instance }} - ## runs-on: ubuntu-20.04 runs-on: ubuntu-latest container: - image: ghcr.io/saltstack/salt-ci-containers/packaging:ubuntu-22.04 + image: ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.distro-slug }} timeout-minutes: ${{ inputs.timeout }} strategy: @@ -48,8 +47,9 @@ jobs: - name: Bootstrap Salt run: | echo "matrix instance ,${{ matrix.instance }}," - echo "inputs.distro-slug ,${{ inputs.distro-slug }}," - sh -x ./bootstrap-salt.sh + $bt_parms = echo "${{ matrix.instance }}" | sed 's/-/ /' + echo "bt parms ,$bt_parms," + sh -x ./bootstrap-salt.sh $bt_parms - name: Test Bootstrap run: | From 78b4b24b42e7944ffb345e72f259a3cebe27dd50 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Thu, 7 Nov 2024 16:22:14 -0700 Subject: [PATCH 056/173] More linux debugging --- .github/workflows/test-linux.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 59177d75e..675d3ff20 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -28,7 +28,9 @@ jobs: name: ${{ matrix.instance }} runs-on: ubuntu-latest container: - image: ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.distro-slug }} + $trgt_version = echo "${{ inputs.display-name }}" | awk -F ' ' '{print $2}' + $trgt_slug = echo "${{ inputs.distro-slug}}" | awk -F '-' '{print $1}' + image: ghcr.io/saltstack/salt-ci-containers/testing:$trgt_slug-$trgt_version timeout-minutes: ${{ inputs.timeout }} strategy: From 9dc22a81782d409b7a576bb3839e3302514eb079 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Thu, 7 Nov 2024 16:44:04 -0700 Subject: [PATCH 057/173] fixed yaml --- .github/workflows/test-linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 675d3ff20..8a2eda0f6 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -30,7 +30,7 @@ jobs: container: $trgt_version = echo "${{ inputs.display-name }}" | awk -F ' ' '{print $2}' $trgt_slug = echo "${{ inputs.distro-slug}}" | awk -F '-' '{print $1}' - image: ghcr.io/saltstack/salt-ci-containers/testing:$trgt_slug-$trgt_version + image: ghcr.io/saltstack/salt-ci-containers/testing:${{ trgt_slug }}-${{ trgt_version }} timeout-minutes: ${{ inputs.timeout }} strategy: From c7e11a76370d9d6e9755986692ae909a3b2f8d1b Mon Sep 17 00:00:00 2001 From: David Murphy Date: Fri, 8 Nov 2024 08:59:06 -0700 Subject: [PATCH 058/173] Using Salt CI containers, removed OSs not supported by them --- .github/workflows/ci.yml | 118 ++++++------------------ .github/workflows/templates/generate.py | 105 ++++++++------------- .github/workflows/test-linux.yml | 4 +- bootstrap-salt.sh | 32 +++---- 4 files changed, 87 insertions(+), 172 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 91e8d2cc1..8139bbff9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -161,35 +161,7 @@ jobs: - almalinux-8: - name: AlmaLinux 8 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: almalinux-8 - display-name: AlmaLinux 8 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' - - - almalinux-9: - name: AlmaLinux 9 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: almalinux-9 - display-name: AlmaLinux 9 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - - - amazon-2: + amazonlinux-2: name: Amazon 2 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' uses: ./.github/workflows/test-linux.yml @@ -197,24 +169,24 @@ jobs: - lint - generate-actions-workflow with: - distro-slug: amazon-2 + distro-slug: amazonlinux-2 display-name: Amazon 2 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' - centos-stream9: - name: CentOS Stream 9 + amazonlinux-2023: + name: Amazon 2023 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' uses: ./.github/workflows/test-linux.yml needs: - lint - generate-actions-workflow with: - distro-slug: centos-stream9 - display-name: CentOS Stream 9 + distro-slug: amazonlinux-2023 + display-name: Amazon 2023 timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' + instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' debian-11: @@ -245,60 +217,32 @@ jobs: instances: '["stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - fedora-40: - name: Fedora 40 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: fedora-40 - display-name: Fedora 40 - timeout: 20 - instances: '["stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - - - opensuse-15: - name: Opensuse 15 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: opensuse-15 - display-name: Opensuse 15 - timeout: 20 - instances: '["latest", "default"]' - - - oraclelinux-8: - name: Oracle Linux 8 + debian-13: + name: Debian 13 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' uses: ./.github/workflows/test-linux.yml needs: - lint - generate-actions-workflow with: - distro-slug: oraclelinux-8 - display-name: Oracle Linux 8 + distro-slug: debian-13 + display-name: Debian 13 timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' + instances: '["stable-3006", "git-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - oraclelinux-9: - name: Oracle Linux 9 + fedora-40: + name: Fedora 40 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' uses: ./.github/workflows/test-linux.yml needs: - lint - generate-actions-workflow with: - distro-slug: oraclelinux-9 - display-name: Oracle Linux 9 + distro-slug: fedora-40 + display-name: Fedora 40 timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' + instances: '["stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' photon-4: @@ -357,7 +301,7 @@ jobs: instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - ubuntu-2004: + ubuntu-20.04: name: Ubuntu 20.04 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' uses: ./.github/workflows/test-linux.yml @@ -365,13 +309,13 @@ jobs: - lint - generate-actions-workflow with: - distro-slug: ubuntu-2004 + distro-slug: ubuntu-20.04 display-name: Ubuntu 20.04 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - ubuntu-2204: + ubuntu-22.04: name: Ubuntu 22.04 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' uses: ./.github/workflows/test-linux.yml @@ -379,13 +323,13 @@ jobs: - lint - generate-actions-workflow with: - distro-slug: ubuntu-2204 + distro-slug: ubuntu-22.04 display-name: Ubuntu 22.04 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - ubuntu-2404: + ubuntu-24.04: name: Ubuntu 24.04 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' uses: ./.github/workflows/test-linux.yml @@ -393,7 +337,7 @@ jobs: - lint - generate-actions-workflow with: - distro-slug: ubuntu-2404 + distro-slug: ubuntu-24.04 display-name: Ubuntu 24.04 timeout: 20 instances: '["stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' @@ -410,23 +354,19 @@ jobs: - macos-12 - macos-13 - windows-2022 - - almalinux-8 - - almalinux-9 - - amazon-2 - - centos-stream9 + - amazonlinux-2 + - amazonlinux-2023 - debian-11 - debian-12 + - debian-13 - fedora-40 - - opensuse-15 - - oraclelinux-8 - - oraclelinux-9 - photon-4 - photon-5 - rockylinux-8 - rockylinux-9 - - ubuntu-2004 - - ubuntu-2204 - - ubuntu-2404 + - ubuntu-20.04 + - ubuntu-22.04 + - ubuntu-24.04 if: always() steps: diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index f1a5f39e4..a9fd62eb2 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -7,23 +7,19 @@ os.chdir(os.path.abspath(os.path.dirname(__file__))) LINUX_DISTROS = [ - "almalinux-8", - "almalinux-9", - "amazon-2", - "centos-stream9", + "amazonlinux-2", + "amazonlinux-2023", "debian-11", "debian-12", + "debian-13", "fedora-40", - "opensuse-15", - "oraclelinux-8", - "oraclelinux-9", "photon-4", "photon-5", "rockylinux-8", "rockylinux-9", - "ubuntu-2004", - "ubuntu-2204", - "ubuntu-2404", + "ubuntu-20.04", + "ubuntu-22.04", + "ubuntu-24.04", ] WINDOWS = [ @@ -36,32 +32,28 @@ ] STABLE_DISTROS = [ - "almalinux-8", - "almalinux-9", - "amazon-2", + "amazonlinux-2", + "amazonlinux-2023", "centos-stream9", "debian-11", "debian-12", + "debian-13", "fedora-40", - "opensuse-15", - "oraclelinux-8", - "oraclelinux-9", "photon-4", "photon-5", "rockylinux-8", "rockylinux-9", - "ubuntu-2004", - "ubuntu-2204", - "ubuntu-2404", + "ubuntu-20.04", + "ubuntu-22.04", + "ubuntu-24.04", ] ONEDIR_DISTROS = [ - "almalinux-8", - "almalinux-9", - "amazon-2", - "centos-stream9", + "amazonlinux-2", + "amazonlinux-2023", "debian-11", "debian-12", + "debian-13", "fedora-40", "oraclelinux-8", "oraclelinux-9", @@ -69,72 +61,64 @@ "photon-5", "rockylinux-8", "rockylinux-9", - "ubuntu-2004", - "ubuntu-2204", - "ubuntu-2404", + "ubuntu-20.04", + "ubuntu-22.04", + "ubuntu-24.04", ] ONEDIR_RC_DISTROS = [ - "almalinux-9", - "amazon-2", - "centos-stream9", + "amazonlinux-2", + "amazonlinux-2023", "debian-12", - "oraclelinux-9", "photon-4", "photon-5", "rockylinux-9", - "ubuntu-2404", + "ubuntu-24.04", ] BLACKLIST_3006 = [ "debian-12", "fedora-40", - "opensuse-15", - "ubuntu-2404", + "ubuntu-24.04", ] BLACKLIST_3007 = [ - "opensuse-15", "photon-4", "photon-5", ] BLACKLIST_GIT_3006 = [ - "almalinux-9", - "amazon-2", - "centos-stream9", + "amazonlinux-2", + "amazonlinux-2023", "debian-11", "debian-12", "fedora-40", - "opensuse-15", - "oraclelinux-9", "photon-4", "photon-5", "rockylinux-9", - "ubuntu-2004", - "ubuntu-2204", - "ubuntu-2404", + "ubuntu-20.04", + "ubuntu-22.04", + "ubuntu-24.04", ] BLACKLIST_GIT_3007 = [ - "almalinux-9", - "amazon-2", - "centos-stream9", + "amazonlinux-2", + "amazonlinux-2023", "debian-11", "debian-12", + "debian-13", "fedora-40", - "opensuse-15", - "oraclelinux-9", "photon-4", "photon-5", "rockylinux-9", - "ubuntu-2004", - "ubuntu-2204", - "ubuntu-2404", + "ubuntu-20.04", + "ubuntu-22.04", + "ubuntu-24.04", ] BLACKLIST_GIT_MASTER = [ - "amazon-2", + "amazonlinux-2", + "amazonlinux-2023", "photon-4", "photon-5", ] @@ -193,32 +177,25 @@ # Use build and pip and other standards-based tools. # GIT_DISTRO_BLACKLIST = [ - "almalinux-8", - "opensuse-15", - "oraclelinux-8", "rockylinux-8", ] LATEST_PKG_BLACKLIST = [] DISTRO_DISPLAY_NAMES = { - "almalinux-8": "AlmaLinux 8", - "almalinux-9": "AlmaLinux 9", - "amazon-2": "Amazon 2", - "centos-stream9": "CentOS Stream 9", + "amazonlinux-2": "Amazon 2", + "amazonlinux-2023": "Amazon 2023", "debian-11": "Debian 11", "debian-12": "Debian 12", + "debian-13": "Debian 13", "fedora-40": "Fedora 40", - "opensuse-15": "Opensuse 15", - "oraclelinux-8": "Oracle Linux 8", - "oraclelinux-9": "Oracle Linux 9", "photon-4": "Photon OS 4", "photon-5": "Photon OS 5", "rockylinux-8": "Rocky Linux 8", "rockylinux-9": "Rocky Linux 9", - "ubuntu-2004": "Ubuntu 20.04", - "ubuntu-2204": "Ubuntu 22.04", - "ubuntu-2404": "Ubuntu 24.04", + "ubuntu-20.04": "Ubuntu 20.04", + "ubuntu-22.04": "Ubuntu 22.04", + "ubuntu-24.04": "Ubuntu 24.04", "macos-12": "macOS 12", "macos-13": "macOS 13", "windows-2022": "Windows 2022", diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 8a2eda0f6..80b2e0917 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -28,9 +28,7 @@ jobs: name: ${{ matrix.instance }} runs-on: ubuntu-latest container: - $trgt_version = echo "${{ inputs.display-name }}" | awk -F ' ' '{print $2}' - $trgt_slug = echo "${{ inputs.distro-slug}}" | awk -F '-' '{print $1}' - image: ghcr.io/saltstack/salt-ci-containers/testing:${{ trgt_slug }}-${{ trgt_version }} + image: ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.distro-slug}} timeout-minutes: ${{ inputs.timeout }} strategy: diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index d4fa965e8..6b81597db 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -4993,79 +4993,79 @@ install_oracle_linux_check_services() { ####################################################################################################################### # -# AlmaLinux Install Functions +# RockyLinux Install Functions # -install_almalinux_stable_deps() { +install_rockylinux_stable_deps() { install_centos_stable_deps || return 1 return 0 } -install_almalinux_git_deps() { +install_rockylinux_git_deps() { install_centos_git_deps || return 1 return 0 } -install_almalinux_onedir_deps() { +install_rockylinux_onedir_deps() { install_centos_onedir_deps || return 1 return 0 } -install_almalinux_testing_deps() { +install_rockylinux_testing_deps() { install_centos_testing_deps || return 1 return 0 } -install_almalinux_stable() { +install_rockylinux_stable() { install_centos_stable || return 1 return 0 } -install_almalinux_git() { +install_rockylinux_git() { install_centos_git || return 1 return 0 } -install_almalinux_onedir() { +install_rockylinux_onedir() { install_centos_onedir || return 1 return 0 } -install_almalinux_testing() { +install_rockylinux_testing() { install_centos_testing || return 1 return 0 } -install_almalinux_stable_post() { +install_rockylinux_stable_post() { install_centos_stable_post || return 1 return 0 } -install_almalinux_git_post() { +install_rockylinux_git_post() { install_centos_git_post || return 1 return 0 } -install_almalinux_onedir_post() { +install_rockylinux_onedir_post() { install_centos_onedir_post || return 1 return 0 } -install_almalinux_testing_post() { +install_rockylinux_testing_post() { install_centos_testing_post || return 1 return 0 } -install_almalinux_restart_daemons() { +install_rockylinux_restart_daemons() { install_centos_restart_daemons || return 1 return 0 } -install_almalinux_check_services() { +install_rockylinux_check_services() { install_centos_check_services || return 1 return 0 } # -# Ended AlmaLinux Install Functions +# Ended RockyLinux Install Functions # ####################################################################################################################### From 906d38d941b6353077b4349a006484f7dff798fe Mon Sep 17 00:00:00 2001 From: David Murphy Date: Fri, 8 Nov 2024 09:29:22 -0700 Subject: [PATCH 059/173] Added container_slug --- .github/workflows/ci.yml | 34 +++++++++---- .github/workflows/templates/generate.py | 63 +++++++++++++++++-------- .github/workflows/test-linux.yml | 7 ++- 3 files changed, 74 insertions(+), 30 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8139bbff9..28e300ae3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -124,6 +124,7 @@ jobs: with: distro-slug: macos-12 display-name: macOS 12 + container-slug: macos-12 timeout: 20 runs-on: macos-12 instances: '["stable-3006", "stable-3006-8", "stable-3007", "stable-3007-1", "latest"]' @@ -139,6 +140,7 @@ jobs: with: distro-slug: macos-13 display-name: macOS 13 + container-slug: macos-13 timeout: 20 runs-on: macos-13 instances: '["stable-3006", "stable-3006-8", "stable-3007", "stable-3007-1", "latest"]' @@ -155,6 +157,7 @@ jobs: with: distro-slug: windows-2022 display-name: Windows 2022 + container-slug: windows-2022 timeout: 20 runs-on: windows-2022 instances: '["stable-3006", "stable-3006-8", "stable-3007", "stable-3007-1", "latest"]' @@ -171,6 +174,7 @@ jobs: with: distro-slug: amazonlinux-2 display-name: Amazon 2 + container-slug: amazonlinux-2 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' @@ -185,6 +189,7 @@ jobs: with: distro-slug: amazonlinux-2023 display-name: Amazon 2023 + container-slug: amazonlinux-2023 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' @@ -199,6 +204,7 @@ jobs: with: distro-slug: debian-11 display-name: Debian 11 + container-slug: debian-11 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' @@ -213,6 +219,7 @@ jobs: with: distro-slug: debian-12 display-name: Debian 12 + container-slug: debian-12 timeout: 20 instances: '["stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' @@ -227,6 +234,7 @@ jobs: with: distro-slug: debian-13 display-name: Debian 13 + container-slug: debian-13 timeout: 20 instances: '["stable-3006", "git-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' @@ -241,6 +249,7 @@ jobs: with: distro-slug: fedora-40 display-name: Fedora 40 + container-slug: fedora-40 timeout: 20 instances: '["stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' @@ -255,6 +264,7 @@ jobs: with: distro-slug: photon-4 display-name: Photon OS 4 + container-slug: photon-4 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "latest", "default"]' @@ -269,6 +279,7 @@ jobs: with: distro-slug: photon-5 display-name: Photon OS 5 + container-slug: photon-5 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "latest", "default"]' @@ -283,6 +294,7 @@ jobs: with: distro-slug: rockylinux-8 display-name: Rocky Linux 8 + container-slug: rockylinux-8 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' @@ -297,11 +309,12 @@ jobs: with: distro-slug: rockylinux-9 display-name: Rocky Linux 9 + container-slug: rockylinux-9 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - ubuntu-20.04: + ubuntu-2004: name: Ubuntu 20.04 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' uses: ./.github/workflows/test-linux.yml @@ -309,13 +322,14 @@ jobs: - lint - generate-actions-workflow with: - distro-slug: ubuntu-20.04 + distro-slug: ubuntu-2004 display-name: Ubuntu 20.04 + container-slug: ubuntu-20.04 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - ubuntu-22.04: + ubuntu-2204: name: Ubuntu 22.04 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' uses: ./.github/workflows/test-linux.yml @@ -323,13 +337,14 @@ jobs: - lint - generate-actions-workflow with: - distro-slug: ubuntu-22.04 + distro-slug: ubuntu-2204 display-name: Ubuntu 22.04 + container-slug: ubuntu-22.04 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - ubuntu-24.04: + ubuntu-2404: name: Ubuntu 24.04 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' uses: ./.github/workflows/test-linux.yml @@ -337,8 +352,9 @@ jobs: - lint - generate-actions-workflow with: - distro-slug: ubuntu-24.04 + distro-slug: ubuntu-2404 display-name: Ubuntu 24.04 + container-slug: ubuntu-24.04 timeout: 20 instances: '["stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' @@ -364,9 +380,9 @@ jobs: - photon-5 - rockylinux-8 - rockylinux-9 - - ubuntu-20.04 - - ubuntu-22.04 - - ubuntu-24.04 + - ubuntu-2004 + - ubuntu-2204 + - ubuntu-2404 if: always() steps: diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index a9fd62eb2..cb7bf05b8 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -17,9 +17,9 @@ "photon-5", "rockylinux-8", "rockylinux-9", - "ubuntu-20.04", - "ubuntu-22.04", - "ubuntu-24.04", + "ubuntu-2004", + "ubuntu-2204", + "ubuntu-2404", ] WINDOWS = [ @@ -43,9 +43,9 @@ "photon-5", "rockylinux-8", "rockylinux-9", - "ubuntu-20.04", - "ubuntu-22.04", - "ubuntu-24.04", + "ubuntu-2004", + "ubuntu-2204", + "ubuntu-2404", ] ONEDIR_DISTROS = [ @@ -61,9 +61,9 @@ "photon-5", "rockylinux-8", "rockylinux-9", - "ubuntu-20.04", - "ubuntu-22.04", - "ubuntu-24.04", + "ubuntu-2004", + "ubuntu-2204", + "ubuntu-2404", ] ONEDIR_RC_DISTROS = [ @@ -73,13 +73,13 @@ "photon-4", "photon-5", "rockylinux-9", - "ubuntu-24.04", + "ubuntu-2404", ] BLACKLIST_3006 = [ "debian-12", "fedora-40", - "ubuntu-24.04", + "ubuntu-2404", ] BLACKLIST_3007 = [ @@ -96,9 +96,9 @@ "photon-4", "photon-5", "rockylinux-9", - "ubuntu-20.04", - "ubuntu-22.04", - "ubuntu-24.04", + "ubuntu-2004", + "ubuntu-2204", + "ubuntu-2404", ] BLACKLIST_GIT_3007 = [ @@ -111,9 +111,9 @@ "photon-4", "photon-5", "rockylinux-9", - "ubuntu-20.04", - "ubuntu-22.04", - "ubuntu-24.04", + "ubuntu-2004", + "ubuntu-2204", + "ubuntu-2404", ] BLACKLIST_GIT_MASTER = [ @@ -193,14 +193,33 @@ "photon-5": "Photon OS 5", "rockylinux-8": "Rocky Linux 8", "rockylinux-9": "Rocky Linux 9", - "ubuntu-20.04": "Ubuntu 20.04", - "ubuntu-22.04": "Ubuntu 22.04", - "ubuntu-24.04": "Ubuntu 24.04", + "ubuntu-2004": "Ubuntu 20.04", + "ubuntu-2204": "Ubuntu 22.04", + "ubuntu-2404": "Ubuntu 24.04", "macos-12": "macOS 12", "macos-13": "macOS 13", "windows-2022": "Windows 2022", } +CONTAINER_SLUG_NAMES = { + "amazonlinux-2": "amazonlinux-2", + "amazonlinux-2023": "amazonlinux-2023", + "debian-11": "debian-11", + "debian-12": "debian-12", + "debian-13": "debian-13", + "fedora-40": "fedora-40", + "photon-4": "photon-4", + "photon-5": "photon-5", + "rockylinux-8": "rockylinux-8", + "rockylinux-9": "rockylinux-9", + "ubuntu-2004": "ubuntu-20.04", + "ubuntu-2204": "ubuntu-22.04", + "ubuntu-2404": "ubuntu-24.04", + "macos-12": "macos-12", + "macos-13": "macos-13", + "windows-2022": "windows-2022", +} + TIMEOUT_DEFAULT = 20 TIMEOUT_OVERRIDES = {} VERSION_ONLY_OVERRIDES = [] @@ -215,6 +234,7 @@ with: distro-slug: {distro} display-name: {display_name} + container-slug: {container_name} timeout: {timeout_minutes}{runs_on} instances: '{instances}' """ @@ -265,6 +285,7 @@ def generate_test_jobs(): ifcheck=ifcheck, instances=json.dumps(instances), display_name=DISTRO_DISPLAY_NAMES[distro], + container_name=CONTAINER_SLUG_NAMES[distro], timeout_minutes=timeout_minutes, ) @@ -310,6 +331,7 @@ def generate_test_jobs(): ifcheck=ifcheck, instances=json.dumps(instances), display_name=DISTRO_DISPLAY_NAMES[distro], + container_name=CONTAINER_SLUG_NAMES[distro], timeout_minutes=timeout_minutes, ) @@ -409,6 +431,7 @@ def generate_test_jobs(): ifcheck=ifcheck, instances=json.dumps(instances), display_name=DISTRO_DISPLAY_NAMES[distro], + container_name=CONTAINER_SLUG_NAMES[distro], timeout_minutes=timeout_minutes, ) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 80b2e0917..2b5268e3c 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -17,6 +17,11 @@ on: required: true description: The Instances To Test + container-slug: + type: string + required: true + description: The Container Slug + timeout: type: number required: false @@ -28,7 +33,7 @@ jobs: name: ${{ matrix.instance }} runs-on: ubuntu-latest container: - image: ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.distro-slug}} + image: ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} timeout-minutes: ${{ inputs.timeout }} strategy: From a902af3c53f7478740b820e6d9c4c70ce1cc1f5c Mon Sep 17 00:00:00 2001 From: David Murphy Date: Fri, 8 Nov 2024 09:36:55 -0700 Subject: [PATCH 060/173] Added container-slug to MacOS and Windows workflows (unused on those platforms) --- .github/workflows/test-macos.yml | 5 +++++ .github/workflows/test-windows.yml | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index f4ad541f0..d6c43f04b 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -22,6 +22,11 @@ on: required: true description: The Instances To Test + container-slug: + type: string + required: true + description: The Container Slug + timeout: type: number required: false diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index e4d4dde34..0451714a7 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -22,6 +22,11 @@ on: required: true description: The Instances To Test + container-slug: + type: string + required: true + description: The Container Slug + timeout: type: number required: false From 5d0e68b3ee59a7a6a0174e7e07b7f6eba1f77e39 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Fri, 8 Nov 2024 10:02:26 -0700 Subject: [PATCH 061/173] Updated Python dependencies --- .github/workflows/test-linux.yml | 5 +++-- .github/workflows/test-macos.yml | 5 +++-- .github/workflows/test-windows.yml | 5 +++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 2b5268e3c..a3246909b 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -45,9 +45,10 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Install Pytest + - name: Install Python Dependencies run: | - pip install -U pytest + python3 -m pip install -U pip + python3 -m pip install -r tests/requirements.txt - name: Bootstrap Salt run: | diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index d6c43f04b..e0341d6b2 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -47,9 +47,10 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Install Pytest + - name: Install Python Dependencies run: | - pip install -U pytest + python3 -m pip install -U pip + python3 -m pip install -r tests/requirements.txt - name: Bootstrap Salt run: | diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index 0451714a7..df2a1148c 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -51,9 +51,10 @@ jobs: with: python-version: "3.10" - - name: Install Pytest + - name: Install Python Dependencies run: | - pip install -r tests\requirements.txt + python3 -m pip install -U pip + python3 -m pip install -r tests/requirements.txt - name: Get Version run: | From e1e8f208a081b0184ba82cbbe4a70e980bd29344 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Fri, 8 Nov 2024 10:48:14 -0700 Subject: [PATCH 062/173] Added '--break-system-packages' for pip install --- .github/workflows/test-linux.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index a3246909b..4e26f077f 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -47,13 +47,13 @@ jobs: - name: Install Python Dependencies run: | - python3 -m pip install -U pip - python3 -m pip install -r tests/requirements.txt + python3 -m pip install --break-system-packages -U pip + python3 -m pip install --break-system-packages -r tests/requirements.txt - name: Bootstrap Salt run: | echo "matrix instance ,${{ matrix.instance }}," - $bt_parms = echo "${{ matrix.instance }}" | sed 's/-/ /' + bt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /') echo "bt parms ,$bt_parms," sh -x ./bootstrap-salt.sh $bt_parms From 0c255cfb80a45823d0c63416f67c7f15106d9d78 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Fri, 8 Nov 2024 11:42:35 -0700 Subject: [PATCH 063/173] Testing --- .github/workflows/test-linux.yml | 2 +- .github/workflows/test-macos.yml | 2 +- .github/workflows/test-windows.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 4e26f077f..8d5dfbaa4 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -47,7 +47,7 @@ jobs: - name: Install Python Dependencies run: | - python3 -m pip install --break-system-packages -U pip + ## python3 -m pip install --break-system-packages -U pip python3 -m pip install --break-system-packages -r tests/requirements.txt - name: Bootstrap Salt diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index e0341d6b2..ba8743005 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -49,7 +49,7 @@ jobs: - name: Install Python Dependencies run: | - python3 -m pip install -U pip + ## python3 -m pip install -U pip python3 -m pip install -r tests/requirements.txt - name: Bootstrap Salt diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index df2a1148c..e97733d36 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -53,7 +53,7 @@ jobs: - name: Install Python Dependencies run: | - python3 -m pip install -U pip + ## python3 -m pip install -U pip python3 -m pip install -r tests/requirements.txt - name: Get Version From ef3d82c1a8f8c5eb686a825d0a646e202d44244b Mon Sep 17 00:00:00 2001 From: David Murphy Date: Fri, 8 Nov 2024 14:49:48 -0700 Subject: [PATCH 064/173] Adding conditionals for needing break-system-packages on Linux wiht pip --- .github/workflows/test-linux.yml | 7 +++++-- .github/workflows/test-macos.yml | 1 - .github/workflows/test-windows.yml | 1 - 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 8d5dfbaa4..714c9db86 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -47,8 +47,11 @@ jobs: - name: Install Python Dependencies run: | - ## python3 -m pip install --break-system-packages -U pip - python3 -m pip install --break-system-packages -r tests/requirements.txt + if [[ ${{ inputs.distro-slug }} == "debian-11" || ${{ inputs.distro-slug }} == "debian-12" || ${{ inputs.distro-slug }} == "debian-13" || ${{ inputs.distro-slug }} == "ubuntu-2404" ]]; then + python3 -m pip install --break-system-packages -r tests/requirements.txt + else + python3 -m pip install -r tests/requirements.txt + fi - name: Bootstrap Salt run: | diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index ba8743005..8ad49f2c4 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -49,7 +49,6 @@ jobs: - name: Install Python Dependencies run: | - ## python3 -m pip install -U pip python3 -m pip install -r tests/requirements.txt - name: Bootstrap Salt diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml index e97733d36..9515dae8d 100644 --- a/.github/workflows/test-windows.yml +++ b/.github/workflows/test-windows.yml @@ -53,7 +53,6 @@ jobs: - name: Install Python Dependencies run: | - ## python3 -m pip install -U pip python3 -m pip install -r tests/requirements.txt - name: Get Version From 5776297744267df68595c3f1c85c86d25241adb5 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Fri, 8 Nov 2024 15:16:59 -0700 Subject: [PATCH 065/173] Debugging pip conditional --- .github/workflows/test-linux.yml | 3 +++ .github/workflows/test-macos.yml | 5 ++++- tests/integration/test_installation.py | 4 ++-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 714c9db86..2d562f21c 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -47,9 +47,12 @@ jobs: - name: Install Python Dependencies run: | + echo "DGM pip test for distro-slug ${{ inputs.distro-slug }}" if [[ ${{ inputs.distro-slug }} == "debian-11" || ${{ inputs.distro-slug }} == "debian-12" || ${{ inputs.distro-slug }} == "debian-13" || ${{ inputs.distro-slug }} == "ubuntu-2404" ]]; then + echo "DGM doing break-system-packages for distro-slug ${{ inputs.distro-slug }}" python3 -m pip install --break-system-packages -r tests/requirements.txt else + echo "DGM plain pip for distro-slug ${{ inputs.distro-slug }}" python3 -m pip install -r tests/requirements.txt fi diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index 8ad49f2c4..790c280bc 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -53,7 +53,10 @@ jobs: - name: Bootstrap Salt run: | - sudo sh -x ./bootstrap-salt.sh + echo "matrix instance ,${{ matrix.instance }}," + bt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /') + echo "bt parms ,$bt_parms," + sudo sh -x ./bootstrap-salt.sh $bt_parms - name: Test Bootstrap run: | diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index f8427b675..928ae829f 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -15,7 +15,7 @@ def path(): salt_path = "C:\\Program Files\\Salt Project\\Salt" if salt_path not in os.environ["path"]: os.environ["path"] = f'{os.environ["path"]};{salt_path}' - yield os.environ["path"] + yield os.environ["path"] def run_salt_call(cmd): @@ -49,4 +49,4 @@ def test_target_salt_version(path, target_salt_version): cmd = ["salt-call", "--local", "grains.item", "saltversion", "--timeout=120"] result = run_salt_call(cmd) # Returns: {'saltversion': '3006.9+217.g53cfa53040'} - assert result["saltversion"] == target_salt_version \ No newline at end of file + assert result["saltversion"] == target_salt_version From 03d451820d05cf99463756a573ba88e1e39e0bd7 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Fri, 8 Nov 2024 15:36:57 -0700 Subject: [PATCH 066/173] More debugging parm usage --- .github/workflows/test-linux.yml | 4 ++-- .github/workflows/test-macos.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 2d562f21c..a77a1ca6f 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -48,7 +48,7 @@ jobs: - name: Install Python Dependencies run: | echo "DGM pip test for distro-slug ${{ inputs.distro-slug }}" - if [[ ${{ inputs.distro-slug }} == "debian-11" || ${{ inputs.distro-slug }} == "debian-12" || ${{ inputs.distro-slug }} == "debian-13" || ${{ inputs.distro-slug }} == "ubuntu-2404" ]]; then + if [[ "${{ inputs.distro-slug }}" == "debian-11" || "${{ inputs.distro-slug }}" == "debian-12" || "${{ inputs.distro-slug }}" == "debian-13" || "${{ inputs.distro-slug }}" == "ubuntu-2404" ]]; then echo "DGM doing break-system-packages for distro-slug ${{ inputs.distro-slug }}" python3 -m pip install --break-system-packages -r tests/requirements.txt else @@ -61,7 +61,7 @@ jobs: echo "matrix instance ,${{ matrix.instance }}," bt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /') echo "bt parms ,$bt_parms," - sh -x ./bootstrap-salt.sh $bt_parms + sh -x ./bootstrap-salt.sh "$bt_parms" - name: Test Bootstrap run: | diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index 790c280bc..171bd2282 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -56,7 +56,7 @@ jobs: echo "matrix instance ,${{ matrix.instance }}," bt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /') echo "bt parms ,$bt_parms," - sudo sh -x ./bootstrap-salt.sh $bt_parms + sudo sh -x ./bootstrap-salt.sh "$bt_parms" - name: Test Bootstrap run: | From 90fc657e7132c0155d23d5593a574b733995a03f Mon Sep 17 00:00:00 2001 From: David Murphy Date: Fri, 8 Nov 2024 15:38:20 -0700 Subject: [PATCH 067/173] more debug --- tests/integration/test_installation.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index 928ae829f..84363c561 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -16,6 +16,7 @@ def path(): if salt_path not in os.environ["path"]: os.environ["path"] = f'{os.environ["path"]};{salt_path}' yield os.environ["path"] + yield "" def run_salt_call(cmd): From 1fa8125a51c019822e6977e6d93e405093334b2e Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 11 Nov 2024 13:49:28 -0700 Subject: [PATCH 068/173] Playing with conditionals --- .github/workflows/test-linux.yml | 15 +++++++++++---- bootstrap-salt.sh | 19 ++++++++++++++++--- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index a77a1ca6f..e6a5be5c1 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -45,17 +45,24 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Install Python Dependencies + - name: Install Python Dependencies with pip breakage + if: ${{ ( inputs.distro-slug = "debian-11" ) || ( inputs.distro-slug = "debian-12" ) || ( inputs.distro-slug = "debian-13" ) || ( inputs.distro-slug = "ubuntu-2404" ) }} run: | echo "DGM pip test for distro-slug ${{ inputs.distro-slug }}" - if [[ "${{ inputs.distro-slug }}" == "debian-11" || "${{ inputs.distro-slug }}" == "debian-12" || "${{ inputs.distro-slug }}" == "debian-13" || "${{ inputs.distro-slug }}" == "ubuntu-2404" ]]; then - echo "DGM doing break-system-packages for distro-slug ${{ inputs.distro-slug }}" - python3 -m pip install --break-system-packages -r tests/requirements.txt + echo "DGM doing break-system-packages for distro-slug ${{ inputs.distro-slug }}" + python3 -m pip install --break-system-packages -r tests/requirements.txt else echo "DGM plain pip for distro-slug ${{ inputs.distro-slug }}" python3 -m pip install -r tests/requirements.txt fi + - name: Install Python Dependencies without pip breakage + if: ${{ ( inputs.distro-slug != "debian-11" ) && ( inputs.distro-slug != "debian-12" ) && ( inputs.distro-slug != "debian-13" ) && ( inputs.distro-slug != "ubuntu-2404" ) }} + run: | + echo "DGM pip test for distro-slug ${{ inputs.distro-slug }}" + echo "DGM plain pip for distro-slug ${{ inputs.distro-slug }}" + python3 -m pip install -r tests/requirements.txt + - name: Bootstrap Salt run: | echo "matrix instance ,${{ matrix.instance }}," diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 6b81597db..9028f7b43 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -26,7 +26,7 @@ #====================================================================================================================== set -o nounset # Treat unset variables as an error -__ScriptVersion="2024.11.07" +__ScriptVersion="2024.11.11" __ScriptName="bootstrap-salt.sh" __ScriptFullName="$0" @@ -221,6 +221,10 @@ __check_config_dir() { # DESCRIPTION: Checks the placed after the install arguments #---------------------------------------------------------------------------------------------------------------------- __check_unparsed_options() { + # DGM debug + set -v + set -x + shellopts="$1" # grep alternative for SunOS if [ -f /usr/xpg4/bin/grep ]; then @@ -619,13 +623,22 @@ if [ "$#" -gt 0 ];then fi # Check installation type -if [ "$(echo "$ITYPE" | grep -E '(stable|testing|git|onedir|onedir_rc)')" = "" ]; then +if [ "$(echo "$ITYPE" | grep -E '(latest|default|stable|testing|git|onedir|onedir_rc)')" = "" ]; then echoerror "Installation type \"$ITYPE\" is not known..." exit 1 fi +## DGM added this to CI/CD easier, needs removal once get compund conditionals resolved in GitHub Actions +if [ "$ITYPE" = "latest" ] || [ "$ITYPE" = "default" ]; then + STABLE_REV="latest" + ONEDIR_REV="latest" + _ONEDIR_REV="latest" + ITYPE="onedir" + shift + echodebug "using ITYPE onedir for input 'latest' or 'default', cmd args left ,$#," + # If doing a git install, check what branch/tag/sha will be checked out -if [ "$ITYPE" = "git" ]; then +elif [ "$ITYPE" = "git" ]; then if [ "$#" -eq 0 ];then GIT_REV="master" else From 03fd45ba97ecbcb4940ba3d1ea00b512962210ba Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 11 Nov 2024 13:53:00 -0700 Subject: [PATCH 069/173] Fix typo --- .github/workflows/test-linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index e6a5be5c1..3706030f3 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -46,7 +46,7 @@ jobs: - uses: actions/checkout@v4 - name: Install Python Dependencies with pip breakage - if: ${{ ( inputs.distro-slug = "debian-11" ) || ( inputs.distro-slug = "debian-12" ) || ( inputs.distro-slug = "debian-13" ) || ( inputs.distro-slug = "ubuntu-2404" ) }} + if: ${{ ( inputs.distro-slug == "debian-11" ) || ( inputs.distro-slug == "debian-12" ) || ( inputs.distro-slug == "debian-13" ) || ( inputs.distro-slug == "ubuntu-2404" ) }} run: | echo "DGM pip test for distro-slug ${{ inputs.distro-slug }}" echo "DGM doing break-system-packages for distro-slug ${{ inputs.distro-slug }}" From 55f219224c12982e0d7d5fbe64a5da0f4a61644d Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 11 Nov 2024 13:57:14 -0700 Subject: [PATCH 070/173] Fixed using ' in conditional github actions --- .github/workflows/test-linux.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 3706030f3..f3ba7af4a 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -46,7 +46,7 @@ jobs: - uses: actions/checkout@v4 - name: Install Python Dependencies with pip breakage - if: ${{ ( inputs.distro-slug == "debian-11" ) || ( inputs.distro-slug == "debian-12" ) || ( inputs.distro-slug == "debian-13" ) || ( inputs.distro-slug == "ubuntu-2404" ) }} + if: ${{ ( inputs.distro-slug == 'debian-11' ) || ( inputs.distro-slug == 'debian-12' ) || ( inputs.distro-slug == 'debian-13' ) || ( inputs.distro-slug == 'ubuntu-2404' ) }} run: | echo "DGM pip test for distro-slug ${{ inputs.distro-slug }}" echo "DGM doing break-system-packages for distro-slug ${{ inputs.distro-slug }}" @@ -57,7 +57,7 @@ jobs: fi - name: Install Python Dependencies without pip breakage - if: ${{ ( inputs.distro-slug != "debian-11" ) && ( inputs.distro-slug != "debian-12" ) && ( inputs.distro-slug != "debian-13" ) && ( inputs.distro-slug != "ubuntu-2404" ) }} + if: ${{ ( inputs.distro-slug != 'debian-11' ) && ( inputs.distro-slug != 'debian-12' ) && ( inputs.distro-slug != 'debian-13' ) && ( inputs.distro-slug != 'ubuntu-2404' ) }} run: | echo "DGM pip test for distro-slug ${{ inputs.distro-slug }}" echo "DGM plain pip for distro-slug ${{ inputs.distro-slug }}" From 1f885ee0916f4068d9a075e50d1a8b1835446034 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 11 Nov 2024 14:28:28 -0700 Subject: [PATCH 071/173] Debug tests output --- .github/workflows/test-linux.yml | 4 ---- bootstrap-salt.sh | 4 +++- tests/integration/test_installation.py | 4 ++++ 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index f3ba7af4a..dcfddc6e9 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -51,10 +51,6 @@ jobs: echo "DGM pip test for distro-slug ${{ inputs.distro-slug }}" echo "DGM doing break-system-packages for distro-slug ${{ inputs.distro-slug }}" python3 -m pip install --break-system-packages -r tests/requirements.txt - else - echo "DGM plain pip for distro-slug ${{ inputs.distro-slug }}" - python3 -m pip install -r tests/requirements.txt - fi - name: Install Python Dependencies without pip breakage if: ${{ ( inputs.distro-slug != 'debian-11' ) && ( inputs.distro-slug != 'debian-12' ) && ( inputs.distro-slug != 'debian-13' ) && ( inputs.distro-slug != 'ubuntu-2404' ) }} diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 9028f7b43..ab94e5f5f 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -634,7 +634,9 @@ if [ "$ITYPE" = "latest" ] || [ "$ITYPE" = "default" ]; then ONEDIR_REV="latest" _ONEDIR_REV="latest" ITYPE="onedir" - shift + if [ "$#" -gt 0 ];then + shift + fi echodebug "using ITYPE onedir for input 'latest' or 'default', cmd args left ,$#," # If doing a git install, check what branch/tag/sha will be checked out diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index 84363c561..9a3749889 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -26,6 +26,10 @@ def run_salt_call(cmd): """ cmd.append("--out=json") result = subprocess.run(cmd, capture_output=True, text=True) + print( + f"DGM run_salt_call, cmd '{cmd}', result '{result}', stdout '{result.stdout}'", + flush=True, + ) json_data = json.loads(result.stdout) return json_data["local"] From f2a5fc19a4aa99e2246d16b50f01dbe494b1e31c Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 11 Nov 2024 15:18:55 -0700 Subject: [PATCH 072/173] Update testing --- bootstrap-salt.sh | 120 ++++++++++++++++++++++++- tests/integration/test_installation.py | 29 ++++-- 2 files changed, 140 insertions(+), 9 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index ab94e5f5f..958e3030f 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -936,6 +936,10 @@ fi # DESCRIPTION: Retrieves a URL and writes it to a given path #---------------------------------------------------------------------------------------------------------------------- __fetch_url() { + # DGM debug + set -v + set -x + # shellcheck disable=SC2086 curl $_CURL_ARGS -L -s -f -o "$1" "$2" >/dev/null 2>&1 || wget $_WGET_ARGS -q -O "$1" "$2" >/dev/null 2>&1 || @@ -950,6 +954,10 @@ __fetch_url() { # DESCRIPTION: Retrieves a URL, verifies its content and writes it to standard output #---------------------------------------------------------------------------------------------------------------------- __fetch_verify() { + # DGM debug + set -v + set -x + fetch_verify_url="$1" fetch_verify_sum="$2" @@ -972,6 +980,10 @@ __fetch_verify() { # DESCRIPTION: Checks if a URL exists #---------------------------------------------------------------------------------------------------------------------- __check_url_exists() { + # DGM debug + set -v + set -x + _URL="$1" if curl --output /dev/null --silent --fail "${_URL}"; then @@ -1940,6 +1952,10 @@ __function_defined() { # process is finished so the script doesn't exit on a locked proc. #---------------------------------------------------------------------------------------------------------------------- __wait_for_apt(){ + # DGM debug + set -v + set -x + # Timeout set at 15 minutes WAIT_TIMEOUT=900 @@ -1977,6 +1993,10 @@ __wait_for_apt(){ # PARAMETERS: packages #---------------------------------------------------------------------------------------------------------------------- __apt_get_install_noinput() { + # DGM debug + set -v + set -x + __wait_for_apt apt-get install -y -o DPkg::Options::=--force-confold "${@}"; return $? } # ---------- end of function __apt_get_install_noinput ---------- @@ -1987,6 +2007,9 @@ __apt_get_install_noinput() { # DESCRIPTION: (DRY) apt-get upgrade with noinput options #---------------------------------------------------------------------------------------------------------------------- __apt_get_upgrade_noinput() { + # DGM debug + set -v + set -x __wait_for_apt apt-get upgrade -y -o DPkg::Options::=--force-confold; return $? } # ---------- end of function __apt_get_upgrade_noinput ---------- @@ -2018,7 +2041,9 @@ __temp_gpg_pub() { # PARAMETERS: url #---------------------------------------------------------------------------------------------------------------------- __apt_key_fetch() { - + # DGM debug + set -v + set -x url=$1 @@ -2627,6 +2652,10 @@ __activate_virtualenv() { #---------------------------------------------------------------------------------------------------------------------- __install_pip_pkgs() { + # DGM debug + set -v + set -x + _pip_pkgs="$1" _py_exe="$2" _py_pkg=$(echo "$_py_exe" | sed -E "s/\\.//g") @@ -2668,6 +2697,10 @@ __install_pip_pkgs() { # PARAMETERS: requirements_file #---------------------------------------------------------------------------------------------------------------------- __install_pip_deps() { + # DGM debug + set -v + set -x + # Install virtualenv to system pip before activating virtualenv if thats going to be used # We assume pip pkg is installed since that is distro specific if [ "$_VIRTUALENV_DIR" != "null" ]; then @@ -2704,6 +2737,9 @@ __install_pip_deps() { # PARAMETERS: py_exe #---------------------------------------------------------------------------------------------------------------------- __install_salt_from_repo() { + # DGM debug + set -v + set -x _py_exe="$1" @@ -2952,6 +2988,9 @@ fi # Ubuntu Install Functions # __enable_universe_repository() { + # DGM debug + set -v + set -x if [ "$(grep -R universe /etc/apt/sources.list /etc/apt/sources.list.d/ | grep -v '#')" != "" ]; then # The universe repository is already enabled @@ -2966,6 +3005,10 @@ __enable_universe_repository() { } __install_saltstack_ubuntu_repository() { + # DGM debug + set -v + set -x + # Workaround for latest non-LTS Ubuntu echodebug "__install_saltstack_ubuntu_repository() entry" @@ -3024,6 +3067,9 @@ __install_saltstack_ubuntu_repository() { } __install_saltstack_ubuntu_onedir_repository() { + # DGM debug + set -v + set -x echodebug "__install_saltstack_ubuntu_onedir_repository() entry" # Workaround for latest non-LTS Ubuntu @@ -3075,6 +3121,9 @@ __install_saltstack_ubuntu_onedir_repository() { } install_ubuntu_deps() { + # DGM debug + set -v + set -x echodebug "install_ubuntu_deps() entry" if [ "$_DISABLE_REPOS" -eq $BS_FALSE ]; then @@ -3133,6 +3182,9 @@ install_ubuntu_deps() { } install_ubuntu_stable_deps() { + # DGM debug + set -v + set -x echodebug "install_ubuntu_stable_deps() entry" @@ -3168,6 +3220,9 @@ install_ubuntu_stable_deps() { } install_ubuntu_git_deps() { + # DGM debug + set -v + set -x echodebug "install_ubuntu_git_deps() entry" @@ -3211,6 +3266,9 @@ install_ubuntu_git_deps() { } install_ubuntu_onedir_deps() { + # DGM debug + set -v + set -x if [ "$_START_DAEMONS" -eq $BS_FALSE ]; then echowarn "Not starting daemons on Debian based distributions is not working mostly because starting them is the default behaviour." @@ -3244,6 +3302,9 @@ install_ubuntu_onedir_deps() { } install_ubuntu_stable() { + # DGM debug + set -v + set -x __PACKAGES="" @@ -3271,6 +3332,9 @@ install_ubuntu_stable() { } install_ubuntu_git() { + # DGM debug + set -v + set -x # Activate virtualenv before install if [ "${_VIRTUALENV_DIR}" != "null" ]; then @@ -3301,6 +3365,9 @@ install_ubuntu_git() { } install_ubuntu_onedir() { + # DGM debug + set -v + set -x __PACKAGES="" @@ -3328,6 +3395,9 @@ install_ubuntu_onedir() { } install_ubuntu_stable_post() { + # DGM debug + set -v + set -x for fname in api master minion syndic; do # Skip salt-api since the service should be opt-in and not necessarily started on boot @@ -3356,6 +3426,9 @@ install_ubuntu_stable_post() { } install_ubuntu_git_post() { + # DGM debug + set -v + set -x for fname in api master minion syndic; do # Skip if not meant to be installed @@ -3401,6 +3474,9 @@ install_ubuntu_git_post() { } install_ubuntu_restart_daemons() { + # DGM debug + set -v + set -x [ "$_START_DAEMONS" -eq $BS_FALSE ] && return @@ -3443,6 +3519,9 @@ install_ubuntu_restart_daemons() { } install_ubuntu_check_services() { + # DGM debug + set -v + set -x for fname in api master minion syndic; do # Skip salt-api since the service should be opt-in and not necessarily started on boot @@ -4159,6 +4238,9 @@ install_fedora_onedir_post() { # CentOS Install Functions # __install_saltstack_rhel_onedir_repository() { + # DGM debug + set -v + set -x if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -ne 3 ]; then echoerror "Python version is no longer supported, only Python 3" @@ -4191,6 +4273,9 @@ __install_saltstack_rhel_onedir_repository() { } install_centos_stable_deps() { + # DGM debug + set -v + set -x if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then yum -y update || return 1 @@ -4229,6 +4314,9 @@ install_centos_stable_deps() { } install_centos_stable() { + # DGM debug + set -v + set -x __PACKAGES="" @@ -4263,6 +4351,9 @@ install_centos_stable() { } install_centos_stable_post() { + # DGM debug + set -v + set -x SYSTEMD_RELOAD=$BS_FALSE @@ -4295,6 +4386,9 @@ install_centos_stable_post() { } install_centos_git_deps() { + # DGM debug + set -v + set -x # First try stable deps then fall back to onedir deps if that one fails # if we're installing on a Red Hat based host that doesn't have the classic @@ -4341,6 +4435,9 @@ install_centos_git_deps() { } install_centos_git() { + # DGM debug + set -v + set -x if [ "${_PY_EXE}" != "" ]; then _PYEXE=${_PY_EXE} @@ -4357,6 +4454,9 @@ install_centos_git() { } install_centos_git_post() { + # DGM debug + set -v + set -x SYSTEMD_RELOAD=$BS_FALSE @@ -4399,6 +4499,9 @@ install_centos_git_post() { } install_centos_onedir_deps() { + # DGM debug + set -v + set -x if [ "$_UPGRADE_SYS" -eq "$BS_TRUE" ]; then yum -y update || return 1 @@ -4436,6 +4539,9 @@ install_centos_onedir_deps() { } install_centos_onedir() { + # DGM debug + set -v + set -x __PACKAGES="" @@ -4463,6 +4569,9 @@ install_centos_onedir() { } install_centos_onedir_post() { + # DGM debug + set -v + set -x SYSTEMD_RELOAD=$BS_FALSE @@ -4495,6 +4604,9 @@ install_centos_onedir_post() { } install_centos_restart_daemons() { + # DGM debug + set -v + set -x [ "$_START_DAEMONS" -eq $BS_FALSE ] && return @@ -4525,6 +4637,9 @@ install_centos_restart_daemons() { } install_centos_testing_deps() { + # DGM debug + set -v + set -x install_centos_stable_deps || return 1 return 0 @@ -4543,6 +4658,9 @@ install_centos_testing_post() { } install_centos_check_services() { + # DGM debug + set -v + set -x for fname in api master minion syndic; do # Skip salt-api since the service should be opt-in and not necessarily started on boot diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index 9a3749889..65294a1f1 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -3,6 +3,7 @@ import os import platform import subprocess +import tempfile import pytest @@ -24,14 +25,26 @@ def run_salt_call(cmd): Runs salt call command and returns a dictionary Accepts cmd as a list """ - cmd.append("--out=json") - result = subprocess.run(cmd, capture_output=True, text=True) - print( - f"DGM run_salt_call, cmd '{cmd}', result '{result}', stdout '{result.stdout}'", - flush=True, - ) - json_data = json.loads(result.stdout) - return json_data["local"] + tmpf = tempfile.NamedTemporaryFile(delete=False) + json_data = {"local": {}} + + try: + cmd.append("--out=json") + cmd.append(f"--log-file={tmpf.name}") + result = subprocess.run(cmd, capture_output=True, text=True) + print( + f"DGM run_salt_call, cmd '{cmd}', result '{result}', stdout '{result.stdout}'", + flush=True, + ) + if 0 == result.returncode: + json_data = json.loads(result.stdout) + else: + log.error(f"failed to produce output result, '{result}'") + + finally: + tmpf.close() + os.unlink(tmpf.name) + return json_data["local"] def test_ping(path): From c1e7783a847373f45417ddeff7f7670739ffe8c8 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 11 Nov 2024 15:28:31 -0700 Subject: [PATCH 073/173] trying sudo --- tests/integration/test_installation.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index 65294a1f1..acd970380 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -29,11 +29,16 @@ def run_salt_call(cmd): json_data = {"local": {}} try: - cmd.append("--out=json") - cmd.append(f"--log-file={tmpf.name}") - result = subprocess.run(cmd, capture_output=True, text=True) + if platform.system() == "Windows": + cmdl = cmd + else: + cmdl = ["sudo"] + cmdl.extend(cmd) + cmdl.append("--out=json") + cmdl.append(f"--log-file={tmpf.name}") + result = subprocess.run(cmdl, capture_output=True, text=True) print( - f"DGM run_salt_call, cmd '{cmd}', result '{result}', stdout '{result.stdout}'", + f"DGM run_salt_call, cmdl '{cmdl}', result '{result}', stdout '{result.stdout}'", flush=True, ) if 0 == result.returncode: From ba77751f992d59e653d712ece02b778be8deb7c0 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 11 Nov 2024 15:39:41 -0700 Subject: [PATCH 074/173] Adjusted test --- tests/integration/test_installation.py | 38 ++++++++++++++------------ 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index acd970380..ada3ddbbd 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -3,7 +3,6 @@ import os import platform import subprocess -import tempfile import pytest @@ -25,20 +24,12 @@ def run_salt_call(cmd): Runs salt call command and returns a dictionary Accepts cmd as a list """ - tmpf = tempfile.NamedTemporaryFile(delete=False) json_data = {"local": {}} - - try: - if platform.system() == "Windows": - cmdl = cmd - else: - cmdl = ["sudo"] - cmdl.extend(cmd) - cmdl.append("--out=json") - cmdl.append(f"--log-file={tmpf.name}") - result = subprocess.run(cmdl, capture_output=True, text=True) + if platform.system() == "Windows": + cmd.append("--out=json") + result = subprocess.run(cmd, capture_output=True, text=True) print( - f"DGM run_salt_call, cmdl '{cmdl}', result '{result}', stdout '{result.stdout}'", + f"DGM run_salt_call, cmd '{cmd}', result '{result}', stdout '{result.stdout}'", flush=True, ) if 0 == result.returncode: @@ -46,10 +37,23 @@ def run_salt_call(cmd): else: log.error(f"failed to produce output result, '{result}'") - finally: - tmpf.close() - os.unlink(tmpf.name) - return json_data["local"] + else: + try: + cmdl = ["sudo"] + cmdl.extend(cmd) + cmdl.append("--out=json") + result = subprocess.run(cmdl, capture_output=True, text=True) + print( + f"DGM run_salt_call, cmdl '{cmdl}', result '{result}', stdout '{result.stdout}'", + flush=True, + ) + if 0 == result.returncode: + json_data = json.loads(result.stdout) + else: + log.error(f"failed to produce output result, '{result}'") + + finally: + return json_data["local"] def test_ping(path): From 22c7173a22c6e6ab27084c5d24ce699f124ccf98 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 11 Nov 2024 15:44:47 -0700 Subject: [PATCH 075/173] Fixing test --- tests/integration/test_installation.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index ada3ddbbd..4d7f409d9 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -16,7 +16,8 @@ def path(): if salt_path not in os.environ["path"]: os.environ["path"] = f'{os.environ["path"]};{salt_path}' yield os.environ["path"] - yield "" + else: + yield "" def run_salt_call(cmd): From dbfd3b0f193bb61f57c4f55bfc435057045ffe51 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 11 Nov 2024 16:29:34 -0700 Subject: [PATCH 076/173] Updated testing --- .github/workflows/test-linux.yml | 6 ++++-- .github/workflows/test-macos.yml | 6 ++++-- tests/integration/test_installation.py | 28 ++++++++++++-------------- 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index dcfddc6e9..323404867 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -61,10 +61,12 @@ jobs: - name: Bootstrap Salt run: | + # sed 1st - becomes space, 2nd - becomes dot echo "matrix instance ,${{ matrix.instance }}," bt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /') - echo "bt parms ,$bt_parms," - sh -x ./bootstrap-salt.sh "$bt_parms" + bt_parms2=$(echo "$bt_parms" | sed 's/-/./') + echo "bt parms ,$bt_parms, bt_parms2 ,$bt_parms2," + sh -x ./bootstrap-salt.sh "$bt_parms2" - name: Test Bootstrap run: | diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index 171bd2282..4a7f2e3d6 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -53,10 +53,12 @@ jobs: - name: Bootstrap Salt run: | + # sed 1st - becomes space, 2nd - becomes dot echo "matrix instance ,${{ matrix.instance }}," bt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /') - echo "bt parms ,$bt_parms," - sudo sh -x ./bootstrap-salt.sh "$bt_parms" + bt_parms2=$(echo "$bt_parms" | sed 's/-/./') + echo "bt parms ,$bt_parms, bt_parms2 ,$bt_parms2," + sh -x ./bootstrap-salt.sh "$bt_parms2" - name: Test Bootstrap run: | diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index 4d7f409d9..727240120 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -39,22 +39,20 @@ def run_salt_call(cmd): log.error(f"failed to produce output result, '{result}'") else: - try: - cmdl = ["sudo"] - cmdl.extend(cmd) - cmdl.append("--out=json") - result = subprocess.run(cmdl, capture_output=True, text=True) - print( - f"DGM run_salt_call, cmdl '{cmdl}', result '{result}', stdout '{result.stdout}'", - flush=True, - ) - if 0 == result.returncode: - json_data = json.loads(result.stdout) - else: - log.error(f"failed to produce output result, '{result}'") + cmdl = ["sudo"] + cmdl.extend(cmd) + cmdl.append("--out=json") + result = subprocess.run(cmdl, capture_output=True, text=True) + print( + f"DGM run_salt_call, cmdl '{cmdl}', result '{result}', stdout '{result.stdout}'", + flush=True, + ) + if 0 == result.returncode: + json_data = json.loads(result.stdout) + else: + log.error(f"failed to produce output result, '{result}'") - finally: - return json_data["local"] + return json_data["local"] def test_ping(path): From a50227f2fe3f74ddd85281595893297eab03fe7e Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 11 Nov 2024 16:49:08 -0700 Subject: [PATCH 077/173] Running script as sudo --- .github/workflows/test-linux.yml | 2 +- .github/workflows/test-macos.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 323404867..ff55f9e63 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -66,7 +66,7 @@ jobs: bt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /') bt_parms2=$(echo "$bt_parms" | sed 's/-/./') echo "bt parms ,$bt_parms, bt_parms2 ,$bt_parms2," - sh -x ./bootstrap-salt.sh "$bt_parms2" + sudo sh -x ./bootstrap-salt.sh "$bt_parms2" - name: Test Bootstrap run: | diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index 4a7f2e3d6..57e96b90f 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -58,7 +58,7 @@ jobs: bt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /') bt_parms2=$(echo "$bt_parms" | sed 's/-/./') echo "bt parms ,$bt_parms, bt_parms2 ,$bt_parms2," - sh -x ./bootstrap-salt.sh "$bt_parms2" + sudo sh -x ./bootstrap-salt.sh "$bt_parms2" - name: Test Bootstrap run: | From 736e8c0bc32bcdca5cb8afde967f6f9b5caee7aa Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 12 Nov 2024 09:23:08 -0700 Subject: [PATCH 078/173] Linux using Ubuntu runner till systemd support on containers --- .github/workflows/test-linux.yml | 6 ++++-- bootstrap-salt.sh | 25 +++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index ff55f9e63..3b325fcba 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -32,8 +32,10 @@ jobs: Test: name: ${{ matrix.instance }} runs-on: ubuntu-latest - container: - image: ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} + ## DGM use runner until have working systemd solution on containers + ## DGM runs-on: ubuntu-latest + ## DGM container: + ## DGM image: ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} timeout-minutes: ${{ inputs.timeout }} strategy: diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 958e3030f..d5a47e3ef 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -2992,6 +2992,8 @@ __enable_universe_repository() { set -v set -x + echodebug "__enable_universe_repository() entry" + if [ "$(grep -R universe /etc/apt/sources.list /etc/apt/sources.list.d/ | grep -v '#')" != "" ]; then # The universe repository is already enabled return 0 @@ -3072,6 +3074,7 @@ __install_saltstack_ubuntu_onedir_repository() { set -x echodebug "__install_saltstack_ubuntu_onedir_repository() entry" + # Workaround for latest non-LTS Ubuntu if { [ "$DISTRO_MAJOR_VERSION" -eq 20 ] && [ "$DISTRO_MINOR_VERSION" -eq 10 ]; } || \ { [ "$DISTRO_MAJOR_VERSION" -eq 22 ] && [ "$DISTRO_MINOR_VERSION" -eq 10 ]; } || \ @@ -3306,6 +3309,8 @@ install_ubuntu_stable() { set -v set -x + __wait_for_apt apt-get update || return 1 + __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then @@ -3369,6 +3374,8 @@ install_ubuntu_onedir() { set -v set -x + __wait_for_apt apt-get update || return 1 + __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then @@ -3735,6 +3742,11 @@ install_debian_git_deps() { } install_debian_stable() { + # DGM debug + set -v + set -x + + __wait_for_apt apt-get update || return 1 __PACKAGES="" @@ -3774,6 +3786,9 @@ install_debian_12_git_deps() { } install_debian_git() { + # DGM debug + set -v + set -x if [ -n "$_PY_EXE" ]; then _PYEXE=${_PY_EXE} @@ -3814,6 +3829,11 @@ install_debian_12_git() { } install_debian_onedir() { + # DGM debug + set -v + set -x + + __wait_for_apt apt-get update || return 1 __PACKAGES="" @@ -3970,6 +3990,7 @@ __install_saltstack_fedora_onedir_repository() { dnf config-manager --set-enabled salt-repo-latest fi dnf clean expire-cache || return 1 + dnf makecache || return 1 elif [ "$ONEDIR_REV" != "latest" ]; then echowarn "salt.repo already exists, ignoring salt version argument." @@ -4264,6 +4285,7 @@ __install_saltstack_rhel_onedir_repository() { dnf config-manager --set-enabled salt-repo-latest fi dnf clean expire-cache || return 1 + dnf makecache || return 1 elif [ "$ONEDIR_REV" != "latest" ]; then echowarn "salt.repo already exists, ignoring salt version argument." echowarn "Use -F (forced overwrite) to install $ONEDIR_REV." @@ -5672,6 +5694,7 @@ install_amazon_linux_ami_2_deps() { dnf config-manager --set-enabled salt-repo-latest fi dnf clean expire-cache || return 1 + dnf makecache || return 1 fi fi @@ -5715,6 +5738,7 @@ install_amazon_linux_ami_2_onedir_deps() { dnf config-manager --set-enabled salt-repo-latest fi dnf clean expire-cache || return 1 + dnf makecache || return 1 fi fi @@ -5842,6 +5866,7 @@ install_amazon_linux_ami_2023_onedir_deps() { dnf config-manager --set-enabled salt-repo-latest fi dnf clean expire-cache || return 1 + dnf makecache || return 1 fi fi From f52fea9e1d22c71d64feaac1f5581ccf4934c6f1 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 12 Nov 2024 10:59:50 -0700 Subject: [PATCH 079/173] Added debugging and setting test version as github environment variable --- .github/workflows/test-linux.yml | 13 +++++++++++++ .github/workflows/test-macos.yml | 13 +++++++++++++ bootstrap-salt.sh | 4 +++- tests/conftest.py | 20 +++++++++++++++++++- tests/integration/test_installation.py | 5 +++++ 5 files changed, 53 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 3b325fcba..fd4da534a 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -61,6 +61,19 @@ jobs: echo "DGM plain pip for distro-slug ${{ inputs.distro-slug }}" python3 -m pip install -r tests/requirements.txt + - name: Get Version + run: | + # We need to get the version here and make it an environment variable + # It is used to install via bootstrap and in the test + # The version is in the instance name + # sed 1st - becomes space, 2nd - becomes dot + echo "matrix instance ,${{ matrix.instance }}," + vt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /') + vt_parms2=$(echo "$vt_parms" | sed 's/-/./') + vt_parm_ver=$(echo "$vt_parms2" | awk -F ' ' '{print $2}') + echo "bt parms ,$vt_parms, vt_parms2 ,$vt_parms2, vt_parms_ver ,$vt_parm_ver," + echo ""SaltVersion=$vt_parm_ver" >> $GITHUB_ENV + - name: Bootstrap Salt run: | # sed 1st - becomes space, 2nd - becomes dot diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index 57e96b90f..052ff9249 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -51,6 +51,19 @@ jobs: run: | python3 -m pip install -r tests/requirements.txt + - name: Get Version + run: | + # We need to get the version here and make it an environment variable + # It is used to install via bootstrap and in the test + # The version is in the instance name + # sed 1st - becomes space, 2nd - becomes dot + echo "matrix instance ,${{ matrix.instance }}," + vt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /') + vt_parms2=$(echo "$vt_parms" | sed 's/-/./') + vt_parm_ver=$(echo "$vt_parms2" | awk -F ' ' '{print $2}') + echo "bt parms ,$vt_parms, vt_parms2 ,$vt_parms2, vt_parms_ver ,$vt_parm_ver," + echo ""SaltVersion=$vt_parm_ver" >> $GITHUB_ENV + - name: Bootstrap Salt run: | # sed 1st - becomes space, 2nd - becomes dot diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index d5a47e3ef..3006848b0 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -618,7 +618,9 @@ __check_services_systemd_functional # Define installation type if [ "$#" -gt 0 ];then __check_unparsed_options "$*" - ITYPE=$1 + ## DGM ITYPE=$1 + # get first arguement + ITYPE=$(echo $1 | awk -F ' ' '{print $1}') shift fi diff --git a/tests/conftest.py b/tests/conftest.py index 154b257c0..2186d066f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -19,6 +19,11 @@ def target_salt_version(): target_salt = os.environ.get("SaltVersion", "") + print( + f"DGM conftest target_salt_version, target_salt '{target_salt}', os.environ '{os.environ}'", + flush=True, + ) + html_response = requests.get(API_URL) content = json.loads(html_response.text) folders = content["children"] @@ -32,10 +37,23 @@ def target_salt_version(): versions[maj_version] = version versions["latest"] = version + print( + f"DGM conftest target_salt_version, target_salt '{target_salt}', versions '{versions}'", + flush=True, + ) + if target_salt.startswith("v"): target_salt = target_salt[1:] if target_salt not in versions: pytest.skip(f"Invalid testing version: {target_salt}") - if target_salt in ("default", "latest", "master", "nightly"): + if target_salt in ( + "default", + "latest", + "master", + "nightly", + "stable", + "onedir", + "git", + ): pytest.skip("Don't have a specific salt version to test against") return versions[target_salt] diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index 727240120..714b8e166 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -74,5 +74,10 @@ def test_target_salt_version(path, target_salt_version): pytest.skip(f"No target version specified") cmd = ["salt-call", "--local", "grains.item", "saltversion", "--timeout=120"] result = run_salt_call(cmd) + dgm_saltversion = result["saltversion"] + print( + f"DGM test_target_salt_version, target_salt_version '{target_salt_version}', result saltversion '{dgm_saltversion }', result '{result}'", + flush=True, + ) # Returns: {'saltversion': '3006.9+217.g53cfa53040'} assert result["saltversion"] == target_salt_version From ce84347128685c297a8d1782da1d3c4b02af3205 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 12 Nov 2024 11:06:17 -0700 Subject: [PATCH 080/173] Fixed typo --- .github/workflows/test-linux.yml | 2 +- .github/workflows/test-macos.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index fd4da534a..d9165a9ba 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -72,7 +72,7 @@ jobs: vt_parms2=$(echo "$vt_parms" | sed 's/-/./') vt_parm_ver=$(echo "$vt_parms2" | awk -F ' ' '{print $2}') echo "bt parms ,$vt_parms, vt_parms2 ,$vt_parms2, vt_parms_ver ,$vt_parm_ver," - echo ""SaltVersion=$vt_parm_ver" >> $GITHUB_ENV + echo "SaltVersion=$vt_parm_ver" >> $GITHUB_ENV - name: Bootstrap Salt run: | diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index 052ff9249..45519f4a1 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -62,7 +62,7 @@ jobs: vt_parms2=$(echo "$vt_parms" | sed 's/-/./') vt_parm_ver=$(echo "$vt_parms2" | awk -F ' ' '{print $2}') echo "bt parms ,$vt_parms, vt_parms2 ,$vt_parms2, vt_parms_ver ,$vt_parm_ver," - echo ""SaltVersion=$vt_parm_ver" >> $GITHUB_ENV + echo "SaltVersion=$vt_parm_ver" >> $GITHUB_ENV - name: Bootstrap Salt run: | From 5a704ec503ce4452226db6cd09405e3b3afa92e9 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 12 Nov 2024 12:22:41 -0700 Subject: [PATCH 081/173] adjusting for command line arguments --- .github/workflows/test-linux.yml | 9 ++++++--- .github/workflows/test-macos.yml | 9 ++++++--- bootstrap-salt.sh | 4 +--- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index d9165a9ba..821305462 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -71,7 +71,7 @@ jobs: vt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /') vt_parms2=$(echo "$vt_parms" | sed 's/-/./') vt_parm_ver=$(echo "$vt_parms2" | awk -F ' ' '{print $2}') - echo "bt parms ,$vt_parms, vt_parms2 ,$vt_parms2, vt_parms_ver ,$vt_parm_ver," + echo "vt parms ,$vt_parms, vt_parms2 ,$vt_parms2, vt_parms_ver ,$vt_parm_ver," echo "SaltVersion=$vt_parm_ver" >> $GITHUB_ENV - name: Bootstrap Salt @@ -80,8 +80,11 @@ jobs: echo "matrix instance ,${{ matrix.instance }}," bt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /') bt_parms2=$(echo "$bt_parms" | sed 's/-/./') - echo "bt parms ,$bt_parms, bt_parms2 ,$bt_parms2," - sudo sh -x ./bootstrap-salt.sh "$bt_parms2" + bt_arg1=$(echo "$bt_parms2" | awk -F ' ' '{print $1}') + bt_arg2=$(echo "$bt_parms2" | awk -F ' ' '{print $2}') + echo "bt parms ,$bt_parms, bt_parms2 ,$bt_parms2, bt_arg1 ,$bt_arg1, bt_arg2 ,$bt_arg2," + ## sudo sh -x ./bootstrap-salt.sh "$bt_parms2" + sudo sh -x ./bootstrap-salt.sh "$bt_arg1" "$bt_arg2" - name: Test Bootstrap run: | diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index 45519f4a1..88f6d7aee 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -61,7 +61,7 @@ jobs: vt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /') vt_parms2=$(echo "$vt_parms" | sed 's/-/./') vt_parm_ver=$(echo "$vt_parms2" | awk -F ' ' '{print $2}') - echo "bt parms ,$vt_parms, vt_parms2 ,$vt_parms2, vt_parms_ver ,$vt_parm_ver," + echo "vt parms ,$vt_parms, vt_parms2 ,$vt_parms2, vt_parms_ver ,$vt_parm_ver," echo "SaltVersion=$vt_parm_ver" >> $GITHUB_ENV - name: Bootstrap Salt @@ -70,8 +70,11 @@ jobs: echo "matrix instance ,${{ matrix.instance }}," bt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /') bt_parms2=$(echo "$bt_parms" | sed 's/-/./') - echo "bt parms ,$bt_parms, bt_parms2 ,$bt_parms2," - sudo sh -x ./bootstrap-salt.sh "$bt_parms2" + bt_arg1=$(echo "$bt_parms2" | awk -F ' ' '{print $1}') + bt_arg2=$(echo "$bt_parms2" | awk -F ' ' '{print $2}') + echo "bt parms ,$bt_parms, bt_parms2 ,$bt_parms2, bt_arg1 ,$bt_arg1, bt_arg2 ,$bt_arg2," + ## sudo sh -x ./bootstrap-salt.sh "$bt_parms2" + sudo sh -x ./bootstrap-salt.sh "$bt_arg1" "$bt_arg2" - name: Test Bootstrap run: | diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 3006848b0..d5a47e3ef 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -618,9 +618,7 @@ __check_services_systemd_functional # Define installation type if [ "$#" -gt 0 ];then __check_unparsed_options "$*" - ## DGM ITYPE=$1 - # get first arguement - ITYPE=$(echo $1 | awk -F ' ' '{print $1}') + ITYPE=$1 shift fi From 0bbe715ad887f72e8138fa2cb230be95c9c81f48 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 12 Nov 2024 12:55:29 -0700 Subject: [PATCH 082/173] Fix MacOS grep -E pattern to allow for no '.x' versions --- .github/workflows/test-linux.yml | 3 ++- .github/workflows/test-macos.yml | 3 ++- bootstrap-salt.sh | 4 ++-- tests/conftest.py | 11 ----------- 4 files changed, 6 insertions(+), 15 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 821305462..e60258384 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -67,6 +67,7 @@ jobs: # It is used to install via bootstrap and in the test # The version is in the instance name # sed 1st - becomes space, 2nd - becomes dot + ## DGM needs cleanup echo "matrix instance ,${{ matrix.instance }}," vt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /') vt_parms2=$(echo "$vt_parms" | sed 's/-/./') @@ -77,13 +78,13 @@ jobs: - name: Bootstrap Salt run: | # sed 1st - becomes space, 2nd - becomes dot + ## DGM needs cleanup echo "matrix instance ,${{ matrix.instance }}," bt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /') bt_parms2=$(echo "$bt_parms" | sed 's/-/./') bt_arg1=$(echo "$bt_parms2" | awk -F ' ' '{print $1}') bt_arg2=$(echo "$bt_parms2" | awk -F ' ' '{print $2}') echo "bt parms ,$bt_parms, bt_parms2 ,$bt_parms2, bt_arg1 ,$bt_arg1, bt_arg2 ,$bt_arg2," - ## sudo sh -x ./bootstrap-salt.sh "$bt_parms2" sudo sh -x ./bootstrap-salt.sh "$bt_arg1" "$bt_arg2" - name: Test Bootstrap diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index 88f6d7aee..7aa78c772 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -57,6 +57,7 @@ jobs: # It is used to install via bootstrap and in the test # The version is in the instance name # sed 1st - becomes space, 2nd - becomes dot + ## DGM needs cleanup echo "matrix instance ,${{ matrix.instance }}," vt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /') vt_parms2=$(echo "$vt_parms" | sed 's/-/./') @@ -67,13 +68,13 @@ jobs: - name: Bootstrap Salt run: | # sed 1st - becomes space, 2nd - becomes dot + ## DGM needs cleanup echo "matrix instance ,${{ matrix.instance }}," bt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /') bt_parms2=$(echo "$bt_parms" | sed 's/-/./') bt_arg1=$(echo "$bt_parms2" | awk -F ' ' '{print $1}') bt_arg2=$(echo "$bt_parms2" | awk -F ' ' '{print $2}') echo "bt parms ,$bt_parms, bt_parms2 ,$bt_parms2, bt_arg1 ,$bt_arg1, bt_arg2 ,$bt_arg2," - ## sudo sh -x ./bootstrap-salt.sh "$bt_parms2" sudo sh -x ./bootstrap-salt.sh "$bt_arg1" "$bt_arg2" - name: Test Bootstrap diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index d5a47e3ef..1946058f5 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -665,7 +665,7 @@ elif [ "$ITYPE" = "stable" ]; then _ONEDIR_REV="$1" ITYPE="onedir" shift - elif [ "$(echo "$1" | grep -E '^([3-9][0-5]{2}[5-9](\.[0-9]*)?)')" != "" ]; then + elif [ "$(echo "$1" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then STABLE_REV="$1" ONEDIR_REV="$1" _ONEDIR_REV="$1" @@ -7453,7 +7453,7 @@ __macosx_get_packagesite_onedir() { SALT_MACOS_PKGDIR_URL="https://${_REPO_URL}/${_ONEDIR_TYPE}/macos" if [ "$(echo "$_ONEDIR_REV" | grep -E '^(latest)$')" != "" ]; then __macosx_get_packagesite_onedir_latest - elif [ "$(echo "$_ONEDIR_REV" | grep -E '^([3-9][0-9]{3}(\.[0-9]*))')" != "" ]; then + elif [ "$(echo "$_ONEDIR_REV" | grep -E '^([3-9][0-9]{3}(\.[0-9]*)?)')" != "" ]; then _PKG_VERSION=$_ONEDIR_REV else __macosx_get_packagesite_onedir_latest diff --git a/tests/conftest.py b/tests/conftest.py index 2186d066f..5d5702c12 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,12 +18,6 @@ def target_python_version(): def target_salt_version(): target_salt = os.environ.get("SaltVersion", "") - - print( - f"DGM conftest target_salt_version, target_salt '{target_salt}', os.environ '{os.environ}'", - flush=True, - ) - html_response = requests.get(API_URL) content = json.loads(html_response.text) folders = content["children"] @@ -37,11 +31,6 @@ def target_salt_version(): versions[maj_version] = version versions["latest"] = version - print( - f"DGM conftest target_salt_version, target_salt '{target_salt}', versions '{versions}'", - flush=True, - ) - if target_salt.startswith("v"): target_salt = target_salt[1:] if target_salt not in versions: From d0d9fa7ed1753535bd394fa0e41216a42604d704 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 12 Nov 2024 13:22:35 -0700 Subject: [PATCH 083/173] Allow for minor version on MacOS and Debian family --- bootstrap-salt.sh | 106 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 79 insertions(+), 27 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 1946058f5..83b166dad 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -3054,14 +3054,23 @@ __install_saltstack_ubuntu_repository() { if [ "$STABLE_REV" != "latest" ]; then # latest is default - STABLE_REV_MAJOR=$(echo "$STABLE_REV" | cut -d '.' -f 1) - if [ "$STABLE_REV_MAJOR" -eq "3006" ]; then + ## DGM STABLE_REV_MAJOR=$(echo "$STABLE_REV" | cut -d '.' -f 1) + ## DGM if [ "$STABLE_REV_MAJOR" -eq "3006" ]; then + ## DGM echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 + ## DGM echo "Pin: version 3006.*" >> /etc/apt/preferences.d/salt-pin-1001 + ## DGM echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 + ## DGM elif [ "$STABLE_REV_MAJOR" -eq "3007" ]; then + ## DGM echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 + ## DGM echo "Pin: version 3007.*" >> /etc/apt/preferences.d/salt-pin-1001 + ## DGM echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 + ## DGM fi + if [ "$(echo "$STABLE_REV" | grep -E '^(3006|3007)$')" != "" ]; then echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - echo "Pin: version 3006.*" >> /etc/apt/preferences.d/salt-pin-1001 + echo "Pin: version $STABLE_REV.*" >> /etc/apt/preferences.d/salt-pin-1001 echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 - elif [ "$STABLE_REV_MAJOR" -eq "3007" ]; then + elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - echo "Pin: version 3007.*" >> /etc/apt/preferences.d/salt-pin-1001 + echo "Pin: version $STABLE_REV" >> /etc/apt/preferences.d/salt-pin-1001 echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 fi fi @@ -3110,14 +3119,23 @@ __install_saltstack_ubuntu_onedir_repository() { if [ "$ONEDIR_REV" != "latest" ]; then # latest is default - ONEDIR_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) - if [ "$ONEDIR_REV_MAJOR" -eq "3006" ]; then + ## DGM ONEDIR_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) + ## DGM if [ "$ONEDIR_REV_MAJOR" -eq "3006" ]; then + ## DGM echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 + ## DGM echo "Pin: version 3006.*" >> /etc/apt/preferences.d/salt-pin-1001 + ## DGM echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 + ## DGM elif [ "$ONEDIR_REV_MAJOR" -eq "3007" ]; then + ## DGM echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 + ## DGM echo "Pin: version 3007.*" >> /etc/apt/preferences.d/salt-pin-1001 + ## DGM echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 + ## DGM fi + if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - echo "Pin: version 3006.*" >> /etc/apt/preferences.d/salt-pin-1001 + echo "Pin: version $ONEDIR_REV.*" >> /etc/apt/preferences.d/salt-pin-1001 echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 - elif [ "$ONEDIR_REV_MAJOR" -eq "3007" ]; then + elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - echo "Pin: version 3007.*" >> /etc/apt/preferences.d/salt-pin-1001 + echo "Pin: version $ONEDIR_REV" >> /etc/apt/preferences.d/salt-pin-1001 echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 fi fi @@ -3593,14 +3611,23 @@ __install_saltstack_debian_repository() { if [ "$STABLE_REV" != "latest" ]; then # latest is default - STABLE_REV_MAJOR=$(echo "$STABLE_REV" | cut -d '.' -f 1) - if [ "$STABLE_REV_MAJOR" -eq "3006" ]; then + ## DGM STABLE_REV_MAJOR=$(echo "$STABLE_REV" | cut -d '.' -f 1) + ## DGM if [ "$STABLE_REV_MAJOR" -eq "3006" ]; then + ## DGM echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 + ## DGM echo "Pin: version 3006.*" >> /etc/apt/preferences.d/salt-pin-1001 + ## DGM echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 + ## DGM elif [ "$STABLE_REV_MAJOR" -eq "3007" ]; then + ## DGM echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 + ## DGM echo "Pin: version 3007.*" >> /etc/apt/preferences.d/salt-pin-1001 + ## DGM echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 + ## DGM fi + if [ "$(echo "$STABLE_REV" | grep -E '^(3006|3007)$')" != "" ]; then echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - echo "Pin: version 3006.*" >> /etc/apt/preferences.d/salt-pin-1001 + echo "Pin: version $STABLE_REV.*" >> /etc/apt/preferences.d/salt-pin-1001 echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 - elif [ "$STABLE_REV_MAJOR" -eq "3007" ]; then + elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - echo "Pin: version 3007.*" >> /etc/apt/preferences.d/salt-pin-1001 + echo "Pin: version $STABLE_REV" >> /etc/apt/preferences.d/salt-pin-1001 echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 fi fi @@ -3637,14 +3664,23 @@ __install_saltstack_debian_onedir_repository() { if [ "$ONEDIR_REV" != "latest" ]; then # latest is default - ONEDIR_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) - if [ "$ONEDIR_REV_MAJOR" -eq "3006" ]; then + ## DGM ONEDIR_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) + ## DGM if [ "$ONEDIR_REV_MAJOR" -eq "3006" ]; then + ## DGM echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 + ## DGM echo "Pin: version 3006.*" >> /etc/apt/preferences.d/salt-pin-1001 + ## DGM echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 + ## DGM elif [ "$ONEDIR_REV_MAJOR" -eq "3007" ]; then + ## DGM echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 + ## DGM echo "Pin: version 3007.*" >> /etc/apt/preferences.d/salt-pin-1001 + ## DGM echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 + ## DGM fi + if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - echo "Pin: version 3006.*" >> /etc/apt/preferences.d/salt-pin-1001 + echo "Pin: version $ONEDIR_REV.*" >> /etc/apt/preferences.d/salt-pin-1001 echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 - elif [ "$ONEDIR_REV_MAJOR" -eq "3007" ]; then + elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - echo "Pin: version 3007.*" >> /etc/apt/preferences.d/salt-pin-1001 + echo "Pin: version $ONEDIR_REV" >> /etc/apt/preferences.d/salt-pin-1001 echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 fi fi @@ -7416,19 +7452,28 @@ daemons_running_voidlinux() { #--- FUNCTION ------------------------------------------------------------------------------------------------------- # NAME: __macosx_get_packagesite_onedir_latest -# DESCRIPTION: Set _PKG_VERSION to the latest for MacOS +# DESCRIPTION: Set _PKG_VERSION to the latest for MacOS or latest for major version input #---------------------------------------------------------------------------------------------------------------------- __macosx_get_packagesite_onedir_latest() { + # DGM debug + set -v + set -x - echodebug "Find latest MacOS release from repository" + echodebug "Find latest MacOS release from repositoryi, input major version $1" # get dir listing from url, sort and pick highest macos_versions_tmpdir=$(mktemp -d) curr_pwd=$(pwd) cd ${macos_versions_tmpdir} || return 1 wget -r -np -nH --exclude-directories=onedir,relenv,windows -x -l 1 "$SALT_MACOS_PKGDIR_URL/" - # shellcheck disable=SC2010 - _PKG_VERSION=$(ls artifactory/saltproject-generic/macos/ | grep -v 'index.html' | sort -V -u | tail -n 1) + if [ -n "$1" ]; then + MAJOR_VER="$1" + # shellcheck disable=SC2010 + _PKG_VERSION=$(ls artifactory/saltproject-generic/macos/ | grep -v 'index.html' | sort -V -u | grep -E "$MAJOR_VER" | tail -n 1) + else + # shellcheck disable=SC2010 + _PKG_VERSION=$(ls artifactory/saltproject-generic/macos/ | grep -v 'index.html' | sort -V -u | tail -n 1) + fi cd ${curr_pwd} || return "${_PKG_VERSION}" rm -fR ${macos_versions_tmpdir} @@ -7438,6 +7483,9 @@ __macosx_get_packagesite_onedir_latest() { __macosx_get_packagesite_onedir() { + # DGM debug + set -v + set -x echodebug "Get package site for onedir from repository" @@ -7452,11 +7500,15 @@ __macosx_get_packagesite_onedir() { _ONEDIR_TYPE="saltproject-generic" SALT_MACOS_PKGDIR_URL="https://${_REPO_URL}/${_ONEDIR_TYPE}/macos" if [ "$(echo "$_ONEDIR_REV" | grep -E '^(latest)$')" != "" ]; then - __macosx_get_packagesite_onedir_latest + __macosx_get_packagesite_onedir_latest + elif [ "$(echo "$_ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # need to get latest for major version + __macosx_get_packagesite_onedir_latest "$_ONEDIR_REV" elif [ "$(echo "$_ONEDIR_REV" | grep -E '^([3-9][0-9]{3}(\.[0-9]*)?)')" != "" ]; then - _PKG_VERSION=$_ONEDIR_REV + _PKG_VERSION=$_ONEDIR_REV else - __macosx_get_packagesite_onedir_latest + # default to getting latest + __macosx_get_packagesite_onedir_latest fi PKG="salt-${_PKG_VERSION}-py3-${DARWIN_ARCH}.pkg" From 447e01db26411f0046d3fa434859395794339aa6 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 12 Nov 2024 14:20:35 -0700 Subject: [PATCH 084/173] Allow for specifying minor version of Salt --- bootstrap-salt.sh | 225 ++++++++++++++++++++++++++++++++++++---------- 1 file changed, 178 insertions(+), 47 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 83b166dad..264488fa0 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -3329,23 +3329,33 @@ install_ubuntu_stable() { __wait_for_apt apt-get update || return 1 + if [ "$(echo "$STABLE_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # Major version Salt, config and repo already setup + MINOR_VER_STRG="" + elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # Minor version Salt, need to add specific minor version + MINOR_VER_STRG="-$STABLE_REV" + else + MINOR_VER_STRG="" + fi + __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-cloud" + __PACKAGES="${__PACKAGES} salt-cloud$MINOR_VER_STRG" fi if [ "$_INSTALL_MASTER" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-master" + __PACKAGES="${__PACKAGES} salt-master$MINOR_VER_STRG" fi if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-minion" + __PACKAGES="${__PACKAGES} salt-minion$MINOR_VER_STRG" fi if [ "$_INSTALL_SYNDIC" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-syndic" + __PACKAGES="${__PACKAGES} salt-syndic$MINOR_VER_STRG" fi if [ "$_INSTALL_SALT_API" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-api" + __PACKAGES="${__PACKAGES} salt-api$MINOR_VER_STRG" fi # shellcheck disable=SC2086 @@ -3394,23 +3404,33 @@ install_ubuntu_onedir() { __wait_for_apt apt-get update || return 1 + if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # Major version Salt, config and repo already setup + MINOR_VER_STRG="" + elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # Minor version Salt, need to add specific minor version + MINOR_VER_STRG="-$ONEDIR_REV" + else + MINOR_VER_STRG="" + fi + __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-cloud" + __PACKAGES="${__PACKAGES} salt-cloud$MINOR_VER_STRG" fi if [ "$_INSTALL_MASTER" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-master" + __PACKAGES="${__PACKAGES} salt-master$MINOR_VER_STRG" fi if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-minion" + __PACKAGES="${__PACKAGES} salt-minion$MINOR_VER_STRG" fi if [ "$_INSTALL_SYNDIC" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-syndic" + __PACKAGES="${__PACKAGES} salt-syndic$MINOR_VER_STRG" fi if [ "$_INSTALL_SALT_API" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-api" + __PACKAGES="${__PACKAGES} salt-api$MINOR_VER_STRG" fi # shellcheck disable=SC2086 @@ -3784,23 +3804,33 @@ install_debian_stable() { __wait_for_apt apt-get update || return 1 + if [ "$(echo "$STABLE_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # Major version Salt, config and repo already setup + MINOR_VER_STRG="" + elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # Minor version Salt, need to add specific minor version + MINOR_VER_STRG="-$STABLE_REV" + else + MINOR_VER_STRG="" + fi + __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-cloud" + __PACKAGES="${__PACKAGES} salt-cloud$MINOR_VER_STRG" fi if [ "$_INSTALL_MASTER" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-master" + __PACKAGES="${__PACKAGES} salt-master$MINOR_VER_STRG" fi if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-minion" + __PACKAGES="${__PACKAGES} salt-minion$MINOR_VER_STRG" fi if [ "$_INSTALL_SYNDIC" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-syndic" + __PACKAGES="${__PACKAGES} salt-syndic$MINOR_VER_STRG" fi if [ "$_INSTALL_SALT_API" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-api" + __PACKAGES="${__PACKAGES} salt-api$MINOR_VER_STRG" fi # shellcheck disable=SC2086 @@ -3871,23 +3901,33 @@ install_debian_onedir() { __wait_for_apt apt-get update || return 1 + if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # Major version Salt, config and repo already setup + MINOR_VER_STRG="" + elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # Minor version Salt, need to add specific minor version + MINOR_VER_STRG="-$ONEDIR_REV" + else + MINOR_VER_STRG="" + fi + __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-cloud" + __PACKAGES="${__PACKAGES} salt-cloud$MINOR_VER_STRG" fi if [ "$_INSTALL_MASTER" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-master" + __PACKAGES="${__PACKAGES} salt-master$MINOR_VER_STRG" fi if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-minion" + __PACKAGES="${__PACKAGES} salt-minion$MINOR_VER_STRG" fi if [ "$_INSTALL_SYNDIC" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-syndic" + __PACKAGES="${__PACKAGES} salt-syndic$MINOR_VER_STRG" fi if [ "$_INSTALL_SALT_API" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-api" + __PACKAGES="${__PACKAGES} salt-api$MINOR_VER_STRG" fi # shellcheck disable=SC2086 @@ -4238,24 +4278,33 @@ install_fedora_onedir() { STABLE_REV=$ONEDIR_REV #install_fedora_stable || return 1 + if [ "$(echo "$STABLE_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # Major version Salt, config and repo already setup + MINOR_VER_STRG="" + elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # Minor version Salt, need to add specific minor version + MINOR_VER_STRG="-$STABLE_REV" + else + MINOR_VER_STRG="" + fi __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-cloud" + __PACKAGES="${__PACKAGES} salt-cloud$MINOR_VER_STRG" fi if [ "$_INSTALL_MASTER" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-master" + __PACKAGES="${__PACKAGES} salt-master$MINOR_VER_STRG" fi if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-minion" + __PACKAGES="${__PACKAGES} salt-minion$MINOR_VER_STRG" fi if [ "$_INSTALL_SYNDIC" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-syndic" + __PACKAGES="${__PACKAGES} salt-syndic$MINOR_VER_STRG" fi if [ "$_INSTALL_SALT_API" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-api" + __PACKAGES="${__PACKAGES} salt-api$MINOR_VER_STRG" fi # shellcheck disable=SC2086 @@ -4376,23 +4425,33 @@ install_centos_stable() { set -v set -x + if [ "$(echo "$STABLE_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # Major version Salt, config and repo already setup + MINOR_VER_STRG="" + elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # Minor version Salt, need to add specific minor version + MINOR_VER_STRG="-$STABLE_REV" + else + MINOR_VER_STRG="" + fi + __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-cloud" + __PACKAGES="${__PACKAGES} salt-cloud$MINOR_VER_STRG" fi if [ "$_INSTALL_MASTER" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-master" + __PACKAGES="${__PACKAGES} salt-master$MINOR_VER_STRG" fi if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-minion" + __PACKAGES="${__PACKAGES} salt-minion$MINOR_VER_STRG" fi if [ "$_INSTALL_SYNDIC" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-syndic" + __PACKAGES="${__PACKAGES} salt-syndic$MINOR_VER_STRG" fi if [ "$_INSTALL_SALT_API" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-api" + __PACKAGES="${__PACKAGES} salt-api$MINOR_VER_STRG" fi # shellcheck disable=SC2086 @@ -4601,23 +4660,33 @@ install_centos_onedir() { set -v set -x + if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # Major version Salt, config and repo already setup + MINOR_VER_STRG="" + elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # Minor version Salt, need to add specific minor version + MINOR_VER_STRG="-$ONEDIR_REV" + else + MINOR_VER_STRG="" + fi + __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-cloud" + __PACKAGES="${__PACKAGES} salt-cloud$MINOR_VER_STRG" fi if [ "$_INSTALL_MASTER" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-master" + __PACKAGES="${__PACKAGES} salt-master$MINOR_VER_STRG" fi if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-minion" + __PACKAGES="${__PACKAGES} salt-minion$MINOR_VER_STRG" fi if [ "$_INSTALL_SYNDIC" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-syndic" + __PACKAGES="${__PACKAGES} salt-syndic$MINOR_VER_STRG" fi if [ "$_INSTALL_SALT_API" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-api" + __PACKAGES="${__PACKAGES} salt-api$MINOR_VER_STRG" fi # shellcheck disable=SC2086 @@ -6219,6 +6288,40 @@ install_arch_linux_onedir_post() { # Photon OS Install Functions # +#--- FUNCTION ------------------------------------------------------------------------------------------------------- +# NAME: __rpm_get_packagesite_onedir_latest +# DESCRIPTION: Set _GENERIC_PKG_VERSION to the latest for RPM or latest for major version input +#---------------------------------------------------------------------------------------------------------------------- +__get_packagesite_onedir_latest() { + # DGM debug + set -v + set -x + + echodebug "Find latest rpm release from repository" + + # get dir listing from url, sort and pick highest + generic_versions_tmpdir=$(mktemp -d) + curr_pwd=$(pwd) + cd ${generic_versions_tmpdir} || return 1 + + # leverage the windows directories since release Windows and Linux + wget -r -np -nH --exclude-directories=onedir,relenv,macos -x -l 1 "https://packages.broadcom.com/artifactory/saltproject-generic/" + if [ "$#" -gt 0 ] && [ -n "$1" ]; then + MAJOR_VER="$1" + # shellcheck disable=SC2010 + _GENERIC_PKG_VERSION=$(ls artifactory/saltproject-generic/windows/ | grep -v 'index.html' | sort -V -u | grep -E "$MAJOR_VER" | tail -n 1) + else + # shellcheck disable=SC2010 + _GENERIC_PKG_VERSION=$(ls artifactory/saltproject-generic/windows/ | grep -v 'index.html' | sort -V -u | tail -n 1) + fi + cd ${curr_pwd} || return "${_GENERIC_PKG_VERSION}" + rm -fR ${generic_versions_tmpdir} + + echodebug "latest rpm release from repository found ${_GENERIC_PKG_VERSION}" + +} + + __install_saltstack_photon_onedir_repository() { echodebug "__install_saltstack_photon_onedir_repository() entry" @@ -6525,27 +6628,45 @@ install_photon_onedir_deps() { install_photon_onedir() { + # DGM debug + set -v + set -x + echodebug "install_photon_onedir() entry" STABLE_REV=$ONEDIR_REV + _GENERIC_PKG_VERSION="" + + if [ "$(echo "$STABLE_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # Major version Salt, config and repo already setup + __get_packagesite_onedir_latest "$STABLE_REV" + MINOR_VER_STRG="$_GENERIC_PKG_VERSION" + elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # Minor version Salt, need to add specific minor version + MINOR_VER_STRG="-$STABLE_REV" + else + # default to latest version Salt, config and repo already setup + __get_packagesite_onedir_latest + MINOR_VER_STRG="$_GENERIC_PKG_VERSION" + fi __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-cloud" + __PACKAGES="${__PACKAGES} salt-cloud$MINOR_VER_STRG" fi if [ "$_INSTALL_MASTER" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-master" + __PACKAGES="${__PACKAGES} salt-master$MINOR_VER_STRG" fi if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-minion" + __PACKAGES="${__PACKAGES} salt-minion$MINOR_VER_STRG" fi if [ "$_INSTALL_SYNDIC" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-syndic" + __PACKAGES="${__PACKAGES} salt-syndic$MINOR_VER_STRG" fi if [ "$_INSTALL_SALT_API" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-api" + __PACKAGES="${__PACKAGES} salt-api$MINOR_VER_STRG" fi # shellcheck disable=SC2086 @@ -6729,23 +6850,33 @@ install_opensuse_onedir_deps() { } install_opensuse_stable() { + if [ "$(echo "$STABLE_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # Major version Salt, config and repo already setup + MINOR_VER_STRG="" + elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # Minor version Salt, need to add specific minor version + MINOR_VER_STRG="-$STABLE_REV" + else + MINOR_VER_STRG="" + fi + __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-cloud" + __PACKAGES="${__PACKAGES} salt-cloud$MINOR_VER_STRG" fi if [ "$_INSTALL_MASTER" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-master" + __PACKAGES="${__PACKAGES} salt-master$MINOR_VER_STRG" fi if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-minion" + __PACKAGES="${__PACKAGES} salt-minion$MINOR_VER_STRG" fi if [ "$_INSTALL_SYNDIC" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-syndic" + __PACKAGES="${__PACKAGES} salt-syndic$MINOR_VER_STRG" fi if [ "$_INSTALL_SALT_API" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-api" + __PACKAGES="${__PACKAGES} salt-api$MINOR_VER_STRG" fi # shellcheck disable=SC2086 @@ -7459,14 +7590,14 @@ __macosx_get_packagesite_onedir_latest() { set -v set -x - echodebug "Find latest MacOS release from repositoryi, input major version $1" + echodebug "Find latest MacOS release from repository" # get dir listing from url, sort and pick highest macos_versions_tmpdir=$(mktemp -d) curr_pwd=$(pwd) cd ${macos_versions_tmpdir} || return 1 wget -r -np -nH --exclude-directories=onedir,relenv,windows -x -l 1 "$SALT_MACOS_PKGDIR_URL/" - if [ -n "$1" ]; then + if [ "$#" -gt 0 ] && [ -n "$1" ]; then MAJOR_VER="$1" # shellcheck disable=SC2010 _PKG_VERSION=$(ls artifactory/saltproject-generic/macos/ | grep -v 'index.html' | sort -V -u | grep -E "$MAJOR_VER" | tail -n 1) From 90c19c02f7752d9600a19391a84a8cbd7121857e Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 12 Nov 2024 14:40:32 -0700 Subject: [PATCH 085/173] Fix typo --- bootstrap-salt.sh | 90 +++++++++++++---------------------------------- 1 file changed, 25 insertions(+), 65 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 264488fa0..6c29c982a 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -3329,33 +3329,23 @@ install_ubuntu_stable() { __wait_for_apt apt-get update || return 1 - if [ "$(echo "$STABLE_REV" | grep -E '^(3006|3007)$')" != "" ]; then - # Major version Salt, config and repo already setup - MINOR_VER_STRG="" - elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then - # Minor version Salt, need to add specific minor version - MINOR_VER_STRG="-$STABLE_REV" - else - MINOR_VER_STRG="" - fi - __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-cloud$MINOR_VER_STRG" + __PACKAGES="${__PACKAGES} salt-cloud" fi if [ "$_INSTALL_MASTER" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-master$MINOR_VER_STRG" + __PACKAGES="${__PACKAGES} salt-master" fi if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-minion$MINOR_VER_STRG" + __PACKAGES="${__PACKAGES} salt-minion" fi if [ "$_INSTALL_SYNDIC" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-syndic$MINOR_VER_STRG" + __PACKAGES="${__PACKAGES} salt-syndic" fi if [ "$_INSTALL_SALT_API" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-api$MINOR_VER_STRG" + __PACKAGES="${__PACKAGES} salt-api" fi # shellcheck disable=SC2086 @@ -3404,33 +3394,23 @@ install_ubuntu_onedir() { __wait_for_apt apt-get update || return 1 - if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then - # Major version Salt, config and repo already setup - MINOR_VER_STRG="" - elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then - # Minor version Salt, need to add specific minor version - MINOR_VER_STRG="-$ONEDIR_REV" - else - MINOR_VER_STRG="" - fi - __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-cloud$MINOR_VER_STRG" + __PACKAGES="${__PACKAGES} salt-cloud" fi if [ "$_INSTALL_MASTER" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-master$MINOR_VER_STRG" + __PACKAGES="${__PACKAGES} salt-master" fi if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-minion$MINOR_VER_STRG" + __PACKAGES="${__PACKAGES} salt-minion" fi if [ "$_INSTALL_SYNDIC" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-syndic$MINOR_VER_STRG" + __PACKAGES="${__PACKAGES} salt-syndic" fi if [ "$_INSTALL_SALT_API" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-api$MINOR_VER_STRG" + __PACKAGES="${__PACKAGES} salt-api" fi # shellcheck disable=SC2086 @@ -3804,33 +3784,23 @@ install_debian_stable() { __wait_for_apt apt-get update || return 1 - if [ "$(echo "$STABLE_REV" | grep -E '^(3006|3007)$')" != "" ]; then - # Major version Salt, config and repo already setup - MINOR_VER_STRG="" - elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then - # Minor version Salt, need to add specific minor version - MINOR_VER_STRG="-$STABLE_REV" - else - MINOR_VER_STRG="" - fi - __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-cloud$MINOR_VER_STRG" + __PACKAGES="${__PACKAGES} salt-cloud" fi if [ "$_INSTALL_MASTER" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-master$MINOR_VER_STRG" + __PACKAGES="${__PACKAGES} salt-master" fi if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-minion$MINOR_VER_STRG" + __PACKAGES="${__PACKAGES} salt-minion" fi if [ "$_INSTALL_SYNDIC" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-syndic$MINOR_VER_STRG" + __PACKAGES="${__PACKAGES} salt-syndic" fi if [ "$_INSTALL_SALT_API" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-api$MINOR_VER_STRG" + __PACKAGES="${__PACKAGES} salt-api" fi # shellcheck disable=SC2086 @@ -3901,33 +3871,23 @@ install_debian_onedir() { __wait_for_apt apt-get update || return 1 - if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then - # Major version Salt, config and repo already setup - MINOR_VER_STRG="" - elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then - # Minor version Salt, need to add specific minor version - MINOR_VER_STRG="-$ONEDIR_REV" - else - MINOR_VER_STRG="" - fi - __PACKAGES="" if [ "$_INSTALL_CLOUD" -eq $BS_TRUE ];then - __PACKAGES="${__PACKAGES} salt-cloud$MINOR_VER_STRG" + __PACKAGES="${__PACKAGES} salt-cloud" fi if [ "$_INSTALL_MASTER" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-master$MINOR_VER_STRG" + __PACKAGES="${__PACKAGES} salt-master" fi if [ "$_INSTALL_MINION" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-minion$MINOR_VER_STRG" + __PACKAGES="${__PACKAGES} salt-minion" fi if [ "$_INSTALL_SYNDIC" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-syndic$MINOR_VER_STRG" + __PACKAGES="${__PACKAGES} salt-syndic" fi if [ "$_INSTALL_SALT_API" -eq $BS_TRUE ]; then - __PACKAGES="${__PACKAGES} salt-api$MINOR_VER_STRG" + __PACKAGES="${__PACKAGES} salt-api" fi # shellcheck disable=SC2086 @@ -4283,7 +4243,7 @@ install_fedora_onedir() { MINOR_VER_STRG="" elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then # Minor version Salt, need to add specific minor version - MINOR_VER_STRG="-$STABLE_REV" + MINOR_VER_STRG="=$STABLE_REV" else MINOR_VER_STRG="" fi @@ -4430,7 +4390,7 @@ install_centos_stable() { MINOR_VER_STRG="" elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then # Minor version Salt, need to add specific minor version - MINOR_VER_STRG="-$STABLE_REV" + MINOR_VER_STRG="=$STABLE_REV" else MINOR_VER_STRG="" fi @@ -4665,7 +4625,7 @@ install_centos_onedir() { MINOR_VER_STRG="" elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then # Minor version Salt, need to add specific minor version - MINOR_VER_STRG="-$ONEDIR_REV" + MINOR_VER_STRG="=$ONEDIR_REV" else MINOR_VER_STRG="" fi @@ -6643,7 +6603,7 @@ install_photon_onedir() { MINOR_VER_STRG="$_GENERIC_PKG_VERSION" elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then # Minor version Salt, need to add specific minor version - MINOR_VER_STRG="-$STABLE_REV" + MINOR_VER_STRG="=$STABLE_REV" else # default to latest version Salt, config and repo already setup __get_packagesite_onedir_latest @@ -6855,7 +6815,7 @@ install_opensuse_stable() { MINOR_VER_STRG="" elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then # Minor version Salt, need to add specific minor version - MINOR_VER_STRG="-$STABLE_REV" + MINOR_VER_STRG="=$STABLE_REV" else MINOR_VER_STRG="" fi From 623b50e8262e3781d810620038e7d785f7f285c1 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 12 Nov 2024 15:01:01 -0700 Subject: [PATCH 086/173] Revert to using containers, cleaned up MacOS workflow --- .github/workflows/test-linux.yml | 22 +++++++++------------- .github/workflows/test-macos.yml | 20 ++++++-------------- 2 files changed, 15 insertions(+), 27 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index e60258384..6f9ed355b 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -32,10 +32,8 @@ jobs: Test: name: ${{ matrix.instance }} runs-on: ubuntu-latest - ## DGM use runner until have working systemd solution on containers - ## DGM runs-on: ubuntu-latest - ## DGM container: - ## DGM image: ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} + container: + image: ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} timeout-minutes: ${{ inputs.timeout }} strategy: @@ -69,10 +67,9 @@ jobs: # sed 1st - becomes space, 2nd - becomes dot ## DGM needs cleanup echo "matrix instance ,${{ matrix.instance }}," - vt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /') - vt_parms2=$(echo "$vt_parms" | sed 's/-/./') - vt_parm_ver=$(echo "$vt_parms2" | awk -F ' ' '{print $2}') - echo "vt parms ,$vt_parms, vt_parms2 ,$vt_parms2, vt_parms_ver ,$vt_parm_ver," + vt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./') + vt_parm_ver=$(echo "$vt_parms" | awk -F ' ' '{print $2}') + echo "vt parms ,$vt_parms, vt_parms_ver ,$vt_parm_ver," echo "SaltVersion=$vt_parm_ver" >> $GITHUB_ENV - name: Bootstrap Salt @@ -80,11 +77,10 @@ jobs: # sed 1st - becomes space, 2nd - becomes dot ## DGM needs cleanup echo "matrix instance ,${{ matrix.instance }}," - bt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /') - bt_parms2=$(echo "$bt_parms" | sed 's/-/./') - bt_arg1=$(echo "$bt_parms2" | awk -F ' ' '{print $1}') - bt_arg2=$(echo "$bt_parms2" | awk -F ' ' '{print $2}') - echo "bt parms ,$bt_parms, bt_parms2 ,$bt_parms2, bt_arg1 ,$bt_arg1, bt_arg2 ,$bt_arg2," + bt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./') + bt_arg1=$(echo "$bt_parms" | awk -F ' ' '{print $1}') + bt_arg2=$(echo "$bt_parms" | awk -F ' ' '{print $2}') + echo "bt parms ,$bt_parms, bt_arg1 ,$bt_arg1, bt_arg2 ,$bt_arg2," sudo sh -x ./bootstrap-salt.sh "$bt_arg1" "$bt_arg2" - name: Test Bootstrap diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index 7aa78c772..2779cf193 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -57,25 +57,17 @@ jobs: # It is used to install via bootstrap and in the test # The version is in the instance name # sed 1st - becomes space, 2nd - becomes dot - ## DGM needs cleanup - echo "matrix instance ,${{ matrix.instance }}," - vt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /') - vt_parms2=$(echo "$vt_parms" | sed 's/-/./') - vt_parm_ver=$(echo "$vt_parms2" | awk -F ' ' '{print $2}') - echo "vt parms ,$vt_parms, vt_parms2 ,$vt_parms2, vt_parms_ver ,$vt_parm_ver," + vt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./') + vt_parm_ver=$(echo "$vt_parms" | awk -F ' ' '{print $2}') echo "SaltVersion=$vt_parm_ver" >> $GITHUB_ENV - name: Bootstrap Salt run: | # sed 1st - becomes space, 2nd - becomes dot - ## DGM needs cleanup - echo "matrix instance ,${{ matrix.instance }}," - bt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /') - bt_parms2=$(echo "$bt_parms" | sed 's/-/./') - bt_arg1=$(echo "$bt_parms2" | awk -F ' ' '{print $1}') - bt_arg2=$(echo "$bt_parms2" | awk -F ' ' '{print $2}') - echo "bt parms ,$bt_parms, bt_parms2 ,$bt_parms2, bt_arg1 ,$bt_arg1, bt_arg2 ,$bt_arg2," - sudo sh -x ./bootstrap-salt.sh "$bt_arg1" "$bt_arg2" + bt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./') + bt_arg1=$(echo "$bt_parms" | awk -F ' ' '{print $1}') + bt_arg2=$(echo "$bt_parms" | awk -F ' ' '{print $2}') + sudo sh ./bootstrap-salt.sh "$bt_arg1" "$bt_arg2" - name: Test Bootstrap run: | From 8a5c2d8598b9426b831bce985cc602a22b9a4737 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 12 Nov 2024 15:52:52 -0700 Subject: [PATCH 087/173] Attempting using systemd container, only Ubuntu 22.04 is currently valid --- .github/workflows/test-linux.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 6f9ed355b..bf155254c 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -33,7 +33,8 @@ jobs: name: ${{ matrix.instance }} runs-on: ubuntu-latest container: - image: ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} + image: ghcr.io/saltstack/salt-ci-containers/testing:systemd-${{ inputs.container-slug }} + options: --privileged timeout-minutes: ${{ inputs.timeout }} strategy: From f33f58e6c4745fbc4e59399b470836a5bb51a6a2 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 12 Nov 2024 16:09:14 -0700 Subject: [PATCH 088/173] Container does not have sudo --- .github/workflows/test-linux.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index bf155254c..96b2a9c5f 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -82,7 +82,8 @@ jobs: bt_arg1=$(echo "$bt_parms" | awk -F ' ' '{print $1}') bt_arg2=$(echo "$bt_parms" | awk -F ' ' '{print $2}') echo "bt parms ,$bt_parms, bt_arg1 ,$bt_arg1, bt_arg2 ,$bt_arg2," - sudo sh -x ./bootstrap-salt.sh "$bt_arg1" "$bt_arg2" + ## DGM sudo sh -x ./bootstrap-salt.sh "$bt_arg1" "$bt_arg2" + sh -x ./bootstrap-salt.sh "$bt_arg1" "$bt_arg2" - name: Test Bootstrap run: | From 2a72acf0e4e405c780d0ac964861d55c34b3c76e Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 12 Nov 2024 16:28:17 -0700 Subject: [PATCH 089/173] Updated to install sudo, needed for testing --- bootstrap-salt.sh | 61 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 58 insertions(+), 3 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 6c29c982a..9044008dc 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -3185,6 +3185,9 @@ install_ubuntu_deps() { # Additionally install procps and pciutils which allows for Docker bootstraps. See 366#issuecomment-39666813 __PACKAGES="${__PACKAGES} procps pciutils" + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + ## include hwclock if not part of base OS (23.10 and up) if [ ! -f /usr/sbin/hwclock ]; then __PACKAGES="${__PACKAGES} util-linux-extra" @@ -3274,6 +3277,9 @@ install_ubuntu_git_deps() { __PACKAGES="${__PACKAGES} util-linux-extra" fi + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + # shellcheck disable=SC2086 __apt_get_install_noinput ${__PACKAGES} || return 1 @@ -3724,6 +3730,9 @@ install_debian_onedir_deps() { # YAML module is used for generating custom master/minion configs __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-yaml" + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + # shellcheck disable=SC2086 __apt_get_install_noinput ${__PACKAGES} || return 1 @@ -3765,6 +3774,9 @@ install_debian_git_deps() { __PACKAGES="python${PY_PKG_VER}-dev python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools gcc" echodebug "install_debian_git_deps() Installing ${__PACKAGES}" + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + # shellcheck disable=SC2086 __apt_get_install_noinput ${__PACKAGES} || return 1 @@ -4061,6 +4073,9 @@ install_fedora_deps() { echoinfo "Installing the following extra packages as requested: ${_EXTRA_PACKAGES}" fi + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + # shellcheck disable=SC2086 __dnf_install_noinput ${__PACKAGES} ${_EXTRA_PACKAGES} || return 1 @@ -4091,6 +4106,10 @@ install_fedora_git_deps() { __git_clone_and_checkout || return 1 __PACKAGES="python${PY_PKG_VER}-devel python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools gcc gcc-c++" + + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + # shellcheck disable=SC2086 __dnf_install_noinput ${__PACKAGES} || return 1 @@ -4220,6 +4239,9 @@ install_fedora_onedir_deps() { __PACKAGES="dnf-utils chkconfig procps-ng" + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -4368,6 +4390,9 @@ install_centos_stable_deps() { __PACKAGES="yum-utils chkconfig procps-ng findutils" + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -4414,6 +4439,9 @@ install_centos_stable() { __PACKAGES="${__PACKAGES} salt-api$MINOR_VER_STRG" fi + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -4498,6 +4526,9 @@ install_centos_git_deps() { __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-devel python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools gcc" + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -4603,6 +4634,9 @@ install_centos_onedir_deps() { __PACKAGES="yum-utils chkconfig procps-ng findutils" + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -5714,6 +5748,9 @@ install_amazon_linux_ami_2_git_deps() { __PACKAGES="python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools python${PY_PKG_VER}-devel gcc" + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -5734,7 +5771,12 @@ install_amazon_linux_ami_2_deps() { # We need to install yum-utils before doing anything else when installing on # Amazon Linux ECS-optimized images. See issue #974. - __yum_install_noinput yum-utils + __PACKAGES="yum-utils" + + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + + __yum_install_noinput ${__PACKAGES} # Do upgrade early if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then @@ -5778,7 +5820,12 @@ install_amazon_linux_ami_2_onedir_deps() { # We need to install yum-utils before doing anything else when installing on # Amazon Linux ECS-optimized images. See issue #974. - __yum_install_noinput yum-utils + __PACKAGES="yum-utils" + + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + + __yum_install_noinput ${__PACKAGES} # Do upgrade early if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then @@ -5890,6 +5937,9 @@ install_amazon_linux_ami_2023_git_deps() { __PACKAGES="python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools python${PY_PKG_VER}-devel gcc" + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -5906,7 +5956,12 @@ install_amazon_linux_ami_2023_onedir_deps() { # We need to install yum-utils before doing anything else when installing on # Amazon Linux ECS-optimized images. See issue #974. - __yum_install_noinput yum-utils + __PACKAGES="yum-utils" + + # ensure sudo installed + __PACKAGES="${__PACKAGES} sudo" + + __yum_install_noinput ${__PACKAGES} # Do upgrade early if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then From 043f8e0b78ca9e08775c9f211cfbd72a52018a63 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 12 Nov 2024 16:35:32 -0700 Subject: [PATCH 090/173] Updated container-slug to prefix systemd- for Linux containers --- .github/workflows/ci.yml | 26 ++++++++++++------------- .github/workflows/templates/generate.py | 26 ++++++++++++------------- .github/workflows/test-linux.yml | 2 +- 3 files changed, 27 insertions(+), 27 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 28e300ae3..2c2eb0a97 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -174,7 +174,7 @@ jobs: with: distro-slug: amazonlinux-2 display-name: Amazon 2 - container-slug: amazonlinux-2 + container-slug: systemd-amazonlinux-2 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' @@ -189,7 +189,7 @@ jobs: with: distro-slug: amazonlinux-2023 display-name: Amazon 2023 - container-slug: amazonlinux-2023 + container-slug: systemd-amazonlinux-2023 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' @@ -204,7 +204,7 @@ jobs: with: distro-slug: debian-11 display-name: Debian 11 - container-slug: debian-11 + container-slug: systemd-debian-11 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' @@ -219,7 +219,7 @@ jobs: with: distro-slug: debian-12 display-name: Debian 12 - container-slug: debian-12 + container-slug: systemd-debian-12 timeout: 20 instances: '["stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' @@ -234,7 +234,7 @@ jobs: with: distro-slug: debian-13 display-name: Debian 13 - container-slug: debian-13 + container-slug: systemd-debian-13 timeout: 20 instances: '["stable-3006", "git-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' @@ -249,7 +249,7 @@ jobs: with: distro-slug: fedora-40 display-name: Fedora 40 - container-slug: fedora-40 + container-slug: systemd-fedora-40 timeout: 20 instances: '["stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' @@ -264,7 +264,7 @@ jobs: with: distro-slug: photon-4 display-name: Photon OS 4 - container-slug: photon-4 + container-slug: systemd-photon-4 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "latest", "default"]' @@ -279,7 +279,7 @@ jobs: with: distro-slug: photon-5 display-name: Photon OS 5 - container-slug: photon-5 + container-slug: systemd-photon-5 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "latest", "default"]' @@ -294,7 +294,7 @@ jobs: with: distro-slug: rockylinux-8 display-name: Rocky Linux 8 - container-slug: rockylinux-8 + container-slug: systemd-rockylinux-8 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' @@ -309,7 +309,7 @@ jobs: with: distro-slug: rockylinux-9 display-name: Rocky Linux 9 - container-slug: rockylinux-9 + container-slug: systemd-rockylinux-9 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' @@ -324,7 +324,7 @@ jobs: with: distro-slug: ubuntu-2004 display-name: Ubuntu 20.04 - container-slug: ubuntu-20.04 + container-slug: systemd-ubuntu-20.04 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' @@ -339,7 +339,7 @@ jobs: with: distro-slug: ubuntu-2204 display-name: Ubuntu 22.04 - container-slug: ubuntu-22.04 + container-slug: systemd-ubuntu-22.04 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' @@ -354,7 +354,7 @@ jobs: with: distro-slug: ubuntu-2404 display-name: Ubuntu 24.04 - container-slug: ubuntu-24.04 + container-slug: systemd-ubuntu-24.04 timeout: 20 instances: '["stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index cb7bf05b8..351db63c2 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -202,19 +202,19 @@ } CONTAINER_SLUG_NAMES = { - "amazonlinux-2": "amazonlinux-2", - "amazonlinux-2023": "amazonlinux-2023", - "debian-11": "debian-11", - "debian-12": "debian-12", - "debian-13": "debian-13", - "fedora-40": "fedora-40", - "photon-4": "photon-4", - "photon-5": "photon-5", - "rockylinux-8": "rockylinux-8", - "rockylinux-9": "rockylinux-9", - "ubuntu-2004": "ubuntu-20.04", - "ubuntu-2204": "ubuntu-22.04", - "ubuntu-2404": "ubuntu-24.04", + "amazonlinux-2": "systemd-amazonlinux-2", + "amazonlinux-2023": "systemd-amazonlinux-2023", + "debian-11": "systemd-debian-11", + "debian-12": "systemd-debian-12", + "debian-13": "systemd-debian-13", + "fedora-40": "systemd-fedora-40", + "photon-4": "systemd-photon-4", + "photon-5": "systemd-photon-5", + "rockylinux-8": "systemd-rockylinux-8", + "rockylinux-9": "systemd-rockylinux-9", + "ubuntu-2004": "systemd-ubuntu-20.04", + "ubuntu-2204": "systemd-ubuntu-22.04", + "ubuntu-2404": "systemd-ubuntu-24.04", "macos-12": "macos-12", "macos-13": "macos-13", "windows-2022": "windows-2022", diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 96b2a9c5f..3d0047358 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -33,7 +33,7 @@ jobs: name: ${{ matrix.instance }} runs-on: ubuntu-latest container: - image: ghcr.io/saltstack/salt-ci-containers/testing:systemd-${{ inputs.container-slug }} + image: ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} options: --privileged timeout-minutes: ${{ inputs.timeout }} From e16cda4c1461ec8601e721a48fac0452d1a66f8d Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 13 Nov 2024 09:42:24 -0700 Subject: [PATCH 091/173] Updated to use _REPO_URL instead of explicit packages.broadcom.com when accessing --- bootstrap-salt.sh | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 9044008dc..5d7655267 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -865,7 +865,7 @@ if [ "$ITYPE" != "git" ]; then fi fi -# Set the _REPO_URL value based on if -R was passed or not. Defaults to packages.broadcom.com +# Set the _REPO_URL value based on if -R was passed or not. Defaults to packages.broadcom.com/artifactory if [ "$_CUSTOM_REPO_URL" != "null" ]; then _REPO_URL="$_CUSTOM_REPO_URL" @@ -2478,10 +2478,12 @@ __check_services_systemd() { _SYSTEMD_ACTIVE=$(/bin/systemctl daemon-reload 2>&1 | grep 'System has not been booted with systemd') echodebug "__check_services_systemd _SYSTEMD_ACTIVE result ,$_SYSTEMD_ACTIVE," - if [ "$_SYSTEMD_ACTIVE" != "" ]; then + if [ -n "$_SYSTEMD_ACTIVE" ]; then _SYSTEMD_FUNCTIONAL=$BS_FALSE echodebug "systemd is not functional, despite systemctl being present, setting _SYSTEMD_FUNCTIONAL false, $_SYSTEMD_FUNCTIONAL" return 1 + else + echodebug "systemd is functional, _SYSTEMD_FUNCTIONAL true, $_SYSTEMD_FUNCTIONAL" fi servicename=$1 @@ -6320,7 +6322,7 @@ __get_packagesite_onedir_latest() { cd ${generic_versions_tmpdir} || return 1 # leverage the windows directories since release Windows and Linux - wget -r -np -nH --exclude-directories=onedir,relenv,macos -x -l 1 "https://packages.broadcom.com/artifactory/saltproject-generic/" + wget -r -np -nH --exclude-directories=onedir,relenv,macos -x -l 1 "https://${_REPO_URL}/saltproject-generic/" if [ "$#" -gt 0 ] && [ -n "$1" ]; then MAJOR_VER="$1" # shellcheck disable=SC2010 @@ -6359,26 +6361,26 @@ __install_saltstack_photon_onedir_repository() { ## tdnf config-manager --set-enabled salt-repo-3007-sts echo "[salt-repo-3007-sts]" > "${YUM_REPO_FILE}" echo "name=Salt Repo for Salt v3007 STS" >> "${YUM_REPO_FILE}" - echo "baseurl=https://packages.broadcom.com/artifactory/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" echo "priority=10" >> "${YUM_REPO_FILE}" echo "enabled=1" >> "${YUM_REPO_FILE}" echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" echo "gpgcheck=1" >> "${YUM_REPO_FILE}" echo "exclude=*3006* *3008* *3009* *3010*" >> "${YUM_REPO_FILE}" - echo "gpgkey=https://packages.broadcom.com/artifactory/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" else # Salt 3006 repo echo "[salt-repo-3006-lts]" > "${YUM_REPO_FILE}" echo "name=Salt Repo for Salt v3006 LTS" >> "${YUM_REPO_FILE}" - echo "baseurl=https://packages.broadcom.com/artifactory/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" echo "priority=10" >> "${YUM_REPO_FILE}" echo "enabled=1" >> "${YUM_REPO_FILE}" echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" echo "gpgcheck=1" >> "${YUM_REPO_FILE}" echo "exclude=*3007* *3008* *3009* *3010*" >> "${YUM_REPO_FILE}" - echo "gpgkey=https://packages.broadcom.com/artifactory/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" fi else # Enable the Salt LATEST repo @@ -6386,13 +6388,13 @@ __install_saltstack_photon_onedir_repository() { ## tdnf config-manager --set-enabled salt-repo-latest echo "[salt-repo-latest]" > "${YUM_REPO_FILE}" echo "name=Salt Repo for Salt LATEST release" >> "${YUM_REPO_FILE}" - echo "baseurl=https://packages.broadcom.com/artifactory/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" echo "priority=10" >> "${YUM_REPO_FILE}" echo "enabled=1" >> "${YUM_REPO_FILE}" echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" echo "gpgcheck=1" >> "${YUM_REPO_FILE}" - echo "gpgkey=https://packages.broadcom.com/artifactory/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" fi tdnf makecache || return 1 elif [ "$ONEDIR_REV" != "latest" ]; then From f2b7f56ceb2eeda421630c825c36cb9dd26a7f9e Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 18 Nov 2024 10:17:55 -0700 Subject: [PATCH 092/173] Minor updates to README, remove support for Redhat 7, Raspbian 8 & 9 --- README.rst | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/README.rst b/README.rst index 2e896305b..7c524e3a7 100644 --- a/README.rst +++ b/README.rst @@ -414,21 +414,19 @@ Debian and derivatives - Devuan GNU/Linux 1/2/3/4/5 - Kali Linux 1.0 (based on Debian 7) - Linux Mint Debian Edition 1 (based on Debian 8) -- Raspbian 8 (``armhf`` packages) and 9 (using ``git`` installation mode only) Red Hat family ~~~~~~~~~~~~~~ -- Amazon Linux 2012.3 and later - Amazon Linux 2 - Amazon Linux 2023 -- CentOS 7/8/9 +- CentOS 8/9 - Cloud Linux 6/7 -- Fedora 38/39/40 (install latest stable from standard repositories) -- Oracle Linux 7/8/9 -- Red Hat Enterprise Linux 7/8/9 -- Scientific Linux 7/8/9 +- Fedora 40 (install latest stable from standard repositories) +- Oracle Linux 8/9 +- Red Hat Enterprise Linux 8/9 +- Scientific Linux 8/9 SUSE family @@ -461,8 +459,8 @@ repositories are not provided on the `Salt Project's Ubuntu repository`_ for the the bootstrap script will attempt to install the packages for the most closely related LTS Ubuntu release instead. -For example, when installing Salt on Ubuntu 21.10, the bootstrap script will setup the repository -for Ubuntu 20.04 from the `Salt Project's Ubuntu repository`_ and install the 20.04 packages. +For example, when installing Salt on Ubuntu 24.10, the bootstrap script will setup the repository +for Ubuntu 24.04 from the `Salt Project's Ubuntu repository`_ and install the 24.04 packages. Non-LTS Ubuntu releases are not supported once the release reaches End-of-Life as defined by `Ubuntu's release schedule`_. From 0c1de07494373b8f1d1f3edea61cc3b55a99796c Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 18 Nov 2024 14:22:26 -0700 Subject: [PATCH 093/173] Updated date, and makecache for RedHat family --- bootstrap-salt.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 5d7655267..2dbda26af 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -26,7 +26,7 @@ #====================================================================================================================== set -o nounset # Treat unset variables as an error -__ScriptVersion="2024.11.11" +__ScriptVersion="2024.11.18" __ScriptName="bootstrap-salt.sh" __ScriptFullName="$0" @@ -2679,8 +2679,10 @@ __install_pip_pkgs() { else __PACKAGES="${__PACKAGES} ${_py_pkg}-devel" if [ "$DISTRO_NAME_L" = "fedora" ];then + dnf makecache || return 1 __dnf_install_noinput ${__PACKAGES} || return 1 else + yum makecache || return 1 __yum_install_noinput ${__PACKAGES} || return 1 fi fi @@ -4292,6 +4294,7 @@ install_fedora_onedir() { fi # shellcheck disable=SC2086 + dnf makecache || return 1 __yum_install_noinput ${__PACKAGES} || return 1 return 0 @@ -4445,6 +4448,7 @@ install_centos_stable() { __PACKAGES="${__PACKAGES} sudo" # shellcheck disable=SC2086 + yum makecache || return 1 __yum_install_noinput ${__PACKAGES} || return 1 # Workaround for 3.11 broken on CentOS Stream 8.x @@ -4686,6 +4690,7 @@ install_centos_onedir() { fi # shellcheck disable=SC2086 + dnf makecache || return 1 __yum_install_noinput ${__PACKAGES} || return 1 return 0 From db5d548d1a1decc074488d6f1d828e608bb4db00 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 18 Nov 2024 14:48:55 -0700 Subject: [PATCH 094/173] debugging --- bootstrap-salt.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 2dbda26af..e25681eb2 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -4691,6 +4691,7 @@ install_centos_onedir() { # shellcheck disable=SC2086 dnf makecache || return 1 + dnf list salt-* || return 1 __yum_install_noinput ${__PACKAGES} || return 1 return 0 From 5bd00aca9cdcd434c333d68659483b55d0222c4d Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 18 Nov 2024 15:32:01 -0700 Subject: [PATCH 095/173] Debugging install for minor version --- bootstrap-salt.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index e25681eb2..e50556965 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -4691,7 +4691,8 @@ install_centos_onedir() { # shellcheck disable=SC2086 dnf makecache || return 1 - dnf list salt-* || return 1 + echo "DGM install_centos_onedir, __PACKAGES '${__PACKAGES}'" + dnf list salt-minion || return 1 __yum_install_noinput ${__PACKAGES} || return 1 return 0 From 12dd54c3329fe919dd33e9764fbd0950b5aaadbf Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 18 Nov 2024 16:15:26 -0700 Subject: [PATCH 096/173] Fixed up script to better allow for Redhat family minor versions of salt --- bootstrap-salt.sh | 193 +++++++++++++++++++++++++++++++++------------- 1 file changed, 141 insertions(+), 52 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index e50556965..76159a9d5 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -4029,12 +4029,27 @@ __install_saltstack_fedora_onedir_repository() { FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" if [ "$ONEDIR_REV" != "latest" ]; then - # 3006.x is default - REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) - if [ "$REPO_REV_MAJOR" -eq "3007" ]; then - # Enable the Salt 3007 STS repo - dnf config-manager --set-disable salt-repo-* - dnf config-manager --set-enabled salt-repo-3007-sts + # 3006.x is default, and latest for 3006.x branch + if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # latest version for branch 3006 | 3007 + REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) + if [ "$REPO_REV_MAJOR" -eq "3007" ]; then + # Enable the Salt 3007 STS repo + dnf config-manager --set-disable salt-repo-* + dnf config-manager --set-enabled salt-repo-3007-sts + fi + elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # using minor version + echo "[salt-repo-${ONEDIR_REV}-lts]" > "${YUM_REPO_FILE}" + # shellcheck disable=SC2129 + echo "name=Salt Repo for Salt v${ONEDIR_REV} LTS" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/${ONEDIR_REV}/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" fi else # Enable the Salt LATEST repo @@ -4344,12 +4359,27 @@ __install_saltstack_rhel_onedir_repository() { FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" if [ "$ONEDIR_REV" != "latest" ]; then - # 3006.x is default - REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) - if [ "$REPO_REV_MAJOR" -eq "3007" ]; then - # Enable the Salt 3007 STS repo - dnf config-manager --set-disable salt-repo-* - dnf config-manager --set-enabled salt-repo-3007-sts + # 3006.x is default, and latest for 3006.x branch + if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # latest version for branch 3006 | 3007 + REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) + if [ "$REPO_REV_MAJOR" -eq "3007" ]; then + # Enable the Salt 3007 STS repo + dnf config-manager --set-disable salt-repo-* + dnf config-manager --set-enabled salt-repo-3007-sts + fi + elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # using minor version + echo "[salt-repo-${ONEDIR_REV}-lts]" > "${YUM_REPO_FILE}" + # shellcheck disable=SC2129 + echo "name=Salt Repo for Salt v${ONEDIR_REV} LTS" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/${ONEDIR_REV}/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" fi else # Enable the Salt LATEST repo @@ -5797,12 +5827,27 @@ install_amazon_linux_ami_2_deps() { FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" if [ "$STABLE_REV" != "latest" ]; then - # 3006.x is default - REPO_REV_MAJOR=$(echo "$STABLE_REV" | cut -d '.' -f 1) - if [ "$REPO_REV_MAJOR" -eq "3007" ]; then - # Enable the Salt 3007 STS repo - dnf config-manager --set-disable salt-repo-* - dnf config-manager --set-enabled salt-repo-3007-sts + # 3006.x is default, and latest for 3006.x branch + if [ "$(echo "$STABLE_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # latest version for branch 3006 | 3007 + REPO_REV_MAJOR=$(echo "$STABLE_REV" | cut -d '.' -f 1) + if [ "$REPO_REV_MAJOR" -eq "3007" ]; then + # Enable the Salt 3007 STS repo + dnf config-manager --set-disable salt-repo-* + dnf config-manager --set-enabled salt-repo-3007-sts + fi + elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # using minor version + echo "[salt-repo-${STABLE_REV}-lts]" > "${YUM_REPO_FILE}" + # shellcheck disable=SC2129 + echo "name=Salt Repo for Salt v${STABLE_REV} LTS" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/${STABLE_REV}/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" fi else # Enable the Salt LATEST repo @@ -5846,12 +5891,27 @@ install_amazon_linux_ami_2_onedir_deps() { FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" if [ "$ONEDIR_REV" != "latest" ]; then - # 3006.x is default - REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) - if [ "$REPO_REV_MAJOR" -eq "3007" ]; then - # Enable the Salt 3007 STS repo - dnf config-manager --set-disable salt-repo-* - dnf config-manager --set-enabled salt-repo-3007-sts + # 3006.x is default, and latest for 3006.x branch + if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # latest version for branch 3006 | 3007 + REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) + if [ "$REPO_REV_MAJOR" -eq "3007" ]; then + # Enable the Salt 3007 STS repo + dnf config-manager --set-disable salt-repo-* + dnf config-manager --set-enabled salt-repo-3007-sts + fi + elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # using minor version + echo "[salt-repo-${ONEDIR_REV}-lts]" > "${YUM_REPO_FILE}" + # shellcheck disable=SC2129 + echo "name=Salt Repo for Salt v${ONEDIR_REV} LTS" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/${ONEDIR_REV}/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" fi else # Enable the Salt LATEST repo @@ -5982,12 +6042,27 @@ install_amazon_linux_ami_2023_onedir_deps() { FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" if [ "$ONEDIR_REV" != "latest" ]; then - # 3006.x is default - REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) - if [ "$REPO_REV_MAJOR" -eq "3007" ]; then - # Enable the Salt 3007 STS repo - dnf config-manager --set-disable salt-repo-* - dnf config-manager --set-enabled salt-repo-3007-sts + # 3006.x is default, and latest for 3006.x branch + if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # latest version for branch 3006 | 3007 + REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) + if [ "$REPO_REV_MAJOR" -eq "3007" ]; then + # Enable the Salt 3007 STS repo + dnf config-manager --set-disable salt-repo-* + dnf config-manager --set-enabled salt-repo-3007-sts + fi + elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # using minor version + echo "[salt-repo-${ONEDIR_REV}-lts]" > "${YUM_REPO_FILE}" + # shellcheck disable=SC2129 + echo "name=Salt Repo for Salt v${ONEDIR_REV} LTS" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/${ONEDIR_REV}/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" fi else # Enable the Salt LATEST repo @@ -6360,33 +6435,47 @@ __install_saltstack_photon_onedir_repository() { ## __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" # shellcheck disable=SC2129 if [ "$ONEDIR_REV" != "latest" ]; then - # 3006.x is default - REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) - if [ "$REPO_REV_MAJOR" -eq "3007" ]; then - # Enable the Salt 3007 STS repo - ## tdnf config-manager --set-disable salt-repo-* - ## tdnf config-manager --set-enabled salt-repo-3007-sts - echo "[salt-repo-3007-sts]" > "${YUM_REPO_FILE}" - echo "name=Salt Repo for Salt v3007 STS" >> "${YUM_REPO_FILE}" - echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" - echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" - echo "priority=10" >> "${YUM_REPO_FILE}" - echo "enabled=1" >> "${YUM_REPO_FILE}" - echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" - echo "gpgcheck=1" >> "${YUM_REPO_FILE}" - echo "exclude=*3006* *3008* *3009* *3010*" >> "${YUM_REPO_FILE}" - echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" - else - # Salt 3006 repo - echo "[salt-repo-3006-lts]" > "${YUM_REPO_FILE}" - echo "name=Salt Repo for Salt v3006 LTS" >> "${YUM_REPO_FILE}" - echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" + # 3006.x is default, and latest for 3006.x branch + if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # latest version for branch 3006 | 3007 + REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) + if [ "$REPO_REV_MAJOR" -eq "3007" ]; then + # Enable the Salt 3007 STS repo + ## tdnf config-manager --set-disable salt-repo-* + ## tdnf config-manager --set-enabled salt-repo-3007-sts + echo "[salt-repo-3007-sts]" > "${YUM_REPO_FILE}" + echo "name=Salt Repo for Salt v3007 STS" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "exclude=*3006* *3008* *3009* *3010*" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" + else + # Salt 3006 repo + echo "[salt-repo-3006-lts]" > "${YUM_REPO_FILE}" + echo "name=Salt Repo for Salt v3006 LTS" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "exclude=*3007* *3008* *3009* *3010*" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" + fi + elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # using minor version + echo "[salt-repo-${ONEDIR_REV}-lts]" > "${YUM_REPO_FILE}" + echo "name=Salt Repo for Salt v${ONEDIR_REV} LTS" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/${ONEDIR_REV}/" >> "${YUM_REPO_FILE}" echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" echo "priority=10" >> "${YUM_REPO_FILE}" echo "enabled=1" >> "${YUM_REPO_FILE}" echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" echo "gpgcheck=1" >> "${YUM_REPO_FILE}" - echo "exclude=*3007* *3008* *3009* *3010*" >> "${YUM_REPO_FILE}" echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" fi else From 3de9da512bcd980c6790731c9c2155f1ae7c8230 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 19 Nov 2024 10:33:08 -0700 Subject: [PATCH 097/173] Updated minor verson handling for RedHat families --- bootstrap-salt.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 76159a9d5..3b6b9ba42 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -4043,7 +4043,7 @@ __install_saltstack_fedora_onedir_repository() { echo "[salt-repo-${ONEDIR_REV}-lts]" > "${YUM_REPO_FILE}" # shellcheck disable=SC2129 echo "name=Salt Repo for Salt v${ONEDIR_REV} LTS" >> "${YUM_REPO_FILE}" - echo "baseurl=https://${_REPO_URL}/saltproject-rpm/${ONEDIR_REV}/" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" echo "priority=10" >> "${YUM_REPO_FILE}" echo "enabled=1" >> "${YUM_REPO_FILE}" @@ -4373,7 +4373,7 @@ __install_saltstack_rhel_onedir_repository() { echo "[salt-repo-${ONEDIR_REV}-lts]" > "${YUM_REPO_FILE}" # shellcheck disable=SC2129 echo "name=Salt Repo for Salt v${ONEDIR_REV} LTS" >> "${YUM_REPO_FILE}" - echo "baseurl=https://${_REPO_URL}/saltproject-rpm/${ONEDIR_REV}/" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" echo "priority=10" >> "${YUM_REPO_FILE}" echo "enabled=1" >> "${YUM_REPO_FILE}" @@ -5841,7 +5841,7 @@ install_amazon_linux_ami_2_deps() { echo "[salt-repo-${STABLE_REV}-lts]" > "${YUM_REPO_FILE}" # shellcheck disable=SC2129 echo "name=Salt Repo for Salt v${STABLE_REV} LTS" >> "${YUM_REPO_FILE}" - echo "baseurl=https://${_REPO_URL}/saltproject-rpm/${STABLE_REV}/" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" echo "priority=10" >> "${YUM_REPO_FILE}" echo "enabled=1" >> "${YUM_REPO_FILE}" @@ -5905,7 +5905,7 @@ install_amazon_linux_ami_2_onedir_deps() { echo "[salt-repo-${ONEDIR_REV}-lts]" > "${YUM_REPO_FILE}" # shellcheck disable=SC2129 echo "name=Salt Repo for Salt v${ONEDIR_REV} LTS" >> "${YUM_REPO_FILE}" - echo "baseurl=https://${_REPO_URL}/saltproject-rpm/${ONEDIR_REV}/" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" echo "priority=10" >> "${YUM_REPO_FILE}" echo "enabled=1" >> "${YUM_REPO_FILE}" @@ -6056,7 +6056,7 @@ install_amazon_linux_ami_2023_onedir_deps() { echo "[salt-repo-${ONEDIR_REV}-lts]" > "${YUM_REPO_FILE}" # shellcheck disable=SC2129 echo "name=Salt Repo for Salt v${ONEDIR_REV} LTS" >> "${YUM_REPO_FILE}" - echo "baseurl=https://${_REPO_URL}/saltproject-rpm/${ONEDIR_REV}/" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" echo "priority=10" >> "${YUM_REPO_FILE}" echo "enabled=1" >> "${YUM_REPO_FILE}" @@ -6470,7 +6470,7 @@ __install_saltstack_photon_onedir_repository() { # using minor version echo "[salt-repo-${ONEDIR_REV}-lts]" > "${YUM_REPO_FILE}" echo "name=Salt Repo for Salt v${ONEDIR_REV} LTS" >> "${YUM_REPO_FILE}" - echo "baseurl=https://${_REPO_URL}/saltproject-rpm/${ONEDIR_REV}/" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" echo "priority=10" >> "${YUM_REPO_FILE}" echo "enabled=1" >> "${YUM_REPO_FILE}" From 8f07647b1ebe288bd1c0e785a97ac82b38301349 Mon Sep 17 00:00:00 2001 From: Ruslan Tumarkin Date: Tue, 19 Nov 2024 18:48:40 +0100 Subject: [PATCH 098/173] fix issue 2047 hwclock check missing additional condition on major version --- bootstrap-salt.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index d4fa965e8..5f2d6816c 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -3032,7 +3032,7 @@ __install_saltstack_ubuntu_onedir_repository() { fi ## include hwclock if not part of base OS (23.10 and up) - if [ ! -f /usr/sbin/hwclock ]; then + if [ "$DISTRO_MAJOR_VERSION" -ge 23 ] && [ ! -f /usr/sbin/hwclock ]; then __PACKAGES="${__PACKAGES} util-linux-extra" fi From 95ff9355c5fa76225fc51d5d51d0729183ac5d9e Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 19 Nov 2024 11:28:37 -0700 Subject: [PATCH 099/173] Updated Redhat family minor salt version commands --- .github/workflows/ci.yml | 144 ------------------------ .github/workflows/templates/generate.py | 79 +++++++------ bootstrap-salt.sh | 12 +- 3 files changed, 50 insertions(+), 185 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2c2eb0a97..06dcdfdb1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -164,111 +164,6 @@ jobs: - amazonlinux-2: - name: Amazon 2 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: amazonlinux-2 - display-name: Amazon 2 - container-slug: systemd-amazonlinux-2 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' - - - amazonlinux-2023: - name: Amazon 2023 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: amazonlinux-2023 - display-name: Amazon 2023 - container-slug: systemd-amazonlinux-2023 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' - - - debian-11: - name: Debian 11 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: debian-11 - display-name: Debian 11 - container-slug: systemd-debian-11 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - - - debian-12: - name: Debian 12 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: debian-12 - display-name: Debian 12 - container-slug: systemd-debian-12 - timeout: 20 - instances: '["stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - - - debian-13: - name: Debian 13 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: debian-13 - display-name: Debian 13 - container-slug: systemd-debian-13 - timeout: 20 - instances: '["stable-3006", "git-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - - - fedora-40: - name: Fedora 40 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: fedora-40 - display-name: Fedora 40 - container-slug: systemd-fedora-40 - timeout: 20 - instances: '["stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - - - photon-4: - name: Photon OS 4 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: photon-4 - display-name: Photon OS 4 - container-slug: systemd-photon-4 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "latest", "default"]' - - photon-5: name: Photon OS 5 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' @@ -314,21 +209,6 @@ jobs: instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - ubuntu-2004: - name: Ubuntu 20.04 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: ubuntu-2004 - display-name: Ubuntu 20.04 - container-slug: systemd-ubuntu-20.04 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - - ubuntu-2204: name: Ubuntu 22.04 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' @@ -344,21 +224,6 @@ jobs: instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - ubuntu-2404: - name: Ubuntu 24.04 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: ubuntu-2404 - display-name: Ubuntu 24.04 - container-slug: systemd-ubuntu-24.04 - timeout: 20 - instances: '["stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - - set-pipeline-exit-status: # This step is just so we can make github require this step, to pass checks # on a pull request instead of requiring all @@ -370,19 +235,10 @@ jobs: - macos-12 - macos-13 - windows-2022 - - amazonlinux-2 - - amazonlinux-2023 - - debian-11 - - debian-12 - - debian-13 - - fedora-40 - - photon-4 - photon-5 - rockylinux-8 - rockylinux-9 - - ubuntu-2004 - ubuntu-2204 - - ubuntu-2404 if: always() steps: diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index 351db63c2..71b805828 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -6,20 +6,22 @@ os.chdir(os.path.abspath(os.path.dirname(__file__))) +# only test against current containers with systemd +# will add these when they become available with systemd +# "amazonlinux-2", +# "amazonlinux-2023", +# "debian-11", +# "debian-12", +# "debian-13", +# "fedora-40", +# "photon-4", +# "ubuntu-2004", +# "ubuntu-2404", LINUX_DISTROS = [ - "amazonlinux-2", - "amazonlinux-2023", - "debian-11", - "debian-12", - "debian-13", - "fedora-40", - "photon-4", "photon-5", "rockylinux-8", "rockylinux-9", - "ubuntu-2004", "ubuntu-2204", - "ubuntu-2404", ] WINDOWS = [ @@ -31,49 +33,56 @@ "macos-13", ] +# only test against current containers with systemd +# will add these when they become available with systemd +# "amazonlinux-2", +# "amazonlinux-2023", +# "centos-stream9", +# "debian-11", +# "debian-12", +# "debian-13", +# "fedora-40", +# "photon-4", +# "ubuntu-2004", +# "ubuntu-2404", STABLE_DISTROS = [ - "amazonlinux-2", - "amazonlinux-2023", - "centos-stream9", - "debian-11", - "debian-12", - "debian-13", - "fedora-40", - "photon-4", "photon-5", "rockylinux-8", "rockylinux-9", - "ubuntu-2004", "ubuntu-2204", - "ubuntu-2404", ] +# only test against current containers with systemd +# will add these when they become available with systemd +# "amazonlinux-2", +# "amazonlinux-2023", +# "debian-11", +# "debian-12", +# "debian-13", +# "fedora-40", +# "photon-4", +# "ubuntu-2004", +# "ubuntu-2404", ONEDIR_DISTROS = [ - "amazonlinux-2", - "amazonlinux-2023", - "debian-11", - "debian-12", - "debian-13", - "fedora-40", - "oraclelinux-8", - "oraclelinux-9", - "photon-4", "photon-5", "rockylinux-8", "rockylinux-9", - "ubuntu-2004", "ubuntu-2204", - "ubuntu-2404", ] +# only test against current containers with systemd +# will add these when they become available with systemd +# "amazonlinux-2", +# "amazonlinux-2023", +# "debian-12", +# "photon-4", +# "photon-5", +# "rockylinux-9", +# "ubuntu-2404", ONEDIR_RC_DISTROS = [ - "amazonlinux-2", - "amazonlinux-2023", - "debian-12", - "photon-4", "photon-5", "rockylinux-9", - "ubuntu-2404", + "ubuntu-2204", ] BLACKLIST_3006 = [ diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 3b6b9ba42..2e34d93dd 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -4284,7 +4284,7 @@ install_fedora_onedir() { MINOR_VER_STRG="" elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then # Minor version Salt, need to add specific minor version - MINOR_VER_STRG="=$STABLE_REV" + MINOR_VER_STRG="-$STABLE_REV" else MINOR_VER_STRG="" fi @@ -4450,7 +4450,7 @@ install_centos_stable() { MINOR_VER_STRG="" elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then # Minor version Salt, need to add specific minor version - MINOR_VER_STRG="=$STABLE_REV" + MINOR_VER_STRG="-$STABLE_REV" else MINOR_VER_STRG="" fi @@ -4695,7 +4695,7 @@ install_centos_onedir() { MINOR_VER_STRG="" elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then # Minor version Salt, need to add specific minor version - MINOR_VER_STRG="=$ONEDIR_REV" + MINOR_VER_STRG="-$ONEDIR_REV" else MINOR_VER_STRG="" fi @@ -6404,7 +6404,7 @@ __get_packagesite_onedir_latest() { cd ${generic_versions_tmpdir} || return 1 # leverage the windows directories since release Windows and Linux - wget -r -np -nH --exclude-directories=onedir,relenv,macos -x -l 1 "https://${_REPO_URL}/saltproject-generic/" + wget -r -np -nH --exclude-directories=onedir,relenv,macos -x -l 1 "https://${_REPO_URL}/saltproject-generic/windows/" if [ "$#" -gt 0 ] && [ -n "$1" ]; then MAJOR_VER="$1" # shellcheck disable=SC2010 @@ -6756,7 +6756,7 @@ install_photon_onedir() { MINOR_VER_STRG="$_GENERIC_PKG_VERSION" elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then # Minor version Salt, need to add specific minor version - MINOR_VER_STRG="=$STABLE_REV" + MINOR_VER_STRG="-$STABLE_REV" else # default to latest version Salt, config and repo already setup __get_packagesite_onedir_latest @@ -6968,7 +6968,7 @@ install_opensuse_stable() { MINOR_VER_STRG="" elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then # Minor version Salt, need to add specific minor version - MINOR_VER_STRG="=$STABLE_REV" + MINOR_VER_STRG="-$STABLE_REV" else MINOR_VER_STRG="" fi From f6906be830cc38ff90817f0f07e46934db513aee Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 19 Nov 2024 12:18:28 -0700 Subject: [PATCH 100/173] debugging --- bootstrap-salt.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 2e34d93dd..4750caad0 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -2849,6 +2849,9 @@ EOM ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --ignore-installed ${_PIP_INSTALL_ARGS} /tmp/git/deps/* || return 1 rm -f /tmp/git/deps/* + echodebug "DGM DEBUG checking pip list for setuptools, ${_pip_cmd} list" + ${_pip_cmd} list + echoinfo "Building Salt Python Wheel" if [ "$_ECHO_DEBUG" -eq $BS_TRUE ]; then From cc583da80c281f1bf384335699a8261d7a1fd811 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 19 Nov 2024 12:48:24 -0700 Subject: [PATCH 101/173] More debugging --- bootstrap-salt.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 4750caad0..377291f43 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -2752,6 +2752,7 @@ __install_salt_from_repo() { fi echodebug "__install_salt_from_repo py_exe=$_py_exe" + echodebug "DGM checking python version $(${_py_exe} --version)" _py_version=$(${_py_exe} -c "import sys; print('{0}.{1}'.format(*sys.version_info))") _pip_cmd="pip${_py_version}" @@ -2827,6 +2828,9 @@ EOM echodebug "Running '${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --upgrade ${_PIP_INSTALL_ARGS} wheel ${_setuptools_dep}" ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --upgrade ${_PIP_INSTALL_ARGS} wheel "${_setuptools_dep}" + echodebug "DGM DEBUG 1 checking pip list for setuptools, ${_pip_cmd} list" + ${_pip_cmd} list + echoinfo "Installing salt using ${_py_exe}, $(${_py_exe} --version)" cd "${_SALT_GIT_CHECKOUT_DIR}" || return 1 @@ -2849,7 +2853,7 @@ EOM ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --ignore-installed ${_PIP_INSTALL_ARGS} /tmp/git/deps/* || return 1 rm -f /tmp/git/deps/* - echodebug "DGM DEBUG checking pip list for setuptools, ${_pip_cmd} list" + echodebug "DGM DEBUG 2 checking pip list for setuptools, ${_pip_cmd} list" ${_pip_cmd} list echoinfo "Building Salt Python Wheel" From 035d769e6027694d0cfa9f6a4b54a674cfe39f58 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 19 Nov 2024 13:19:44 -0700 Subject: [PATCH 102/173] more debuging --- bootstrap-salt.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 377291f43..0844f2dcb 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -2827,10 +2827,18 @@ EOM echodebug "Running '${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --upgrade ${_PIP_INSTALL_ARGS} wheel ${_setuptools_dep}" ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --upgrade ${_PIP_INSTALL_ARGS} wheel "${_setuptools_dep}" + __retn="$?" + echodebug "DGM DEBUG 0 return code from installing setuptools '$__retn'" echodebug "DGM DEBUG 1 checking pip list for setuptools, ${_pip_cmd} list" ${_pip_cmd} list + echodebug "DGM DEBUG 1a checking pip list for setuptools" + python3 -m pip list + + echodebug "DGM DEBUG 1b checking pip list for setuptools" + ${_py_exe} -m pip list + echoinfo "Installing salt using ${_py_exe}, $(${_py_exe} --version)" cd "${_SALT_GIT_CHECKOUT_DIR}" || return 1 From a9cddcf5c6c332d675d4c97d0777d415028d8e69 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 19 Nov 2024 13:55:50 -0700 Subject: [PATCH 103/173] Removed Rocky 9 from testing with git master --- .github/workflows/ci.yml | 2 +- .github/workflows/templates/generate.py | 1 + bootstrap-salt.sh | 15 --------------- 3 files changed, 2 insertions(+), 16 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 06dcdfdb1..3f57936a2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -206,7 +206,7 @@ jobs: display-name: Rocky Linux 9 container-slug: systemd-rockylinux-9 timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' + instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' ubuntu-2204: diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index 71b805828..15b3861f4 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -130,6 +130,7 @@ "amazonlinux-2023", "photon-4", "photon-5", + "rockylinux-9", ] SALT_VERSIONS = [ diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 0844f2dcb..d9c113635 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -2827,17 +2827,6 @@ EOM echodebug "Running '${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --upgrade ${_PIP_INSTALL_ARGS} wheel ${_setuptools_dep}" ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --upgrade ${_PIP_INSTALL_ARGS} wheel "${_setuptools_dep}" - __retn="$?" - - echodebug "DGM DEBUG 0 return code from installing setuptools '$__retn'" - echodebug "DGM DEBUG 1 checking pip list for setuptools, ${_pip_cmd} list" - ${_pip_cmd} list - - echodebug "DGM DEBUG 1a checking pip list for setuptools" - python3 -m pip list - - echodebug "DGM DEBUG 1b checking pip list for setuptools" - ${_py_exe} -m pip list echoinfo "Installing salt using ${_py_exe}, $(${_py_exe} --version)" cd "${_SALT_GIT_CHECKOUT_DIR}" || return 1 @@ -2861,11 +2850,7 @@ EOM ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --ignore-installed ${_PIP_INSTALL_ARGS} /tmp/git/deps/* || return 1 rm -f /tmp/git/deps/* - echodebug "DGM DEBUG 2 checking pip list for setuptools, ${_pip_cmd} list" - ${_pip_cmd} list - echoinfo "Building Salt Python Wheel" - if [ "$_ECHO_DEBUG" -eq $BS_TRUE ]; then SETUP_PY_INSTALL_ARGS="-v" fi From 27461c79bb816e52fbef98c1658052f0b903d0cb Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 19 Nov 2024 14:13:01 -0700 Subject: [PATCH 104/173] Clean up of most of the debug code and commented out code --- bootstrap-salt.sh | 187 +--------------------------------------------- 1 file changed, 2 insertions(+), 185 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index d9c113635..16d91e6d7 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -26,7 +26,7 @@ #====================================================================================================================== set -o nounset # Treat unset variables as an error -__ScriptVersion="2024.11.18" +__ScriptVersion="2024.11.19" __ScriptName="bootstrap-salt.sh" __ScriptFullName="$0" @@ -221,9 +221,6 @@ __check_config_dir() { # DESCRIPTION: Checks the placed after the install arguments #---------------------------------------------------------------------------------------------------------------------- __check_unparsed_options() { - # DGM debug - set -v - set -x shellopts="$1" # grep alternative for SunOS @@ -628,7 +625,7 @@ if [ "$(echo "$ITYPE" | grep -E '(latest|default|stable|testing|git|onedir|onedi exit 1 fi -## DGM added this to CI/CD easier, needs removal once get compund conditionals resolved in GitHub Actions +## allows GitHub Actions CI/CD easier handling of latest and default if [ "$ITYPE" = "latest" ] || [ "$ITYPE" = "default" ]; then STABLE_REV="latest" ONEDIR_REV="latest" @@ -936,9 +933,6 @@ fi # DESCRIPTION: Retrieves a URL and writes it to a given path #---------------------------------------------------------------------------------------------------------------------- __fetch_url() { - # DGM debug - set -v - set -x # shellcheck disable=SC2086 curl $_CURL_ARGS -L -s -f -o "$1" "$2" >/dev/null 2>&1 || @@ -954,10 +948,6 @@ __fetch_url() { # DESCRIPTION: Retrieves a URL, verifies its content and writes it to standard output #---------------------------------------------------------------------------------------------------------------------- __fetch_verify() { - # DGM debug - set -v - set -x - fetch_verify_url="$1" fetch_verify_sum="$2" @@ -980,10 +970,6 @@ __fetch_verify() { # DESCRIPTION: Checks if a URL exists #---------------------------------------------------------------------------------------------------------------------- __check_url_exists() { - # DGM debug - set -v - set -x - _URL="$1" if curl --output /dev/null --silent --fail "${_URL}"; then @@ -1952,9 +1938,6 @@ __function_defined() { # process is finished so the script doesn't exit on a locked proc. #---------------------------------------------------------------------------------------------------------------------- __wait_for_apt(){ - # DGM debug - set -v - set -x # Timeout set at 15 minutes WAIT_TIMEOUT=900 @@ -1993,10 +1976,6 @@ __wait_for_apt(){ # PARAMETERS: packages #---------------------------------------------------------------------------------------------------------------------- __apt_get_install_noinput() { - # DGM debug - set -v - set -x - __wait_for_apt apt-get install -y -o DPkg::Options::=--force-confold "${@}"; return $? } # ---------- end of function __apt_get_install_noinput ---------- @@ -2007,9 +1986,6 @@ __apt_get_install_noinput() { # DESCRIPTION: (DRY) apt-get upgrade with noinput options #---------------------------------------------------------------------------------------------------------------------- __apt_get_upgrade_noinput() { - # DGM debug - set -v - set -x __wait_for_apt apt-get upgrade -y -o DPkg::Options::=--force-confold; return $? } # ---------- end of function __apt_get_upgrade_noinput ---------- @@ -2041,9 +2017,6 @@ __temp_gpg_pub() { # PARAMETERS: url #---------------------------------------------------------------------------------------------------------------------- __apt_key_fetch() { - # DGM debug - set -v - set -x url=$1 @@ -2654,9 +2627,6 @@ __activate_virtualenv() { #---------------------------------------------------------------------------------------------------------------------- __install_pip_pkgs() { - # DGM debug - set -v - set -x _pip_pkgs="$1" _py_exe="$2" @@ -2701,9 +2671,6 @@ __install_pip_pkgs() { # PARAMETERS: requirements_file #---------------------------------------------------------------------------------------------------------------------- __install_pip_deps() { - # DGM debug - set -v - set -x # Install virtualenv to system pip before activating virtualenv if thats going to be used # We assume pip pkg is installed since that is distro specific @@ -2741,9 +2708,6 @@ __install_pip_deps() { # PARAMETERS: py_exe #---------------------------------------------------------------------------------------------------------------------- __install_salt_from_repo() { - # DGM debug - set -v - set -x _py_exe="$1" @@ -2752,7 +2716,6 @@ __install_salt_from_repo() { fi echodebug "__install_salt_from_repo py_exe=$_py_exe" - echodebug "DGM checking python version $(${_py_exe} --version)" _py_version=$(${_py_exe} -c "import sys; print('{0}.{1}'.format(*sys.version_info))") _pip_cmd="pip${_py_version}" @@ -2992,9 +2955,6 @@ fi # Ubuntu Install Functions # __enable_universe_repository() { - # DGM debug - set -v - set -x echodebug "__enable_universe_repository() entry" @@ -3011,9 +2971,6 @@ __enable_universe_repository() { } __install_saltstack_ubuntu_repository() { - # DGM debug - set -v - set -x # Workaround for latest non-LTS Ubuntu echodebug "__install_saltstack_ubuntu_repository() entry" @@ -3058,16 +3015,6 @@ __install_saltstack_ubuntu_repository() { if [ "$STABLE_REV" != "latest" ]; then # latest is default - ## DGM STABLE_REV_MAJOR=$(echo "$STABLE_REV" | cut -d '.' -f 1) - ## DGM if [ "$STABLE_REV_MAJOR" -eq "3006" ]; then - ## DGM echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - ## DGM echo "Pin: version 3006.*" >> /etc/apt/preferences.d/salt-pin-1001 - ## DGM echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 - ## DGM elif [ "$STABLE_REV_MAJOR" -eq "3007" ]; then - ## DGM echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - ## DGM echo "Pin: version 3007.*" >> /etc/apt/preferences.d/salt-pin-1001 - ## DGM echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 - ## DGM fi if [ "$(echo "$STABLE_REV" | grep -E '^(3006|3007)$')" != "" ]; then echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 echo "Pin: version $STABLE_REV.*" >> /etc/apt/preferences.d/salt-pin-1001 @@ -3082,9 +3029,6 @@ __install_saltstack_ubuntu_repository() { } __install_saltstack_ubuntu_onedir_repository() { - # DGM debug - set -v - set -x echodebug "__install_saltstack_ubuntu_onedir_repository() entry" @@ -3123,16 +3067,6 @@ __install_saltstack_ubuntu_onedir_repository() { if [ "$ONEDIR_REV" != "latest" ]; then # latest is default - ## DGM ONEDIR_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) - ## DGM if [ "$ONEDIR_REV_MAJOR" -eq "3006" ]; then - ## DGM echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - ## DGM echo "Pin: version 3006.*" >> /etc/apt/preferences.d/salt-pin-1001 - ## DGM echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 - ## DGM elif [ "$ONEDIR_REV_MAJOR" -eq "3007" ]; then - ## DGM echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - ## DGM echo "Pin: version 3007.*" >> /etc/apt/preferences.d/salt-pin-1001 - ## DGM echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 - ## DGM fi if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 echo "Pin: version $ONEDIR_REV.*" >> /etc/apt/preferences.d/salt-pin-1001 @@ -3146,9 +3080,6 @@ __install_saltstack_ubuntu_onedir_repository() { } install_ubuntu_deps() { - # DGM debug - set -v - set -x echodebug "install_ubuntu_deps() entry" if [ "$_DISABLE_REPOS" -eq $BS_FALSE ]; then @@ -3210,9 +3141,6 @@ install_ubuntu_deps() { } install_ubuntu_stable_deps() { - # DGM debug - set -v - set -x echodebug "install_ubuntu_stable_deps() entry" @@ -3248,9 +3176,6 @@ install_ubuntu_stable_deps() { } install_ubuntu_git_deps() { - # DGM debug - set -v - set -x echodebug "install_ubuntu_git_deps() entry" @@ -3297,9 +3222,6 @@ install_ubuntu_git_deps() { } install_ubuntu_onedir_deps() { - # DGM debug - set -v - set -x if [ "$_START_DAEMONS" -eq $BS_FALSE ]; then echowarn "Not starting daemons on Debian based distributions is not working mostly because starting them is the default behaviour." @@ -3333,9 +3255,6 @@ install_ubuntu_onedir_deps() { } install_ubuntu_stable() { - # DGM debug - set -v - set -x __wait_for_apt apt-get update || return 1 @@ -3365,9 +3284,6 @@ install_ubuntu_stable() { } install_ubuntu_git() { - # DGM debug - set -v - set -x # Activate virtualenv before install if [ "${_VIRTUALENV_DIR}" != "null" ]; then @@ -3398,9 +3314,6 @@ install_ubuntu_git() { } install_ubuntu_onedir() { - # DGM debug - set -v - set -x __wait_for_apt apt-get update || return 1 @@ -3430,9 +3343,6 @@ install_ubuntu_onedir() { } install_ubuntu_stable_post() { - # DGM debug - set -v - set -x for fname in api master minion syndic; do # Skip salt-api since the service should be opt-in and not necessarily started on boot @@ -3443,7 +3353,6 @@ install_ubuntu_stable_post() { [ $fname = "master" ] && [ "$_INSTALL_MASTER" -eq $BS_FALSE ] && continue [ $fname = "syndic" ] && [ "$_INSTALL_SYNDIC" -eq $BS_FALSE ] && continue - ## if [ -f /bin/systemctl ]; then if [ "$_SYSTEMD_FUNCTIONAL" -eq $BS_TRUE ]; then # Using systemd /bin/systemctl is-enabled salt-$fname.service > /dev/null 2>&1 || ( @@ -3461,9 +3370,6 @@ install_ubuntu_stable_post() { } install_ubuntu_git_post() { - # DGM debug - set -v - set -x for fname in api master minion syndic; do # Skip if not meant to be installed @@ -3509,9 +3415,6 @@ install_ubuntu_git_post() { } install_ubuntu_restart_daemons() { - # DGM debug - set -v - set -x [ "$_START_DAEMONS" -eq $BS_FALSE ] && return @@ -3554,9 +3457,6 @@ install_ubuntu_restart_daemons() { } install_ubuntu_check_services() { - # DGM debug - set -v - set -x for fname in api master minion syndic; do # Skip salt-api since the service should be opt-in and not necessarily started on boot @@ -3610,27 +3510,12 @@ __install_saltstack_debian_repository() { # shellcheck disable=SC2086,SC2090 __apt_get_install_noinput ${__PACKAGES} || return 1 - ## SALTSTACK_DEBIAN_URL="${HTTP_VAL}://${_REPO_URL}/${_ONEDIR_DIR}/${__PY_VERSION_REPO}/debian/${DEBIAN_RELEASE}/${__REPO_ARCH}/${STABLE_REV}" - ## echo "$__REPO_ARCH_DEB $SALTSTACK_DEBIAN_URL $DEBIAN_CODENAME main" > "/etc/apt/sources.list.d/salt.list" - ## __apt_key_fetch "$SALTSTACK_DEBIAN_URL/SALT-PROJECT-GPG-PUBKEY-2023.gpg" || return 1 - ## __wait_for_apt apt-get update || return 1 - __fetch_url "/etc/apt/sources.list.d/salt.sources" "https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.sources" __apt_key_fetch "${HTTP_VAL}://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" || return 1 __wait_for_apt apt-get update || return 1 if [ "$STABLE_REV" != "latest" ]; then # latest is default - ## DGM STABLE_REV_MAJOR=$(echo "$STABLE_REV" | cut -d '.' -f 1) - ## DGM if [ "$STABLE_REV_MAJOR" -eq "3006" ]; then - ## DGM echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - ## DGM echo "Pin: version 3006.*" >> /etc/apt/preferences.d/salt-pin-1001 - ## DGM echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 - ## DGM elif [ "$STABLE_REV_MAJOR" -eq "3007" ]; then - ## DGM echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - ## DGM echo "Pin: version 3007.*" >> /etc/apt/preferences.d/salt-pin-1001 - ## DGM echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 - ## DGM fi if [ "$(echo "$STABLE_REV" | grep -E '^(3006|3007)$')" != "" ]; then echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 echo "Pin: version $STABLE_REV.*" >> /etc/apt/preferences.d/salt-pin-1001 @@ -3674,16 +3559,6 @@ __install_saltstack_debian_onedir_repository() { if [ "$ONEDIR_REV" != "latest" ]; then # latest is default - ## DGM ONEDIR_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) - ## DGM if [ "$ONEDIR_REV_MAJOR" -eq "3006" ]; then - ## DGM echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - ## DGM echo "Pin: version 3006.*" >> /etc/apt/preferences.d/salt-pin-1001 - ## DGM echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 - ## DGM elif [ "$ONEDIR_REV_MAJOR" -eq "3007" ]; then - ## DGM echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - ## DGM echo "Pin: version 3007.*" >> /etc/apt/preferences.d/salt-pin-1001 - ## DGM echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 - ## DGM fi if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 echo "Pin: version $ONEDIR_REV.*" >> /etc/apt/preferences.d/salt-pin-1001 @@ -3794,9 +3669,6 @@ install_debian_git_deps() { } install_debian_stable() { - # DGM debug - set -v - set -x __wait_for_apt apt-get update || return 1 @@ -3838,9 +3710,6 @@ install_debian_12_git_deps() { } install_debian_git() { - # DGM debug - set -v - set -x if [ -n "$_PY_EXE" ]; then _PYEXE=${_PY_EXE} @@ -3881,9 +3750,6 @@ install_debian_12_git() { } install_debian_onedir() { - # DGM debug - set -v - set -x __wait_for_apt apt-get update || return 1 @@ -4346,9 +4212,6 @@ install_fedora_onedir_post() { # CentOS Install Functions # __install_saltstack_rhel_onedir_repository() { - # DGM debug - set -v - set -x if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -ne 3 ]; then echoerror "Python version is no longer supported, only Python 3" @@ -4397,9 +4260,6 @@ __install_saltstack_rhel_onedir_repository() { } install_centos_stable_deps() { - # DGM debug - set -v - set -x if [ "$_UPGRADE_SYS" -eq $BS_TRUE ]; then yum -y update || return 1 @@ -4441,9 +4301,6 @@ install_centos_stable_deps() { } install_centos_stable() { - # DGM debug - set -v - set -x if [ "$(echo "$STABLE_REV" | grep -E '^(3006|3007)$')" != "" ]; then # Major version Salt, config and repo already setup @@ -4492,9 +4349,6 @@ install_centos_stable() { } install_centos_stable_post() { - # DGM debug - set -v - set -x SYSTEMD_RELOAD=$BS_FALSE @@ -4527,9 +4381,6 @@ install_centos_stable_post() { } install_centos_git_deps() { - # DGM debug - set -v - set -x # First try stable deps then fall back to onedir deps if that one fails # if we're installing on a Red Hat based host that doesn't have the classic @@ -4579,9 +4430,6 @@ install_centos_git_deps() { } install_centos_git() { - # DGM debug - set -v - set -x if [ "${_PY_EXE}" != "" ]; then _PYEXE=${_PY_EXE} @@ -4598,9 +4446,6 @@ install_centos_git() { } install_centos_git_post() { - # DGM debug - set -v - set -x SYSTEMD_RELOAD=$BS_FALSE @@ -4643,9 +4488,6 @@ install_centos_git_post() { } install_centos_onedir_deps() { - # DGM debug - set -v - set -x if [ "$_UPGRADE_SYS" -eq "$BS_TRUE" ]; then yum -y update || return 1 @@ -4686,9 +4528,6 @@ install_centos_onedir_deps() { } install_centos_onedir() { - # DGM debug - set -v - set -x if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then # Major version Salt, config and repo already setup @@ -4721,7 +4560,6 @@ install_centos_onedir() { # shellcheck disable=SC2086 dnf makecache || return 1 - echo "DGM install_centos_onedir, __PACKAGES '${__PACKAGES}'" dnf list salt-minion || return 1 __yum_install_noinput ${__PACKAGES} || return 1 @@ -4729,9 +4567,6 @@ install_centos_onedir() { } install_centos_onedir_post() { - # DGM debug - set -v - set -x SYSTEMD_RELOAD=$BS_FALSE @@ -4764,9 +4599,6 @@ install_centos_onedir_post() { } install_centos_restart_daemons() { - # DGM debug - set -v - set -x [ "$_START_DAEMONS" -eq $BS_FALSE ] && return @@ -4797,9 +4629,6 @@ install_centos_restart_daemons() { } install_centos_testing_deps() { - # DGM debug - set -v - set -x install_centos_stable_deps || return 1 return 0 @@ -4818,9 +4647,6 @@ install_centos_testing_post() { } install_centos_check_services() { - # DGM debug - set -v - set -x for fname in api master minion syndic; do # Skip salt-api since the service should be opt-in and not necessarily started on boot @@ -6392,9 +6218,6 @@ install_arch_linux_onedir_post() { # DESCRIPTION: Set _GENERIC_PKG_VERSION to the latest for RPM or latest for major version input #---------------------------------------------------------------------------------------------------------------------- __get_packagesite_onedir_latest() { - # DGM debug - set -v - set -x echodebug "Find latest rpm release from repository" @@ -7699,9 +7522,6 @@ daemons_running_voidlinux() { # DESCRIPTION: Set _PKG_VERSION to the latest for MacOS or latest for major version input #---------------------------------------------------------------------------------------------------------------------- __macosx_get_packagesite_onedir_latest() { - # DGM debug - set -v - set -x echodebug "Find latest MacOS release from repository" @@ -7727,9 +7547,6 @@ __macosx_get_packagesite_onedir_latest() { __macosx_get_packagesite_onedir() { - # DGM debug - set -v - set -x echodebug "Get package site for onedir from repository" From 9ef4b48bc53f3ad905a1019a7835863c6031b9c1 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Thu, 21 Nov 2024 12:19:48 -0700 Subject: [PATCH 105/173] Added testing for macos-14 --- .github/workflows/ci.yml | 17 +++++++++++++++++ .github/workflows/templates/generate.py | 3 +++ .github/workflows/test-macos.yml | 4 ++-- 3 files changed, 22 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3f57936a2..fab937bd2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -146,6 +146,22 @@ jobs: instances: '["stable-3006", "stable-3006-8", "stable-3007", "stable-3007-1", "latest"]' + macos-14: + name: macOS 14 + if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' + uses: ./.github/workflows/test-macos.yml + needs: + - lint + - generate-actions-workflow + with: + distro-slug: macos-14 + display-name: macOS 14 + container-slug: macOS 14 + timeout: 20 + runs-on: macos-14 + instances: '["stable-3006", "stable-3006-8", "stable-3007", "stable-3007-1", "latest"]' + + windows-2022: name: Windows 2022 @@ -234,6 +250,7 @@ jobs: - generate-actions-workflow - macos-12 - macos-13 + - macos-14 - windows-2022 - photon-5 - rockylinux-8 diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index 15b3861f4..8a312c5af 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -31,6 +31,7 @@ OSX = [ "macos-12", "macos-13", + "macos-14", ] # only test against current containers with systemd @@ -208,6 +209,7 @@ "ubuntu-2404": "Ubuntu 24.04", "macos-12": "macOS 12", "macos-13": "macOS 13", + "macos-14": "macOS 14", "windows-2022": "Windows 2022", } @@ -227,6 +229,7 @@ "ubuntu-2404": "systemd-ubuntu-24.04", "macos-12": "macos-12", "macos-13": "macos-13", + "macos-14": "macOS 14", "windows-2022": "windows-2022", } diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index 2779cf193..d59bb5f0c 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -36,8 +36,8 @@ on: jobs: Test: name: ${{ matrix.instance }} - ## runs-on: ${{ inputs.runs-on }} - runs-on: macos-13 + runs-on: ${{ inputs.runs-on }} + ## runs-on: macos-13 timeout-minutes: ${{ inputs.timeout }} strategy: fail-fast: false From bcffbb3affbb81c46177774f59fdb7dddd074ac1 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Thu, 21 Nov 2024 12:34:43 -0700 Subject: [PATCH 106/173] Updated for MacOS 14 and pip --- .github/workflows/test-linux.yml | 18 +++--------------- .github/workflows/test-macos.yml | 8 +++++++- 2 files changed, 10 insertions(+), 16 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 3d0047358..120f23eca 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -49,15 +49,11 @@ jobs: - name: Install Python Dependencies with pip breakage if: ${{ ( inputs.distro-slug == 'debian-11' ) || ( inputs.distro-slug == 'debian-12' ) || ( inputs.distro-slug == 'debian-13' ) || ( inputs.distro-slug == 'ubuntu-2404' ) }} run: | - echo "DGM pip test for distro-slug ${{ inputs.distro-slug }}" - echo "DGM doing break-system-packages for distro-slug ${{ inputs.distro-slug }}" python3 -m pip install --break-system-packages -r tests/requirements.txt - name: Install Python Dependencies without pip breakage if: ${{ ( inputs.distro-slug != 'debian-11' ) && ( inputs.distro-slug != 'debian-12' ) && ( inputs.distro-slug != 'debian-13' ) && ( inputs.distro-slug != 'ubuntu-2404' ) }} run: | - echo "DGM pip test for distro-slug ${{ inputs.distro-slug }}" - echo "DGM plain pip for distro-slug ${{ inputs.distro-slug }}" python3 -m pip install -r tests/requirements.txt - name: Get Version @@ -66,23 +62,15 @@ jobs: # It is used to install via bootstrap and in the test # The version is in the instance name # sed 1st - becomes space, 2nd - becomes dot - ## DGM needs cleanup - echo "matrix instance ,${{ matrix.instance }}," - vt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./') - vt_parm_ver=$(echo "$vt_parms" | awk -F ' ' '{print $2}') - echo "vt parms ,$vt_parms, vt_parms_ver ,$vt_parm_ver," + vt_parm_ver=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./' | awk -F ' ' '{print $2}') echo "SaltVersion=$vt_parm_ver" >> $GITHUB_ENV - name: Bootstrap Salt run: | # sed 1st - becomes space, 2nd - becomes dot - ## DGM needs cleanup - echo "matrix instance ,${{ matrix.instance }}," - bt_parms=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./') - bt_arg1=$(echo "$bt_parms" | awk -F ' ' '{print $1}') - bt_arg2=$(echo "$bt_parms" | awk -F ' ' '{print $2}') + bt_arg1=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./' | awk -F ' ' '{print $1}') + bt_arg2=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./' | awk -F ' ' '{print $2}') echo "bt parms ,$bt_parms, bt_arg1 ,$bt_arg1, bt_arg2 ,$bt_arg2," - ## DGM sudo sh -x ./bootstrap-salt.sh "$bt_arg1" "$bt_arg2" sh -x ./bootstrap-salt.sh "$bt_arg1" "$bt_arg2" - name: Test Bootstrap diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index d59bb5f0c..870fb5965 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -47,7 +47,13 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Install Python Dependencies + - name: Install Python Dependencies with pip breakage + if: ${{ ( inputs.distro-slug != 'macos-12' ) && ( inputs.distro-slug != 'macos-13' ) }} + run: | + python3 -m pip install --break-system-packages -r tests/requirements.txt + + - name: Install Python Dependencies without pip breakage + if: ${{ ( inputs.distro-slug == 'macos-12' ) || ( inputs.distro-slug == 'macos-13' ) }} run: | python3 -m pip install -r tests/requirements.txt From c7705ab73e4cb46fece55acb499369475a06ac91 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Thu, 21 Nov 2024 13:56:18 -0700 Subject: [PATCH 107/173] Add quiet option for wget, and hand-tested Photon usage --- bootstrap-salt.sh | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 16d91e6d7..897f1d5ca 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -6227,7 +6227,7 @@ __get_packagesite_onedir_latest() { cd ${generic_versions_tmpdir} || return 1 # leverage the windows directories since release Windows and Linux - wget -r -np -nH --exclude-directories=onedir,relenv,macos -x -l 1 "https://${_REPO_URL}/saltproject-generic/windows/" + wget -q -r -np -nH --exclude-directories=onedir,relenv,macos -x -l 1 "https://${_REPO_URL}/saltproject-generic/windows/" if [ "$#" -gt 0 ] && [ -n "$1" ]; then MAJOR_VER="$1" # shellcheck disable=SC2010 @@ -6564,9 +6564,6 @@ install_photon_onedir_deps() { install_photon_onedir() { - # DGM debug - set -v - set -x echodebug "install_photon_onedir() entry" @@ -6576,14 +6573,14 @@ install_photon_onedir() { if [ "$(echo "$STABLE_REV" | grep -E '^(3006|3007)$')" != "" ]; then # Major version Salt, config and repo already setup __get_packagesite_onedir_latest "$STABLE_REV" - MINOR_VER_STRG="$_GENERIC_PKG_VERSION" + MINOR_VER_STRG="-$_GENERIC_PKG_VERSION" elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then # Minor version Salt, need to add specific minor version MINOR_VER_STRG="-$STABLE_REV" else # default to latest version Salt, config and repo already setup __get_packagesite_onedir_latest - MINOR_VER_STRG="$_GENERIC_PKG_VERSION" + MINOR_VER_STRG="-$_GENERIC_PKG_VERSION" fi __PACKAGES="" @@ -7529,7 +7526,7 @@ __macosx_get_packagesite_onedir_latest() { macos_versions_tmpdir=$(mktemp -d) curr_pwd=$(pwd) cd ${macos_versions_tmpdir} || return 1 - wget -r -np -nH --exclude-directories=onedir,relenv,windows -x -l 1 "$SALT_MACOS_PKGDIR_URL/" + wget -q -r -np -nH --exclude-directories=onedir,relenv,windows -x -l 1 "$SALT_MACOS_PKGDIR_URL/" if [ "$#" -gt 0 ] && [ -n "$1" ]; then MAJOR_VER="$1" # shellcheck disable=SC2010 From edaa710bcb778d01eb6e2548f82f9482f32e37ae Mon Sep 17 00:00:00 2001 From: David Murphy Date: Thu, 21 Nov 2024 13:57:55 -0700 Subject: [PATCH 108/173] Updated date for bootstrap script --- bootstrap-salt.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 897f1d5ca..c8f0a7ecb 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -26,7 +26,7 @@ #====================================================================================================================== set -o nounset # Treat unset variables as an error -__ScriptVersion="2024.11.19" +__ScriptVersion="2024.11.21" __ScriptName="bootstrap-salt.sh" __ScriptFullName="$0" From f2abbd4e2f8fec1e717e34ec70310d96ef14529e Mon Sep 17 00:00:00 2001 From: Salt Project Packaging Date: Thu, 21 Nov 2024 22:10:08 +0000 Subject: [PATCH 109/173] Update develop branch for the v2024.11.21 release --- CHANGELOG.md | 14 ++++++++++++++ bootstrap-salt.ps1 | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 725022f71..2a70b9fa1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,17 @@ +# v2024.11.21 + +## What's Changed + +- Make universal repo url by @twangboy in https://github.com/saltstack/salt-bootstrap/pull/2045 +- Update bootstrap shell CI/CD to utilize containers and clean up by @dmurphy18 in https://github.com/saltstack/salt-bootstrap/pull/2039 +- \[WIP\] hwclock check missing additional condition on major version by @ruslantum in https://github.com/saltstack/salt-bootstrap/pull/2048 + +## New Contributors + +- @ruslantum made their first contribution in https://github.com/saltstack/salt-bootstrap/pull/2048 + +**Full Changelog**: https://github.com/saltstack/salt-bootstrap/compare/v2024.11.07...v2024.11.21 + # v2024.11.07 ## What's Changed diff --git a/bootstrap-salt.ps1 b/bootstrap-salt.ps1 index 350ea84ab..815a0eb74 100644 --- a/bootstrap-salt.ps1 +++ b/bootstrap-salt.ps1 @@ -107,7 +107,7 @@ if ($help) { exit 0 } -$__ScriptVersion = "2024.11.07" +$__ScriptVersion = "2024.11.21" $ScriptName = $myInvocation.MyCommand.Name # We'll check for the Version next, because it also has no requirements From 6ac1dcddaa71ca36844d8ba61299016a1f8d7acd Mon Sep 17 00:00:00 2001 From: Salt Project Packaging Date: Thu, 21 Nov 2024 22:10:53 +0000 Subject: [PATCH 110/173] Update README.rst with 2024.11.21 release sha256sum --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index 7c524e3a7..adbe4524b 100644 --- a/README.rst +++ b/README.rst @@ -37,6 +37,7 @@ sum** of the downloaded ``bootstrap-salt.sh`` file. The SHA256 sum of the ``bootstrap-salt.sh`` file, per release, is: +- 2024.11.21: ``ddf624c3a94d721da3f7629402a6c7ecc9dd96d13c1ead2a626314e97cee982a`` - 2024.11.07: ``70a9783649e129985563d1a86cf28b8984499643e62ae1dc47dc008bd204fcbb`` - 2024.09.24: ``88e4e4cad4b115a7b721dd9c21d5ee5df390b5b73b63de48f99399146f43f371`` - 2024.07.23: ``7212b6b497b5c3d2bf15bfe5301625ec7bc1bf3e2949cd47d8e2073614935bf8`` From 0e00e1d3ca51d7c3d448c34c590a96c486306609 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Fri, 22 Nov 2024 12:38:27 -0700 Subject: [PATCH 111/173] Updated RUL for downloading the bootstrap script etc to GitHub repository --- README.rst | 68 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 35 insertions(+), 33 deletions(-) diff --git a/README.rst b/README.rst index adbe4524b..c52b6847e 100644 --- a/README.rst +++ b/README.rst @@ -59,8 +59,9 @@ If you're looking for a *one-liner* to install Salt, please scroll to the bottom instructions for `Installing via an Insecure One-Liner`_. There are also .sha256 files for verifying against in the repo for the stable branch. You can also -get the correct sha256 sum for the stable release from https://bootstrap.saltproject.io/sha256 and -https://winbootstrap.saltproject.io/sha256 +get the correct sha256 sum for the stable release from +https://github.com/saltstack/salt-bootstrap/releases/latest/download/bootstrap-salt.sh.sha256 and +https://github.com/saltstack/salt-bootstrap/releases/latest/download/bootstrap-salt.ps1.sha256 Contributing ------------ @@ -86,9 +87,9 @@ To view the latest options and descriptions for ``salt-bootstrap``, use ``-h`` a - stable Install latest stable release. This is the default install type - stable [branch] Install latest version on a branch. Only supported - for packages available at repo.saltproject.io + for packages available at packages.broadcom.com - stable [version] Install a specific version. Only supported for - packages available at repo.saltproject.io + packages available at packages.broadcom.com To pin a 3xxx minor version, specify it as 3xxx.0 - testing RHEL-family specific: configure EPEL testing repo - git Install from the head of the master branch @@ -171,8 +172,8 @@ To view the latest options and descriptions for ``salt-bootstrap``, use ``-h`` a on the system. -R Specify a custom repository URL. Assumes the custom repository URL points to a repository that mirrors Salt packages located at - repo.saltproject.io. The option passed with -R replaces the - "repo.saltproject.io". If -R is passed, -r is also set. Currently only + packages.broadcom.com. The option passed with -R replaces the + "packages.broadcom.com". If -R is passed, -r is also set. Currently only works on CentOS/RHEL and Debian based distributions. -J Replace the Master config file with data passed in as a JSON string. If a Master config file is found, a reasonable effort will be made to save @@ -208,35 +209,35 @@ If you want to install a package of a specific release version, from the Salt Pr .. code:: console - curl -o bootstrap-salt.sh -L https://bootstrap.saltproject.io + curl -o bootstrap-salt.sh -L https://github.com/saltstack/salt-bootstrap/releases/latest/download/bootstrap-salt.sh sudo sh bootstrap-salt.sh -P stable 3006.1 If you want to install a specific release version, based on the Git tags: .. code:: console - curl -o bootstrap-salt.sh -L https://bootstrap.saltproject.io + curl -o bootstrap-salt.sh -L https://github.com/saltstack/salt-bootstrap/releases/latest/download/bootstrap-salt.sh sudo sh bootstrap-salt.sh git v3006.1 Using ``curl`` to install latest development version from GitHub: .. code:: console - curl -o bootstrap-salt.sh -L https://bootstrap.saltproject.io - sudo sh bootstrap-salt.sh git master + curl -o bootstrap-salt.sh -L https://github.com/saltstack/salt-bootstrap/releases/latest/download/bootstrap-salt.sh sudo sh + bootstrap-salt.sh git master To install a specific branch from a Git fork: .. code:: console - curl -o bootstrap-salt.sh -L https://bootstrap.saltproject.io + curl -o bootstrap-salt.sh -L https://github.com/saltstack/salt-bootstrap/releases/latest/download/bootstrap-salt.sh sudo sh bootstrap-salt.sh -g https://github.com/myuser/salt.git git mybranch If all you want is to install a ``salt-master`` using latest Git: .. code:: console - curl -o bootstrap-salt.sh -L https://bootstrap.saltproject.io + curl -o bootstrap-salt.sh -L https://github.com/saltstack/salt-bootstrap/releases/latest/download/bootstrap-salt.sh sudo sh bootstrap-salt.sh -M -N git master If your host has Internet access only via HTTP proxy, from the Salt Project repo: @@ -244,7 +245,7 @@ If your host has Internet access only via HTTP proxy, from the Salt Project repo .. code:: console PROXY='http://user:password@myproxy.example.com:3128' - curl -o bootstrap-salt.sh -L -x "$PROXY" https://bootstrap.saltproject.io + curl -o bootstrap-salt.sh -L -x "$PROXY" https://github.com/saltstack/salt-bootstrap/releases/latest/download/bootstrap-salt.sh sudo sh bootstrap-salt.sh -P -H "$PROXY" stable If your host has Internet access only via HTTP proxy, installing via Git: @@ -252,7 +253,8 @@ If your host has Internet access only via HTTP proxy, installing via Git: .. code:: console PROXY='http://user:password@myproxy.example.com:3128' - curl -o bootstrap-salt.sh -L -x "$PROXY" https://bootstrap.saltproject.io + curl -o bootstrap-salt.sh -L -x "$PROXY" + https://github.com/saltstack/salt-bootstrap/releases/latest/download/bootstrap-salt.sh sudo sh bootstrap-salt.sh -H "$PROXY" git @@ -263,22 +265,23 @@ Using ``wget`` to install your distribution's stable packages: .. code:: console - wget -O bootstrap-salt.sh https://bootstrap.saltproject.io + wget -O bootstrap-salt.sh + https://github.com/saltstack/salt-bootstrap/releases/latest/download/bootstrap-salt.sh sudo sh bootstrap-salt.sh Installing a specific version from git using ``wget``: .. code:: console - wget -O bootstrap-salt.sh https://bootstrap.saltproject.io - sudo sh bootstrap-salt.sh git v3004.1 + wget -O bootstrap-salt.sh https://github.com/saltstack/salt-bootstrap/releases/latest/download/bootstrap-salt.sh + sudo sh bootstrap-salt.sh git v3006.8 Installing a specific version package from the Salt Project repo using ``wget``: .. code:: console - wget -O bootstrap-salt.sh https://bootstrap.saltproject.io - sudo sh bootstrap-salt.sh -P stable 3006.1 + wget -O bootstrap-salt.sh https://github.com/saltstack/salt-bootstrap/releases/latest/download/bootstrap-salt.sh + sudo sh bootstrap-salt.sh -P stable 3006.8 **NOTE** @@ -293,14 +296,14 @@ If you already have Python installed, ``python 3.10``, then it's as easy as: .. code:: console - python -m urllib "https://bootstrap.saltproject.io" > bootstrap-salt.sh + python -m urllib "https://github.com/saltstack/salt-bootstrap/releases/latest/download/bootstrap-salt.sh" > bootstrap-salt.sh sudo sh bootstrap-salt.sh -P stable 3006.1 With python version 3: .. code:: console - python3 -c 'import urllib.request; print(urllib.request.urlopen("https://bootstrap.saltproject.io").read().decode("ascii"))' > bootstrap-salt.sh + python3 -c 'import urllib.request; print(urllib.request.urlopen("https://github.com/saltstack/salt-bootstrap/releases/latest/download/bootstrap-salt.sh").read().decode("ascii"))' > bootstrap-salt.sh sudo sh bootstrap-salt.sh git v3006.1 Note: Python 2.x is no longer supported given it reached it's End-Of-Life Jan. 1st, 2020 @@ -323,25 +326,26 @@ Installing the latest stable release of Salt (default): .. code:: console - curl -L https://bootstrap.saltproject.io | sudo sh + curl -L https://github.com/saltstack/salt-bootstrap/releases/latest/download/bootstrap-salt.sh | sudo sh Using ``wget`` to install your distribution's stable packages: .. code:: console - wget -O - https://bootstrap.saltproject.io | sudo sh + wget -O - https://github.com/saltstack/salt-bootstrap/releases/latest/download/bootstrap-salt.sh | sudo sh Installing a target version package of Salt from the Salt Project repo: .. code:: console - curl -L https://bootstrap.saltproject.io | sudo sh -s -- stable 3006.8 + curl -L https://github.com/saltstack/salt-bootstrap/releases/latest/download/bootstrap-salt.sh | sudo sh -s -- stable 3006.8 Installing the latest master branch of Salt from git: .. code:: console - curl -L https://bootstrap.saltproject.io | sudo sh -s -- git master + curl -L https://github.com/saltstack/salt-bootstrap/releases/latest/download/bootstrap-salt.sh | + sudo sh -s -- git master Note: use of git is recommended for development environments, for example: testing new features of Salt which have not yet been released. @@ -356,9 +360,7 @@ Using ``PowerShell`` to install latest stable version: .. code:: powershell [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]'Tls12' - Invoke-WebRequest -Uri https://winbootstrap.saltproject.io -OutFile "$env:TEMP\bootstrap-salt.ps1" - Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser - & "$env:TEMP\bootstrap-salt.ps1" + Invoke-WebRequest -Uri https://github.com/saltstack/salt-bootstrap/releases/latest/download/bootstrap-salt.ps1 -OutFile "$env:TEMP\bootstrap-salt.ps1" Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser & "$env:TEMP\bootstrap-salt.ps1" Display information about the install script parameters: @@ -370,7 +372,7 @@ Using ``cygwin`` to install latest stable version: .. code:: console - curl -o bootstrap-salt.ps1 -L https://winbootstrap.saltproject.io + curl -o bootstrap-salt.ps1 -L https://github.com/saltstack/salt-bootstrap/releases/latest/download/bootstrap-salt.ps1 "/cygdrive/c/WINDOWS/System32/WindowsPowerShell/v1.0/powershell.exe" -NoProfile -InputFormat None -ExecutionPolicy Bypass -Command "[System.Net.ServicePointManager]::SecurityProtocol = 3072; iex ./bootstrap-salt.ps1" @@ -380,13 +382,13 @@ Supported Operating Systems The salt-bootstrap script officially supports the distributions outlined in `Salt's Supported Operating Systems `_ -document, (BSD-based OSs, Solaris and AIX are no longer -supported). The operating systems listed below should reflect this document but may become out of -date. If an operating system is listed below, but is not listed on the official supported operating +document, (BSD-based OSs, Solaris and AIX are no longer supported). +The operating systems listed below should reflect this document but may become out of date. +If an operating system is listed below, but is not listed on the official supported operating systems document, the level of support is "best-effort". Since Salt is written in Python, the packages available from the `Salt Project's repository -`_ are +`_ are CPU architecture independent and could be installed on any hardware supported by Linux kernel. However, the Salt Project does package Salt's binary dependencies only for ``x86_64`` (``amd64``) and ``AArch64`` (``arm64``). From e894ebce3e57333b8a74bf4c36fd2fb107e9c1cf Mon Sep 17 00:00:00 2001 From: David Murphy Date: Fri, 22 Nov 2024 14:12:30 -0700 Subject: [PATCH 112/173] Update README.rst Co-authored-by: Derek Ardolf --- README.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.rst b/README.rst index c52b6847e..fe018f655 100644 --- a/README.rst +++ b/README.rst @@ -253,8 +253,7 @@ If your host has Internet access only via HTTP proxy, installing via Git: .. code:: console PROXY='http://user:password@myproxy.example.com:3128' - curl -o bootstrap-salt.sh -L -x "$PROXY" - https://github.com/saltstack/salt-bootstrap/releases/latest/download/bootstrap-salt.sh + curl -o bootstrap-salt.sh -L -x "$PROXY" https://github.com/saltstack/salt-bootstrap/releases/latest/download/bootstrap-salt.sh sudo sh bootstrap-salt.sh -H "$PROXY" git From 6c22d58daccc7fd3aae6818c64126fbf972775f1 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Fri, 22 Nov 2024 14:12:46 -0700 Subject: [PATCH 113/173] Update README.rst Co-authored-by: Derek Ardolf --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index fe018f655..1d65414b3 100644 --- a/README.rst +++ b/README.rst @@ -223,8 +223,8 @@ Using ``curl`` to install latest development version from GitHub: .. code:: console - curl -o bootstrap-salt.sh -L https://github.com/saltstack/salt-bootstrap/releases/latest/download/bootstrap-salt.sh sudo sh - bootstrap-salt.sh git master + curl -o bootstrap-salt.sh -L https://github.com/saltstack/salt-bootstrap/releases/latest/download/bootstrap-salt.sh + sudo sh bootstrap-salt.sh git master To install a specific branch from a Git fork: From 360a923190a751310813dc0ef4a94e5671b3db67 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Fri, 22 Nov 2024 14:13:07 -0700 Subject: [PATCH 114/173] Update README.rst Co-authored-by: Derek Ardolf --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 1d65414b3..644538c51 100644 --- a/README.rst +++ b/README.rst @@ -325,7 +325,7 @@ Installing the latest stable release of Salt (default): .. code:: console - curl -L https://github.com/saltstack/salt-bootstrap/releases/latest/download/bootstrap-salt.sh | sudo sh + curl -L https://github.com/saltstack/salt-bootstrap/releases/latest/download/bootstrap-salt.sh | sudo sh Using ``wget`` to install your distribution's stable packages: From d0d73fa9f4d27751dad5c85d13c85561cce23b80 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Fri, 22 Nov 2024 14:13:17 -0700 Subject: [PATCH 115/173] Update README.rst Co-authored-by: Derek Ardolf --- README.rst | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 644538c51..e387dc23e 100644 --- a/README.rst +++ b/README.rst @@ -343,8 +343,7 @@ Installing the latest master branch of Salt from git: .. code:: console - curl -L https://github.com/saltstack/salt-bootstrap/releases/latest/download/bootstrap-salt.sh | - sudo sh -s -- git master + curl -L https://github.com/saltstack/salt-bootstrap/releases/latest/download/bootstrap-salt.sh | sudo sh -s -- git master Note: use of git is recommended for development environments, for example: testing new features of Salt which have not yet been released. From b5d70f024c265725b23d43a8a997abf2d501b35d Mon Sep 17 00:00:00 2001 From: David Murphy Date: Fri, 22 Nov 2024 14:13:39 -0700 Subject: [PATCH 116/173] Update README.rst Co-authored-by: Derek Ardolf --- README.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index e387dc23e..39febeccb 100644 --- a/README.rst +++ b/README.rst @@ -358,7 +358,9 @@ Using ``PowerShell`` to install latest stable version: .. code:: powershell [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]'Tls12' - Invoke-WebRequest -Uri https://github.com/saltstack/salt-bootstrap/releases/latest/download/bootstrap-salt.ps1 -OutFile "$env:TEMP\bootstrap-salt.ps1" Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser & "$env:TEMP\bootstrap-salt.ps1" + Invoke-WebRequest -Uri https://github.com/saltstack/salt-bootstrap/releases/latest/download/bootstrap-salt.ps1 -OutFile "$env:TEMP\bootstrap-salt.ps1" + Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser + & "$env:TEMP\bootstrap-salt.ps1" Display information about the install script parameters: From c2071bd44321aada4502256b4fe3bf746b6ef059 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Fri, 22 Nov 2024 14:13:48 -0700 Subject: [PATCH 117/173] Update README.rst Co-authored-by: Derek Ardolf --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 39febeccb..3d3bc8fab 100644 --- a/README.rst +++ b/README.rst @@ -388,7 +388,7 @@ If an operating system is listed below, but is not listed on the official suppor systems document, the level of support is "best-effort". Since Salt is written in Python, the packages available from the `Salt Project's repository -`_ are +`_ are CPU architecture independent and could be installed on any hardware supported by Linux kernel. However, the Salt Project does package Salt's binary dependencies only for ``x86_64`` (``amd64``) and ``AArch64`` (``arm64``). From 320b3b787e75a63521c8ed515fd2076fe2c65936 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Fri, 22 Nov 2024 15:02:06 -0700 Subject: [PATCH 118/173] Fixed help output and README --- README.rst | 6 ++---- bootstrap-salt.sh | 10 ++++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/README.rst b/README.rst index 3d3bc8fab..929d93315 100644 --- a/README.rst +++ b/README.rst @@ -264,8 +264,7 @@ Using ``wget`` to install your distribution's stable packages: .. code:: console - wget -O bootstrap-salt.sh - https://github.com/saltstack/salt-bootstrap/releases/latest/download/bootstrap-salt.sh + wget -O bootstrap-salt.sh https://github.com/saltstack/salt-bootstrap/releases/latest/download/bootstrap-salt.sh sudo sh bootstrap-salt.sh Installing a specific version from git using ``wget``: @@ -359,8 +358,7 @@ Using ``PowerShell`` to install latest stable version: [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]'Tls12' Invoke-WebRequest -Uri https://github.com/saltstack/salt-bootstrap/releases/latest/download/bootstrap-salt.ps1 -OutFile "$env:TEMP\bootstrap-salt.ps1" - Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser - & "$env:TEMP\bootstrap-salt.ps1" + Set-ExecutionPolicy -ExecutionPolicy RemoteSigned -Scope CurrentUser & "$env:TEMP\bootstrap-salt.ps1" Display information about the install script parameters: diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index da3c2a5e9..5e17fc15b 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -26,7 +26,7 @@ #====================================================================================================================== set -o nounset # Treat unset variables as an error -__ScriptVersion="2024.11.21" +__ScriptVersion="2024.11.22" __ScriptName="bootstrap-salt.sh" __ScriptFullName="$0" @@ -314,6 +314,9 @@ __usage() { Usage : ${__ScriptName} [options] [install-type-args] + + Usage : bootstrap-salt.sh [options] [install-type-args] + Installation types: - stable Install latest stable release. This is the default install type @@ -351,7 +354,6 @@ __usage() { - ${__ScriptName} onedir_rc - ${__ScriptName} onedir_rc 3008 - Options: -a Pip install all Python pkg dependencies for Salt. Requires -V to install all pip pkgs into the virtualenv. @@ -373,7 +375,7 @@ __usage() { -f Force shallow cloning for git installations. This may result in an "n/a" in the version number. -F Allow copied files to overwrite existing (config, init.d, etc) - -g Salt Git repository URL. Default: ${_SALTSTACK_REPO_URL} + -g Salt Git repository URL. Default: https://github.com/saltstack/salt.git -h Display this message -H Use the specified HTTP proxy for all download URLs (including https://). For example: http://myproxy.example.com:3128 @@ -419,7 +421,7 @@ __usage() { "packages.broadcom.com". If -R is passed, -r is also set. Currently only works on CentOS/RHEL and Debian based distributions and macOS. -s Sleep time used when waiting for daemons to start, restart and when - checking for the services running. Default: ${__DEFAULT_SLEEP} + checking for the services running. Default: 3 -S Also install salt-syndic -r Disable all repository configuration performed by this script. This option assumes all necessary repository configuration is already present From 344b942c289c970da651456f096016574b93cc1c Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 25 Nov 2024 11:35:17 -0700 Subject: [PATCH 119/173] Fixed git build issue with deps on RedHat family, and added more systemd containers to test against --- .github/workflows/ci.yml | 68 ++++++++++++++++++++++++- .github/workflows/templates/generate.py | 39 +++++++------- bootstrap-salt.sh | 55 ++++---------------- 3 files changed, 96 insertions(+), 66 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fab937bd2..2464d773b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -180,6 +180,66 @@ jobs: + amazonlinux-2: + name: Amazon 2 + if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' + uses: ./.github/workflows/test-linux.yml + needs: + - lint + - generate-actions-workflow + with: + distro-slug: amazonlinux-2 + display-name: Amazon 2 + container-slug: systemd-amazonlinux-2 + timeout: 20 + instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' + + + amazonlinux-2023: + name: Amazon 2023 + if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' + uses: ./.github/workflows/test-linux.yml + needs: + - lint + - generate-actions-workflow + with: + distro-slug: amazonlinux-2023 + display-name: Amazon 2023 + container-slug: systemd-amazonlinux-2023 + timeout: 20 + instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' + + + debian-11: + name: Debian 11 + if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' + uses: ./.github/workflows/test-linux.yml + needs: + - lint + - generate-actions-workflow + with: + distro-slug: debian-11 + display-name: Debian 11 + container-slug: systemd-debian-11 + timeout: 20 + instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' + + + debian-12: + name: Debian 12 + if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' + uses: ./.github/workflows/test-linux.yml + needs: + - lint + - generate-actions-workflow + with: + distro-slug: debian-12 + display-name: Debian 12 + container-slug: systemd-debian-12 + timeout: 20 + instances: '["git-3006", "stable-3007", "git-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' + + photon-5: name: Photon OS 5 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' @@ -222,7 +282,7 @@ jobs: display-name: Rocky Linux 9 container-slug: systemd-rockylinux-9 timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' + instances: '["stable-3006", "git-3006", "onedir-3006", "stable-3006-8", "stable-3007", "git-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' ubuntu-2204: @@ -237,7 +297,7 @@ jobs: display-name: Ubuntu 22.04 container-slug: systemd-ubuntu-22.04 timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' + instances: '["stable-3006", "git-3006", "onedir-3006", "stable-3006-8", "stable-3007", "git-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' set-pipeline-exit-status: @@ -252,6 +312,10 @@ jobs: - macos-13 - macos-14 - windows-2022 + - amazonlinux-2 + - amazonlinux-2023 + - debian-11 + - debian-12 - photon-5 - rockylinux-8 - rockylinux-9 diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index 8a312c5af..bf88486d1 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -18,6 +18,10 @@ # "ubuntu-2004", # "ubuntu-2404", LINUX_DISTROS = [ + "amazonlinux-2", + "amazonlinux-2023", + "debian-11", + "debian-12", "photon-5", "rockylinux-8", "rockylinux-9", @@ -36,17 +40,16 @@ # only test against current containers with systemd # will add these when they become available with systemd -# "amazonlinux-2", -# "amazonlinux-2023", -# "centos-stream9", -# "debian-11", -# "debian-12", # "debian-13", # "fedora-40", # "photon-4", # "ubuntu-2004", # "ubuntu-2404", STABLE_DISTROS = [ + "amazonlinux-2", + "amazonlinux-2023", + "debian-11", + "debian-12", "photon-5", "rockylinux-8", "rockylinux-9", @@ -55,16 +58,16 @@ # only test against current containers with systemd # will add these when they become available with systemd -# "amazonlinux-2", -# "amazonlinux-2023", -# "debian-11", -# "debian-12", # "debian-13", # "fedora-40", # "photon-4", # "ubuntu-2004", # "ubuntu-2404", ONEDIR_DISTROS = [ + "amazonlinux-2", + "amazonlinux-2023", + "debian-11", + "debian-12", "photon-5", "rockylinux-8", "rockylinux-9", @@ -75,12 +78,12 @@ # will add these when they become available with systemd # "amazonlinux-2", # "amazonlinux-2023", -# "debian-12", # "photon-4", # "photon-5", -# "rockylinux-9", +# "rockylinux-8", # "ubuntu-2404", ONEDIR_RC_DISTROS = [ + "debian-12", "photon-5", "rockylinux-9", "ubuntu-2204", @@ -97,41 +100,41 @@ "photon-5", ] +# "debian-12", +# "rockylinux-9", +# "ubuntu-2204", BLACKLIST_GIT_3006 = [ "amazonlinux-2", "amazonlinux-2023", "debian-11", - "debian-12", "fedora-40", "photon-4", "photon-5", - "rockylinux-9", "ubuntu-2004", - "ubuntu-2204", "ubuntu-2404", ] +# "debian-12", +# "rockylinux-9", +# "ubuntu-2204", BLACKLIST_GIT_3007 = [ "amazonlinux-2", "amazonlinux-2023", "debian-11", - "debian-12", "debian-13", "fedora-40", "photon-4", "photon-5", - "rockylinux-9", "ubuntu-2004", - "ubuntu-2204", "ubuntu-2404", ] +## "rockylinux-9", BLACKLIST_GIT_MASTER = [ "amazonlinux-2", "amazonlinux-2023", "photon-4", "photon-5", - "rockylinux-9", ] SALT_VERSIONS = [ diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 5e17fc15b..103ea07ef 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -26,7 +26,7 @@ #====================================================================================================================== set -o nounset # Treat unset variables as an error -__ScriptVersion="2024.11.22" +__ScriptVersion="2024.11.21" __ScriptName="bootstrap-salt.sh" __ScriptFullName="$0" @@ -314,9 +314,6 @@ __usage() { Usage : ${__ScriptName} [options] [install-type-args] - - Usage : bootstrap-salt.sh [options] [install-type-args] - Installation types: - stable Install latest stable release. This is the default install type @@ -354,6 +351,7 @@ __usage() { - ${__ScriptName} onedir_rc - ${__ScriptName} onedir_rc 3008 + Options: -a Pip install all Python pkg dependencies for Salt. Requires -V to install all pip pkgs into the virtualenv. @@ -375,7 +373,7 @@ __usage() { -f Force shallow cloning for git installations. This may result in an "n/a" in the version number. -F Allow copied files to overwrite existing (config, init.d, etc) - -g Salt Git repository URL. Default: https://github.com/saltstack/salt.git + -g Salt Git repository URL. Default: ${_SALTSTACK_REPO_URL} -h Display this message -H Use the specified HTTP proxy for all download URLs (including https://). For example: http://myproxy.example.com:3128 @@ -421,7 +419,7 @@ __usage() { "packages.broadcom.com". If -R is passed, -r is also set. Currently only works on CentOS/RHEL and Debian based distributions and macOS. -s Sleep time used when waiting for daemons to start, restart and when - checking for the services running. Default: 3 + checking for the services running. Default: ${__DEFAULT_SLEEP} -S Also install salt-syndic -r Disable all repository configuration performed by this script. This option assumes all necessary repository configuration is already present @@ -546,8 +544,8 @@ __exit_cleanup() { echodebug "Cleaning up the Salt Temporary Git Repository" # shellcheck disable=SC2164 cd "${__SALT_GIT_CHECKOUT_PARENT_DIR}" - rm -rf "${_SALT_GIT_CHECKOUT_DIR}" - #rm -rf "${_SALT_GIT_CHECKOUT_DIR}/deps" + rm -fR "${_SALT_GIT_CHECKOUT_DIR}" + #rm -fR "${_SALT_GIT_CHECKOUT_DIR}/deps" else echowarn "Not cleaning up the Salt Temporary git repository on request" echowarn "Note that if you intend to re-run this script using the git approach, you might encounter some issues" @@ -603,7 +601,6 @@ fi echoinfo "Running version: ${__ScriptVersion}" echoinfo "Executed by: ${CALLER}" echoinfo "Command line: '${__ScriptFullName} ${__ScriptArgs}'" -echowarn "Running the unstable version of ${__ScriptName}" # Defaults STABLE_REV="latest" @@ -2737,44 +2734,10 @@ __install_salt_from_repo() { __check_pip_allowed echodebug "Installed pip version: $(${_pip_cmd} --version)" + echodebug "Upgrading pip to latest, running '${_pip_cmd} install ${_PIP_INSTALL_ARGS} -U pip>=${_MINIMUM_PIP_VERSION}'" + ${_pip_cmd} install ${_PIP_INSTALL_ARGS} -v -U "pip>=${_MINIMUM_PIP_VERSION}" - CHECK_PIP_VERSION_SCRIPT=$(cat << EOM -import sys -try: - import pip - installed_pip_version=tuple([int(part.strip()) for part in pip.__version__.split('.') if part.isdigit()]) - desired_pip_version=($(echo ${_MINIMUM_PIP_VERSION} | sed 's/\./, /g' )) - if installed_pip_version < desired_pip_version: - print('Desired pip version {!r} > Installed pip version {!r}'.format('.'.join(map(str, desired_pip_version)), '.'.join(map(str, installed_pip_version)))) - sys.exit(1) - print('Desired pip version {!r} < Installed pip version {!r}'.format('.'.join(map(str, desired_pip_version)), '.'.join(map(str, installed_pip_version)))) - sys.exit(0) -except ImportError: - print('Failed to import pip') - sys.exit(1) -EOM -) - if ! ${_py_exe} -c "$CHECK_PIP_VERSION_SCRIPT"; then - # Upgrade pip to at least 1.2 which is when we can start using "python3 -m pip" - echodebug "Running '${_pip_cmd} install ${_PIP_INSTALL_ARGS} pip>=${_MINIMUM_PIP_VERSION}'" - ${_pip_cmd} install ${_PIP_INSTALL_ARGS} -v "pip>=${_MINIMUM_PIP_VERSION}" - sleep 1 - echodebug "PATH: ${PATH}" - _pip_cmd="pip${_py_version}" - if ! __check_command_exists "${_pip_cmd}"; then - echodebug "The pip binary '${_pip_cmd}' was not found in PATH" - _pip_cmd="pip$(echo "${_py_version}" | cut -c -1)" - if ! __check_command_exists "${_pip_cmd}"; then - echodebug "The pip binary '${_pip_cmd}' was not found in PATH" - _pip_cmd="pip" - if ! __check_command_exists "${_pip_cmd}"; then - echoerror "Unable to find a pip binary" - return 1 - fi - fi - fi - echodebug "Installed pip version: $(${_pip_cmd} --version)" - fi + echodebug "Upgraded pip version: $(${_pip_cmd} --version)" _setuptools_dep="setuptools>=${_MINIMUM_SETUPTOOLS_VERSION},<${_MAXIMUM_SETUPTOOLS_VERSION}" if [ "$_PY_MAJOR_VERSION" -ne 3 ]; then From 86d1bb96629fbfcdc3f26af1e5f82dbe4c7e660b Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 25 Nov 2024 12:15:39 -0700 Subject: [PATCH 120/173] Added missing procps for Debian / Ubuntu container, and reduced containers used --- .github/workflows/ci.yml | 48 ------------------------- .github/workflows/templates/generate.py | 18 +++++----- bootstrap-salt.sh | 16 +++++++-- 3 files changed, 22 insertions(+), 60 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2464d773b..a94508b54 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -180,51 +180,6 @@ jobs: - amazonlinux-2: - name: Amazon 2 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: amazonlinux-2 - display-name: Amazon 2 - container-slug: systemd-amazonlinux-2 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' - - - amazonlinux-2023: - name: Amazon 2023 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: amazonlinux-2023 - display-name: Amazon 2023 - container-slug: systemd-amazonlinux-2023 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' - - - debian-11: - name: Debian 11 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: debian-11 - display-name: Debian 11 - container-slug: systemd-debian-11 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - - debian-12: name: Debian 12 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' @@ -312,9 +267,6 @@ jobs: - macos-13 - macos-14 - windows-2022 - - amazonlinux-2 - - amazonlinux-2023 - - debian-11 - debian-12 - photon-5 - rockylinux-8 diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index bf88486d1..097bb9464 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -18,9 +18,6 @@ # "ubuntu-2004", # "ubuntu-2404", LINUX_DISTROS = [ - "amazonlinux-2", - "amazonlinux-2023", - "debian-11", "debian-12", "photon-5", "rockylinux-8", @@ -40,15 +37,15 @@ # only test against current containers with systemd # will add these when they become available with systemd +# "amazonlinux-2", +# "amazonlinux-2023", +# "debian-11", # "debian-13", # "fedora-40", # "photon-4", # "ubuntu-2004", # "ubuntu-2404", STABLE_DISTROS = [ - "amazonlinux-2", - "amazonlinux-2023", - "debian-11", "debian-12", "photon-5", "rockylinux-8", @@ -58,15 +55,15 @@ # only test against current containers with systemd # will add these when they become available with systemd +# "amazonlinux-2", +# "amazonlinux-2023", +# "debian-11", # "debian-13", # "fedora-40", # "photon-4", # "ubuntu-2004", # "ubuntu-2404", ONEDIR_DISTROS = [ - "amazonlinux-2", - "amazonlinux-2023", - "debian-11", "debian-12", "photon-5", "rockylinux-8", @@ -133,6 +130,9 @@ BLACKLIST_GIT_MASTER = [ "amazonlinux-2", "amazonlinux-2023", + "debian-11", + "debian-13", + "fedora-40", "photon-4", "photon-5", ] diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 103ea07ef..52a93ff9d 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -26,7 +26,7 @@ #====================================================================================================================== set -o nounset # Treat unset variables as an error -__ScriptVersion="2024.11.21" +__ScriptVersion="2024.11.25" __ScriptName="bootstrap-salt.sh" __ScriptFullName="$0" @@ -640,7 +640,11 @@ elif [ "$ITYPE" = "git" ]; then if [ "$#" -eq 0 ];then GIT_REV="master" else - GIT_REV="$1" + if [ "$(echo "$1" | grep -E '^(3006|3007)$')" != "" ]; then + GIT_REV="$1.x" # branches are 3006.x or 3007.x + else + GIT_REV="$1" + fi shift fi @@ -3085,7 +3089,7 @@ install_ubuntu_deps() { # Additionally install procps and pciutils which allows for Docker bootstraps. See 366#issuecomment-39666813 __PACKAGES="${__PACKAGES} procps pciutils" - # ensure sudo installed + # ensure sudo, ps installed __PACKAGES="${__PACKAGES} sudo" ## include hwclock if not part of base OS (23.10 and up) @@ -3171,6 +3175,9 @@ install_ubuntu_git_deps() { __PACKAGES="${__PACKAGES} util-linux-extra" fi + # Additionally install procps and pciutils which allows for Docker bootstraps. See 366#issuecomment-39666813 + __PACKAGES="${__PACKAGES} procps pciutils" + # ensure sudo installed __PACKAGES="${__PACKAGES} sudo" @@ -3618,6 +3625,9 @@ install_debian_git_deps() { __PACKAGES="python${PY_PKG_VER}-dev python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools gcc" echodebug "install_debian_git_deps() Installing ${__PACKAGES}" + # Additionally install procps and pciutils which allows for Docker bootstraps. See 366#issuecomment-39666813 + __PACKAGES="${__PACKAGES} procps pciutils" + # ensure sudo installed __PACKAGES="${__PACKAGES} sudo" From 8fa55b714e5380b395c6f7aa266e911e39075526 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 25 Nov 2024 12:32:09 -0700 Subject: [PATCH 121/173] Adjust allowing for git version input 3006 or 3007 --- bootstrap-salt.sh | 57 +++++++++++++++++--------- tests/integration/test_installation.py | 13 ------ 2 files changed, 37 insertions(+), 33 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 52a93ff9d..8f3403798 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -640,11 +640,7 @@ elif [ "$ITYPE" = "git" ]; then if [ "$#" -eq 0 ];then GIT_REV="master" else - if [ "$(echo "$1" | grep -E '^(3006|3007)$')" != "" ]; then - GIT_REV="$1.x" # branches are 3006.x or 3007.x - else - GIT_REV="$1" - fi + GIT_REV="$1" shift fi @@ -2097,6 +2093,7 @@ __tdnf_install_noinput() { # DESCRIPTION: (DRY) Helper function to clone and checkout salt to a # specific revision. #---------------------------------------------------------------------------------------------------------------------- +# shellcheck disable=SC2120 __git_clone_and_checkout() { echodebug "Installed git version: $(git --version | awk '{ print $3 }')" @@ -2105,6 +2102,12 @@ __git_clone_and_checkout() { export GIT_SSL_NO_VERIFY=1 fi + if [ "$(echo "$GIT_REV" | grep -E '^(3006|3007)$')" != "" ]; then + GIT_REV_ADJ="$1.x" # branches are 3006.x or 3007.x + else + GIT_REV_ADJ="$GIT_REV" + fi + __SALT_GIT_CHECKOUT_PARENT_DIR=$(dirname "${_SALT_GIT_CHECKOUT_DIR}" 2>/dev/null) __SALT_GIT_CHECKOUT_PARENT_DIR="${__SALT_GIT_CHECKOUT_PARENT_DIR:-/tmp/git}" __SALT_CHECKOUT_REPONAME="$(basename "${_SALT_GIT_CHECKOUT_DIR}" 2>/dev/null)" @@ -2133,15 +2136,15 @@ __git_clone_and_checkout() { git fetch --tags upstream fi - echodebug "Hard reseting the cloned repository to ${GIT_REV}" - git reset --hard "$GIT_REV" || return 1 + echodebug "Hard reseting the cloned repository to ${GIT_REV_ADJ}" + git reset --hard "$GIT_REV_ADJ" || return 1 - # Just calling `git reset --hard $GIT_REV` on a branch name that has + # Just calling `git reset --hard $GIT_REV_ADJ` on a branch name that has # already been checked out will not update that branch to the upstream # HEAD; instead it will simply reset to itself. Check the ref to see # if it is a branch name, check out the branch, and pull in the # changes. - if git branch -a | grep -q "${GIT_REV}"; then + if git branch -a | grep -q "${GIT_REV_ADJ}"; then echodebug "Rebasing the cloned repository branch" git pull --rebase || return 1 fi @@ -2163,16 +2166,16 @@ __git_clone_and_checkout() { # cloning we need actually works if [ "$(git clone 2>&1 | grep 'single-branch')" != "" ]; then # The "--single-branch" option is supported, attempt shallow cloning - echoinfo "Attempting to shallow clone $GIT_REV from Salt's repository ${_SALT_REPO_URL}" + echoinfo "Attempting to shallow clone $GIT_REV_ADJ from Salt's repository ${_SALT_REPO_URL}" ## Shallow cloning is resulting in the wrong version of Salt, even with a depth of 5 ## getting 3007.0+0na.246d066 when it should be 3007.1+410.g246d066457, disabling for now - ## if git clone --depth 1 --branch "$GIT_REV" "$_SALT_REPO_URL" "$__SALT_CHECKOUT_REPONAME"; then - echodebug "git command, git clone --branch $GIT_REV $_SALT_REPO_URL $__SALT_CHECKOUT_REPONAME" - if git clone --branch "$GIT_REV" "$_SALT_REPO_URL" "$__SALT_CHECKOUT_REPONAME"; then + ## if git clone --depth 1 --branch "$GIT_REV_ADJ" "$_SALT_REPO_URL" "$__SALT_CHECKOUT_REPONAME"; then + echodebug "git command, git clone --branch $GIT_REV_ADJ $_SALT_REPO_URL $__SALT_CHECKOUT_REPONAME" + if git clone --branch "$GIT_REV_ADJ" "$_SALT_REPO_URL" "$__SALT_CHECKOUT_REPONAME"; then # shellcheck disable=SC2164 cd "${_SALT_GIT_CHECKOUT_DIR}" __SHALLOW_CLONE=$BS_TRUE - echoinfo "shallow path (disabled shallow) git cloned $GIT_REV, version $(python3 salt/version.py)" + echoinfo "shallow path (disabled shallow) git cloned $GIT_REV_ADJ, version $(python3 salt/version.py)" else # Shallow clone above failed(missing upstream tags???), let's resume the old behaviour. echowarn "Failed to shallow clone." @@ -2191,7 +2194,7 @@ __git_clone_and_checkout() { # shellcheck disable=SC2164 cd "${_SALT_GIT_CHECKOUT_DIR}" - echoinfo "git cloned $GIT_REV, version $(python3 salt/version.py)" + echoinfo "git cloned $GIT_REV_ADJ, version $(python3 salt/version.py)" if ! echo "$_SALT_REPO_URL" | grep -q -F -w "${_SALTSTACK_REPO_URL#*://}"; then # We need to add the saltstack repository as a remote and fetch tags for proper versioning @@ -2201,14 +2204,14 @@ __git_clone_and_checkout() { echodebug "Fetching upstream (SaltStack's Salt repository) git tags" git fetch --tags upstream || return 1 - # Check if GIT_REV is a remote branch or just a commit hash - if git branch -r | grep -q -F -w "origin/$GIT_REV"; then - GIT_REV="origin/$GIT_REV" + # Check if GIT_REV_ADJ is a remote branch or just a commit hash + if git branch -r | grep -q -F -w "origin/$GIT_REV_ADJ"; then + GIT_REV_ADJ="origin/$GIT_REV_ADJ" fi fi - echodebug "Checking out $GIT_REV" - git checkout "$GIT_REV" || return 1 + echodebug "Checking out $GIT_REV_ADJ" + git checkout "$GIT_REV_ADJ" || return 1 fi fi @@ -3158,6 +3161,7 @@ install_ubuntu_git_deps() { __apt_get_install_noinput ca-certificates fi + # shellcheck disable=SC2119 __git_clone_and_checkout || return 1 if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -ne 3 ]; then @@ -3615,6 +3619,7 @@ install_debian_git_deps() { __apt_get_install_noinput ca-certificates fi + # shellcheck disable=SC2119 __git_clone_and_checkout || return 1 if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -ne 3 ]; then @@ -3963,6 +3968,7 @@ install_fedora_git_deps() { __PACKAGES="" fi + # shellcheck disable=SC2119 __git_clone_and_checkout || return 1 __PACKAGES="python${PY_PKG_VER}-devel python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools gcc gcc-c++" @@ -4373,6 +4379,7 @@ install_centos_git_deps() { __yum_install_noinput git || return 1 fi + # shellcheck disable=SC2119 __git_clone_and_checkout || return 1 __PACKAGES="" @@ -5418,6 +5425,7 @@ install_alpine_linux_git_deps() { apk -U add git || return 1 fi + # shellcheck disable=SC2119 __git_clone_and_checkout || return 1 apk -U add python3 python3-dev py3-pip py3-setuptools g++ linux-headers zeromq-dev openrc || return 1 @@ -5584,6 +5592,7 @@ install_amazon_linux_ami_2_git_deps() { __yum_install_noinput git || return 1 fi + # shellcheck disable=SC2119 __git_clone_and_checkout || return 1 __PACKAGES="python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools python${PY_PKG_VER}-devel gcc" @@ -5803,6 +5812,7 @@ install_amazon_linux_ami_2023_git_deps() { __yum_install_noinput git || return 1 fi + # shellcheck disable=SC2119 __git_clone_and_checkout || return 1 __PACKAGES="python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools python${PY_PKG_VER}-devel gcc" @@ -5990,6 +6000,7 @@ install_arch_linux_git_deps() { pacman -Sy --noconfirm --needed git || return 1 fi + # shellcheck disable=SC2119 __git_clone_and_checkout || return 1 if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -eq 2 ]; then @@ -6373,6 +6384,7 @@ install_photon_git_deps() { __PACKAGES="" fi + # shellcheck disable=SC2119 __git_clone_and_checkout || return 1 __PACKAGES="python${PY_PKG_VER}-devel python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools gcc glibc-devel linux-devel.x86_64 cython${PY_PKG_VER}" @@ -6732,6 +6744,7 @@ install_opensuse_git_deps() { __zypper_install git || return 1 fi + # shellcheck disable=SC2119 __git_clone_and_checkout || return 1 # Check for Tumbleweed @@ -6966,6 +6979,7 @@ install_opensuse_15_git_deps() { __zypper_install git || return 1 fi + # shellcheck disable=SC2119 __git_clone_and_checkout || return 1 if [ -n "$_PY_EXE" ] && [ "$_PY_MAJOR_VERSION" -ne 3 ]; then @@ -7221,6 +7235,7 @@ install_gentoo_git_deps() { echoinfo "Running emerge -v1 setuptools" __emerge -v1 setuptools || return 1 + # shellcheck disable=SC2119 __git_clone_and_checkout || return 1 __gentoo_post_dep || return 1 } @@ -7589,6 +7604,7 @@ install_macosx_git_deps() { # Install PIP $_PYEXE /tmp/get-pip.py || return 1 + # shellcheck disable=SC2119 __git_clone_and_checkout || return 1 return 0 @@ -8125,6 +8141,7 @@ fi if [ "${ITYPE}" = "git" ] && [ ${_NO_DEPS} -eq ${BS_TRUE} ]; then + # shellcheck disable=SC2119 if ! __git_clone_and_checkout; then echo "Failed to clone and checkout git repository." exit 1 diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index 714b8e166..5e498afaf 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -29,10 +29,6 @@ def run_salt_call(cmd): if platform.system() == "Windows": cmd.append("--out=json") result = subprocess.run(cmd, capture_output=True, text=True) - print( - f"DGM run_salt_call, cmd '{cmd}', result '{result}', stdout '{result.stdout}'", - flush=True, - ) if 0 == result.returncode: json_data = json.loads(result.stdout) else: @@ -43,10 +39,6 @@ def run_salt_call(cmd): cmdl.extend(cmd) cmdl.append("--out=json") result = subprocess.run(cmdl, capture_output=True, text=True) - print( - f"DGM run_salt_call, cmdl '{cmdl}', result '{result}', stdout '{result.stdout}'", - flush=True, - ) if 0 == result.returncode: json_data = json.loads(result.stdout) else: @@ -74,10 +66,5 @@ def test_target_salt_version(path, target_salt_version): pytest.skip(f"No target version specified") cmd = ["salt-call", "--local", "grains.item", "saltversion", "--timeout=120"] result = run_salt_call(cmd) - dgm_saltversion = result["saltversion"] - print( - f"DGM test_target_salt_version, target_salt_version '{target_salt_version}', result saltversion '{dgm_saltversion }', result '{result}'", - flush=True, - ) # Returns: {'saltversion': '3006.9+217.g53cfa53040'} assert result["saltversion"] == target_salt_version From 543d8313129a012794aa646fe5db18aafd28b726 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 25 Nov 2024 12:58:58 -0700 Subject: [PATCH 122/173] Fixed typo --- bootstrap-salt.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 8f3403798..d59288f7e 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -2103,7 +2103,7 @@ __git_clone_and_checkout() { fi if [ "$(echo "$GIT_REV" | grep -E '^(3006|3007)$')" != "" ]; then - GIT_REV_ADJ="$1.x" # branches are 3006.x or 3007.x + GIT_REV_ADJ="$GIT_REV.x" # branches are 3006.x or 3007.x else GIT_REV_ADJ="$GIT_REV" fi From 6360fd19f21f439b7ce433195eed9d1ad9f9c3f9 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 25 Nov 2024 13:07:35 -0700 Subject: [PATCH 123/173] Adjust version test to allow for git versions of salt --- tests/integration/test_installation.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index 5e498afaf..38e6e814a 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -67,4 +67,5 @@ def test_target_salt_version(path, target_salt_version): cmd = ["salt-call", "--local", "grains.item", "saltversion", "--timeout=120"] result = run_salt_call(cmd) # Returns: {'saltversion': '3006.9+217.g53cfa53040'} - assert result["saltversion"] == target_salt_version + adj_saltversion = result["saltversion"].split("+")[0] + assert adj_saltversion == target_salt_version From f8da32f34b7a8bb6aa0a3770c03a15d044caf157 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 25 Nov 2024 13:29:21 -0700 Subject: [PATCH 124/173] Upgrade pip using python3 -m pip install --- bootstrap-salt.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index d59288f7e..6b3b008ec 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -2742,7 +2742,8 @@ __install_salt_from_repo() { echodebug "Installed pip version: $(${_pip_cmd} --version)" echodebug "Upgrading pip to latest, running '${_pip_cmd} install ${_PIP_INSTALL_ARGS} -U pip>=${_MINIMUM_PIP_VERSION}'" - ${_pip_cmd} install ${_PIP_INSTALL_ARGS} -v -U "pip>=${_MINIMUM_PIP_VERSION}" + ## DGM ${_pip_cmd} install ${_PIP_INSTALL_ARGS} -v -U "pip>=${_MINIMUM_PIP_VERSION}" + ${_py_exe} -m pip install ${_PIP_INSTALL_ARGS} -v -U "pip>=${_MINIMUM_PIP_VERSION}" echodebug "Upgraded pip version: $(${_pip_cmd} --version)" From d7d03738da209a3eaa457940501c1196e8377946 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 25 Nov 2024 16:35:33 -0700 Subject: [PATCH 125/173] Adjusting install from git --- bootstrap-salt.sh | 5 ----- 1 file changed, 5 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 6b3b008ec..d04a1f873 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -2741,11 +2741,6 @@ __install_salt_from_repo() { __check_pip_allowed echodebug "Installed pip version: $(${_pip_cmd} --version)" - echodebug "Upgrading pip to latest, running '${_pip_cmd} install ${_PIP_INSTALL_ARGS} -U pip>=${_MINIMUM_PIP_VERSION}'" - ## DGM ${_pip_cmd} install ${_PIP_INSTALL_ARGS} -v -U "pip>=${_MINIMUM_PIP_VERSION}" - ${_py_exe} -m pip install ${_PIP_INSTALL_ARGS} -v -U "pip>=${_MINIMUM_PIP_VERSION}" - - echodebug "Upgraded pip version: $(${_pip_cmd} --version)" _setuptools_dep="setuptools>=${_MINIMUM_SETUPTOOLS_VERSION},<${_MAXIMUM_SETUPTOOLS_VERSION}" if [ "$_PY_MAJOR_VERSION" -ne 3 ]; then From 6b27fc3e979cefd824e7854db569487b0a63fd7e Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 25 Nov 2024 16:52:36 -0700 Subject: [PATCH 126/173] Restored pip check, updated README and Windows ps1 for packages.broadcom.com --- bootstrap-salt.ps1 | 3 ++- bootstrap-salt.sh | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/bootstrap-salt.ps1 b/bootstrap-salt.ps1 index 815a0eb74..fd665c902 100644 --- a/bootstrap-salt.ps1 +++ b/bootstrap-salt.ps1 @@ -39,7 +39,8 @@ Salt Bootstrap GitHub Project (script home) - https://github.com/saltstack/salt-bootstrap Original Vagrant Provisioner Project - https://github.com/saltstack/salty-vagrant Vagrant Project (utilizes this script) - https://github.com/mitchellh/vagrant - Salt Download Location - https://repo.saltproject.io/salt/py3/windows + Salt Download Location - https://packages.broadcom.com/artifactory/saltproject-generic/windows/ + Salt Manual Install Directions (Windows) - https://docs.saltproject.io/salt/install-guide/en/latest/topics/install-by-operating-system/windows.html #> #=============================================================================== diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index d04a1f873..25cad457d 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -2742,6 +2742,44 @@ __install_salt_from_repo() { echodebug "Installed pip version: $(${_pip_cmd} --version)" + CHECK_PIP_VERSION_SCRIPT=$(cat << EOM +import sys +try: + import pip + installed_pip_version=tuple([int(part.strip()) for part in pip.__version__.split('.') if part.isdigit()]) + desired_pip_version=($(echo ${_MINIMUM_PIP_VERSION} | sed 's/\./, /g' )) + if installed_pip_version < desired_pip_version: + print('Desired pip version {!r} > Installed pip version {!r}'.format('.'.join(map(str, desired_pip_version)), '.'.join(map(str, installed_pip_version)))) + sys.exit(1) + print('Desired pip version {!r} < Installed pip version {!r}'.format('.'.join(map(str, desired_pip_version)), '.'.join(map(str, installed_pip_version)))) + sys.exit(0) +except ImportError: + print('Failed to import pip') + sys.exit(1) +EOM +) + if ! ${_py_exe} -c "$CHECK_PIP_VERSION_SCRIPT"; then + # Upgrade pip to at least 1.2 which is when we can start using "python3 -m pip" + echodebug "Running '${_pip_cmd} install ${_PIP_INSTALL_ARGS} pip>=${_MINIMUM_PIP_VERSION}'" + ${_pip_cmd} install ${_PIP_INSTALL_ARGS} -v "pip>=${_MINIMUM_PIP_VERSION}" + sleep 1 + echodebug "PATH: ${PATH}" + _pip_cmd="pip${_py_version}" + if ! __check_command_exists "${_pip_cmd}"; then + echodebug "The pip binary '${_pip_cmd}' was not found in PATH" + _pip_cmd="pip$(echo "${_py_version}" | cut -c -1)" + if ! __check_command_exists "${_pip_cmd}"; then + echodebug "The pip binary '${_pip_cmd}' was not found in PATH" + _pip_cmd="pip" + if ! __check_command_exists "${_pip_cmd}"; then + echoerror "Unable to find a pip binary" + return 1 + fi + fi + fi + echodebug "Installed pip version: $(${_pip_cmd} --version)" + fi + _setuptools_dep="setuptools>=${_MINIMUM_SETUPTOOLS_VERSION},<${_MAXIMUM_SETUPTOOLS_VERSION}" if [ "$_PY_MAJOR_VERSION" -ne 3 ]; then echoerror "Python version is no longer supported, only Python 3" From 6629b40bfcd2dafe814a9372ba80a2c03e9618be Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 25 Nov 2024 17:11:35 -0700 Subject: [PATCH 127/173] Trying using requirements pkg file --- bootstrap-salt.sh | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 25cad457d..956c03923 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -2807,12 +2807,27 @@ EOM if [ ${DISTRO_NAME_L} = "ubuntu" ] && [ "$DISTRO_MAJOR_VERSION" -eq 22 ]; then echodebug "Ubuntu 22.04 has problem with base.txt requirements file, not parsing sys_platform == 'win32', upgrading from default pip works" echodebug "${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --upgrade pip" - ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --upgrade pip || (echo "Failed to upgrade pip" && return 1) + ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --upgrade pip + # shellcheck disable=SC2181 + if [ $? -ne 0 ]; then + echo "Failed to upgrade pip" + return 1 + fi fi echoinfo "Downloading Salt Dependencies from PyPi" echodebug "Running '${_pip_cmd} download -d /tmp/git/deps ${_PIP_DOWNLOAD_ARGS} .'" - ${_pip_cmd} download -d /tmp/git/deps ${_PIP_DOWNLOAD_ARGS} . || (echo "Failed to download salt dependencies" && return 1) + if [ "${OS_NAME}" = "Linux" ]; then + ${_pip_cmd} download -d /tmp/git/deps ${_PIP_DOWNLOAD_ARGS} -r "requirements/static/pkg/py${_py_version}/linux.txt" + else + ${_pip_cmd} download -d /tmp/git/deps ${_PIP_DOWNLOAD_ARGS} . + fi + # shellcheck disable=SC2181 + if [ $? -ne 0 ]; then + echo "Failed to download salt dependencies" + return 1 + fi + echoinfo "Installing Downloaded Salt Dependencies" echodebug "Running '${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --ignore-installed ${_PIP_INSTALL_ARGS} /tmp/git/deps/*'" From ba0a5f275906df80a5094a8535f966819262a7f9 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 25 Nov 2024 17:28:36 -0700 Subject: [PATCH 128/173] Added back more containers to test on, revised git usage due to Salt git errors, see https://github.com/saltstack/salt/issues/67070 --- .github/workflows/ci.yml | 52 ++++++++++++++++++++++++- .github/workflows/templates/generate.py | 26 ++++++------- bootstrap-salt.sh | 6 +-- 3 files changed, 64 insertions(+), 20 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a94508b54..7df5157f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -180,6 +180,51 @@ jobs: + amazonlinux-2: + name: Amazon 2 + if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' + uses: ./.github/workflows/test-linux.yml + needs: + - lint + - generate-actions-workflow + with: + distro-slug: amazonlinux-2 + display-name: Amazon 2 + container-slug: systemd-amazonlinux-2 + timeout: 20 + instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' + + + amazonlinux-2023: + name: Amazon 2023 + if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' + uses: ./.github/workflows/test-linux.yml + needs: + - lint + - generate-actions-workflow + with: + distro-slug: amazonlinux-2023 + display-name: Amazon 2023 + container-slug: systemd-amazonlinux-2023 + timeout: 20 + instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' + + + debian-11: + name: Debian 11 + if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' + uses: ./.github/workflows/test-linux.yml + needs: + - lint + - generate-actions-workflow + with: + distro-slug: debian-11 + display-name: Debian 11 + container-slug: systemd-debian-11 + timeout: 20 + instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' + + debian-12: name: Debian 12 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' @@ -192,7 +237,7 @@ jobs: display-name: Debian 12 container-slug: systemd-debian-12 timeout: 20 - instances: '["git-3006", "stable-3007", "git-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' + instances: '["stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' photon-5: @@ -237,7 +282,7 @@ jobs: display-name: Rocky Linux 9 container-slug: systemd-rockylinux-9 timeout: 20 - instances: '["stable-3006", "git-3006", "onedir-3006", "stable-3006-8", "stable-3007", "git-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' + instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' ubuntu-2204: @@ -267,6 +312,9 @@ jobs: - macos-13 - macos-14 - windows-2022 + - amazonlinux-2 + - amazonlinux-2023 + - debian-11 - debian-12 - photon-5 - rockylinux-8 diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index 097bb9464..e873b61a8 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -8,9 +8,6 @@ # only test against current containers with systemd # will add these when they become available with systemd -# "amazonlinux-2", -# "amazonlinux-2023", -# "debian-11", # "debian-12", # "debian-13", # "fedora-40", @@ -18,6 +15,9 @@ # "ubuntu-2004", # "ubuntu-2404", LINUX_DISTROS = [ + "amazonlinux-2", + "amazonlinux-2023", + "debian-11", "debian-12", "photon-5", "rockylinux-8", @@ -37,15 +37,15 @@ # only test against current containers with systemd # will add these when they become available with systemd -# "amazonlinux-2", -# "amazonlinux-2023", -# "debian-11", # "debian-13", # "fedora-40", # "photon-4", # "ubuntu-2004", # "ubuntu-2404", STABLE_DISTROS = [ + "amazonlinux-2", + "amazonlinux-2023", + "debian-11", "debian-12", "photon-5", "rockylinux-8", @@ -55,15 +55,15 @@ # only test against current containers with systemd # will add these when they become available with systemd -# "amazonlinux-2", -# "amazonlinux-2023", -# "debian-11", # "debian-13", # "fedora-40", # "photon-4", # "ubuntu-2004", # "ubuntu-2404", ONEDIR_DISTROS = [ + "amazonlinux-2", + "amazonlinux-2023", + "debian-11", "debian-12", "photon-5", "rockylinux-8", @@ -97,31 +97,31 @@ "photon-5", ] -# "debian-12", -# "rockylinux-9", # "ubuntu-2204", BLACKLIST_GIT_3006 = [ "amazonlinux-2", "amazonlinux-2023", "debian-11", + "debian-12", "fedora-40", "photon-4", "photon-5", + "rockylinux-9", "ubuntu-2004", "ubuntu-2404", ] -# "debian-12", -# "rockylinux-9", # "ubuntu-2204", BLACKLIST_GIT_3007 = [ "amazonlinux-2", "amazonlinux-2023", "debian-11", + "debian-12", "debian-13", "fedora-40", "photon-4", "photon-5", + "rockylinux-9", "ubuntu-2004", "ubuntu-2404", ] diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 956c03923..314d2de5e 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -2817,11 +2817,7 @@ EOM echoinfo "Downloading Salt Dependencies from PyPi" echodebug "Running '${_pip_cmd} download -d /tmp/git/deps ${_PIP_DOWNLOAD_ARGS} .'" - if [ "${OS_NAME}" = "Linux" ]; then - ${_pip_cmd} download -d /tmp/git/deps ${_PIP_DOWNLOAD_ARGS} -r "requirements/static/pkg/py${_py_version}/linux.txt" - else - ${_pip_cmd} download -d /tmp/git/deps ${_PIP_DOWNLOAD_ARGS} . - fi + ${_pip_cmd} download -d /tmp/git/deps ${_PIP_DOWNLOAD_ARGS} . # shellcheck disable=SC2181 if [ $? -ne 0 ]; then echo "Failed to download salt dependencies" From 13e00e0bbc2f4166de6a2209368291bd537f69a2 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 25 Nov 2024 19:01:19 -0700 Subject: [PATCH 129/173] Limited the containers tested on, and skip git testing till Salt requirements fixed --- .github/workflows/ci.yml | 52 +------------------------ .github/workflows/templates/generate.py | 21 +++++----- 2 files changed, 13 insertions(+), 60 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7df5157f2..980dc7e15 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -180,51 +180,6 @@ jobs: - amazonlinux-2: - name: Amazon 2 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: amazonlinux-2 - display-name: Amazon 2 - container-slug: systemd-amazonlinux-2 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' - - - amazonlinux-2023: - name: Amazon 2023 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: amazonlinux-2023 - display-name: Amazon 2023 - container-slug: systemd-amazonlinux-2023 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' - - - debian-11: - name: Debian 11 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: debian-11 - display-name: Debian 11 - container-slug: systemd-debian-11 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' - - debian-12: name: Debian 12 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' @@ -237,7 +192,7 @@ jobs: display-name: Debian 12 container-slug: systemd-debian-12 timeout: 20 - instances: '["stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' + instances: '["stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' photon-5: @@ -282,7 +237,7 @@ jobs: display-name: Rocky Linux 9 container-slug: systemd-rockylinux-9 timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' + instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' ubuntu-2204: @@ -312,9 +267,6 @@ jobs: - macos-13 - macos-14 - windows-2022 - - amazonlinux-2 - - amazonlinux-2023 - - debian-11 - debian-12 - photon-5 - rockylinux-8 diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index e873b61a8..0207d9888 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -8,6 +8,9 @@ # only test against current containers with systemd # will add these when they become available with systemd +# "amazonlinux-2", +# "amazonlinux-2023", +# "debian-11", # "debian-12", # "debian-13", # "fedora-40", @@ -15,9 +18,6 @@ # "ubuntu-2004", # "ubuntu-2404", LINUX_DISTROS = [ - "amazonlinux-2", - "amazonlinux-2023", - "debian-11", "debian-12", "photon-5", "rockylinux-8", @@ -37,15 +37,15 @@ # only test against current containers with systemd # will add these when they become available with systemd +# "amazonlinux-2", +# "amazonlinux-2023", +# "debian-11", # "debian-13", # "fedora-40", # "photon-4", # "ubuntu-2004", # "ubuntu-2404", STABLE_DISTROS = [ - "amazonlinux-2", - "amazonlinux-2023", - "debian-11", "debian-12", "photon-5", "rockylinux-8", @@ -55,15 +55,15 @@ # only test against current containers with systemd # will add these when they become available with systemd +# "amazonlinux-2", +# "amazonlinux-2023", +# "debian-11", # "debian-13", # "fedora-40", # "photon-4", # "ubuntu-2004", # "ubuntu-2404", ONEDIR_DISTROS = [ - "amazonlinux-2", - "amazonlinux-2023", - "debian-11", "debian-12", "photon-5", "rockylinux-8", @@ -126,15 +126,16 @@ "ubuntu-2404", ] -## "rockylinux-9", BLACKLIST_GIT_MASTER = [ "amazonlinux-2", "amazonlinux-2023", "debian-11", + "debian-12", "debian-13", "fedora-40", "photon-4", "photon-5", + "rockylinux-9", ] SALT_VERSIONS = [ From a336478623befc8b22499ab00c724358723be9e6 Mon Sep 17 00:00:00 2001 From: ScriptAutomate Date: Tue, 26 Nov 2024 10:16:06 -0600 Subject: [PATCH 130/173] Update README script help output and scriptversions --- README.rst | 169 ++++++++++++++++++++++++--------------------- bootstrap-salt.ps1 | 2 +- bootstrap-salt.sh | 2 +- 3 files changed, 92 insertions(+), 81 deletions(-) diff --git a/README.rst b/README.rst index 929d93315..db15cc47a 100644 --- a/README.rst +++ b/README.rst @@ -84,112 +84,123 @@ To view the latest options and descriptions for ``salt-bootstrap``, use ``-h`` a Usage : bootstrap-salt.sh [options] [install-type-args] Installation types: - - stable Install latest stable release. This is the default - install type - - stable [branch] Install latest version on a branch. Only supported - for packages available at packages.broadcom.com - - stable [version] Install a specific version. Only supported for - packages available at packages.broadcom.com - To pin a 3xxx minor version, specify it as 3xxx.0 - - testing RHEL-family specific: configure EPEL testing repo - - git Install from the head of the master branch - - git [ref] Install from any git ref (such as a branch, tag, or - commit) + - stable Install latest stable release. This is the default + install type + - stable [branch] Install latest version on a branch. Only supported + for packages available at packages.broadcom.com + - stable [version] Install a specific version. Only supported for + packages available at packages.broadcom.com + To pin a 3xxx minor version, specify it as 3xxx.0 + - testing RHEL-family specific: configure EPEL testing repo + - git Install from the head of the master branch + - git [ref] Install from any git ref (such as a branch, tag, or + commit) + - onedir Install latest onedir release. + - onedir [version] Install a specific version. Only supported for + onedir packages available at packages.broadcom.com + + - onedir_rc Install latest onedir RC release. + - onedir_rc [version] Install a specific version. Only supported for + onedir RC packages available at packages.broadcom.com Examples: - bootstrap-salt.sh - bootstrap-salt.sh stable - - bootstrap-salt.sh stable 3006.7 - - bootstrap-salt.sh stable v3006.8 + - bootstrap-salt.sh stable 3006 + - bootstrap-salt.sh stable 3006.1 - bootstrap-salt.sh testing - bootstrap-salt.sh git + - bootstrap-salt.sh git 3006.7 + - bootstrap-salt.sh git v3006.8 - bootstrap-salt.sh git 3007.1 - bootstrap-salt.sh git v3007.1 - bootstrap-salt.sh git 06f249901a2e2f1ed310d58ea3921a129f214358 + - bootstrap-salt.sh onedir + - bootstrap-salt.sh onedir 3006 + - bootstrap-salt.sh onedir_rc + - bootstrap-salt.sh onedir_rc 3008 + Options: - -h Display this message - -v Display script version - -n No colours - -D Show debug output + -a Pip install all Python pkg dependencies for Salt. Requires -V to install + all pip pkgs into the virtualenv. + (Only available for Ubuntu based distributions) + -A Pass the salt-master DNS name or IP. This will be stored under + \${BS_SALT_ETC_DIR}/minion.d/99-master-address.conf + -b Assume that dependencies are already installed and software sources are + set up. If git is selected, git tree is still checked out as dependency + step. -c Temporary configuration directory - -g Salt Git repository URL. Default: https://github.com/saltstack/salt.git - -w Install packages from downstream package repository rather than - upstream, saltstack package repository. This is currently only - implemented for SUSE. - -k Temporary directory holding the minion keys which will pre-seed - the master. - -s Sleep time used when waiting for daemons to start, restart and when - checking for the services running. Default: 3 - -L Also install salt-cloud and required python-libcloud package - -M Also install salt-master - -S Also install salt-syndic - -W Also install salt-api - -N Do not install salt-minion - -X Do not start daemons after installation + -C Only run the configuration function. Implies -F (forced overwrite). + To overwrite Master, Syndic or Api configs, -M,-S or -W, respectively, must + also be specified. Salt installation will be ommitted, but some of the + dependencies could be installed to write configuration with -j or -J. -d Disables checking if Salt services are enabled to start on system boot. You can also do this by touching /tmp/disable_salt_checks on the target - host. Default: ${BS_FALSE} - -P Allow pip based installations. On some distributions the required salt - packages or its dependencies are not available as a package for that - distribution. Using this flag allows the script to use pip as a last - resort method. NOTE: This only works for functions which actually - implement pip based installations. - -U If set, fully upgrade the system prior to bootstrapping Salt + host. Default: \${BS_FALSE} + -D Show debug output + -f Force shallow cloning for git installations. + This may result in an "n/a" in the version number. + -F Allow copied files to overwrite existing (config, init.d, etc) + -g Salt Git repository URL. Default: https://github.com/saltstack/salt.git + -h Display this message + -H Use the specified HTTP proxy for all download URLs (including https://). + For example: http://myproxy.example.com:3128 + -i Pass the salt-minion id. This will be stored under + \${BS_SALT_ETC_DIR}/minion_id -I If set, allow insecure connections while downloading any files. For example, pass '--no-check-certificate' to 'wget' or '--insecure' to 'curl'. On Debian and Ubuntu, using this option with -U allows obtaining GnuPG archive keys insecurely if distro has changed release signatures. - -F Allow copied files to overwrite existing (config, init.d, etc) - -K If set, keep the temporary files in the temporary directories specified - with -c and -k - -C Only run the configuration function. Implies -F (forced overwrite). - To overwrite Master or Syndic configs, -M or -S, respectively, must - also be specified. Salt installation will be ommitted, but some of the - dependencies could be installed to write configuration with -j or -J. - -A Pass the salt-master DNS name or IP. This will be stored under - ${BS_SALT_ETC_DIR}/minion.d/99-master-address.conf - -i Pass the salt-minion id. This will be stored under - ${BS_SALT_ETC_DIR}/minion_id - -p Extra-package to install while installing Salt dependencies. One package - per -p flag. You are responsible for providing the proper package name. - -H Use the specified HTTP proxy for all download URLs (including https://). - For example: http://myproxy.example.com:3128 - -b Assume that dependencies are already installed and software sources are - set up. If git is selected, git tree is still checked out as dependency - step. - -f Force shallow cloning for git installations. - This may result in an "n/a" in the version number. - -l Disable ssl checks. When passed, switches "https" calls to "http" where - possible. - -V Install Salt into virtualenv - (only available for Ubuntu based distributions) - -a Pip install all Python pkg dependencies for Salt. Requires -V to install - all pip pkgs into the virtualenv. - (Only available for Ubuntu based distributions) - -r Disable all repository configuration performed by this script. This - option assumes all necessary repository configuration is already present - on the system. - -R Specify a custom repository URL. Assumes the custom repository URL - points to a repository that mirrors Salt packages located at - packages.broadcom.com. The option passed with -R replaces the - "packages.broadcom.com". If -R is passed, -r is also set. Currently only - works on CentOS/RHEL and Debian based distributions. - -J Replace the Master config file with data passed in as a JSON string. If - a Master config file is found, a reasonable effort will be made to save + -j Replace the Minion config file with data passed in as a JSON string. If + a Minion config file is found, a reasonable effort will be made to save the file with a ".bak" extension. If used in conjunction with -C or -F, no ".bak" file will be created as either of those options will force a complete overwrite of the file. - -j Replace the Minion config file with data passed in as a JSON string. If - a Minion config file is found, a reasonable effort will be made to save + -J Replace the Master config file with data passed in as a JSON string. If + a Master config file is found, a reasonable effort will be made to save the file with a ".bak" extension. If used in conjunction with -C or -F, no ".bak" file will be created as either of those options will force a complete overwrite of the file. + -k Temporary directory holding the minion keys which will pre-seed + the master. + -K If set, keep the temporary files in the temporary directories specified + with -c and -k + -l Disable ssl checks. When passed, switches "https" calls to "http" where + possible. + -L Also install salt-cloud and required python-libcloud package + -M Also install salt-master + -n No colours + -N Do not install salt-minion + -p Extra-package to install while installing Salt dependencies. One package + per -p flag. You are responsible for providing the proper package name. + -P Allow pip based installations. On some distributions the required salt + packages or its dependencies are not available as a package for that + distribution. Using this flag allows the script to use pip as a last + resort method. NOTE: This only works for functions which actually + implement pip based installations. -q Quiet salt installation from git (setup.py install -q) - -x Changes the Python version used to install Salt (default: Python 3) - Python 2.7 is not longer supported. -Q Quickstart, install the Salt master and the Salt minion. And automatically accept the minion key. + -R Specify a custom repository URL. Assumes the custom repository URL + points to a repository that mirrors Salt packages located at + packages.broadcom.com. The option passed with -R replaces the + "packages.broadcom.com". If -R is passed, -r is also set. Currently only + works on CentOS/RHEL and Debian based distributions and macOS. + -s Sleep time used when waiting for daemons to start, restart and when + checking for the services running. Default: 3 + -S Also install salt-syndic + -r Disable all repository configuration performed by this script. This + option assumes all necessary repository configuration is already present + on the system. + -U If set, fully upgrade the system prior to bootstrapping Salt + -v Display script version + -V Install Salt into virtualenv + (only available for Ubuntu based distributions) + -W Also install salt-api + -x Changes the Python version used to install Salt (default: Python 3). + Python 2.7 is no longer supported. + -X Do not start daemons after installation The Salt Bootstrap script has a wide variety of options that can be passed as well as several ways of obtaining the bootstrap script itself. Note that the use of ``sudo`` diff --git a/bootstrap-salt.ps1 b/bootstrap-salt.ps1 index fd665c902..cd70eebad 100644 --- a/bootstrap-salt.ps1 +++ b/bootstrap-salt.ps1 @@ -108,7 +108,7 @@ if ($help) { exit 0 } -$__ScriptVersion = "2024.11.21" +$__ScriptVersion = "2024.11.26" $ScriptName = $myInvocation.MyCommand.Name # We'll check for the Version next, because it also has no requirements diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 314d2de5e..ea0f9f391 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -26,7 +26,7 @@ #====================================================================================================================== set -o nounset # Treat unset variables as an error -__ScriptVersion="2024.11.25" +__ScriptVersion="2024.11.26" __ScriptName="bootstrap-salt.sh" __ScriptFullName="$0" From 7a445852b42d58d96d7f5c7750ed165843284eaa Mon Sep 17 00:00:00 2001 From: Salt Project Packaging Date: Tue, 26 Nov 2024 16:40:38 +0000 Subject: [PATCH 131/173] Update develop branch for the v2024.11.26 release --- CHANGELOG.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2a70b9fa1..a4d9ac944 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +# v2024.11.26 + +## What's Changed + +- Updated URLs in README, and fix deps download issue on RedHat family by @dmurphy18 in https://github.com/saltstack/salt-bootstrap/pull/2052 +- Update README script help output and scriptversions by @ScriptAutomate in https://github.com/saltstack/salt-bootstrap/pull/2056 + +**Full Changelog**: https://github.com/saltstack/salt-bootstrap/compare/v2024.11.21...v2024.11.26 + # v2024.11.21 ## What's Changed From 124e62e9fab3ba743989fdad9c019d31ebd9970e Mon Sep 17 00:00:00 2001 From: Salt Project Packaging Date: Tue, 26 Nov 2024 16:41:23 +0000 Subject: [PATCH 132/173] Update README.rst with 2024.11.26 release sha256sum --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index db15cc47a..8a2310760 100644 --- a/README.rst +++ b/README.rst @@ -37,6 +37,7 @@ sum** of the downloaded ``bootstrap-salt.sh`` file. The SHA256 sum of the ``bootstrap-salt.sh`` file, per release, is: +- 2024.11.26: ``832c7a20b96e1df171d715323df9afff8a11aef42d15598c007f240bc89d723c`` - 2024.11.21: ``ddf624c3a94d721da3f7629402a6c7ecc9dd96d13c1ead2a626314e97cee982a`` - 2024.11.07: ``70a9783649e129985563d1a86cf28b8984499643e62ae1dc47dc008bd204fcbb`` - 2024.09.24: ``88e4e4cad4b115a7b721dd9c21d5ee5df390b5b73b63de48f99399146f43f371`` From b96a669d39485d761c5552703f36e4f00d74f055 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 26 Nov 2024 09:57:47 -0700 Subject: [PATCH 133/173] Testin ggit install methods --- bootstrap-salt.sh | 83 +++++++++++++++++++++++++---------------------- 1 file changed, 44 insertions(+), 39 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index ea0f9f391..b41c25688 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -2742,43 +2742,43 @@ __install_salt_from_repo() { echodebug "Installed pip version: $(${_pip_cmd} --version)" - CHECK_PIP_VERSION_SCRIPT=$(cat << EOM -import sys -try: - import pip - installed_pip_version=tuple([int(part.strip()) for part in pip.__version__.split('.') if part.isdigit()]) - desired_pip_version=($(echo ${_MINIMUM_PIP_VERSION} | sed 's/\./, /g' )) - if installed_pip_version < desired_pip_version: - print('Desired pip version {!r} > Installed pip version {!r}'.format('.'.join(map(str, desired_pip_version)), '.'.join(map(str, installed_pip_version)))) - sys.exit(1) - print('Desired pip version {!r} < Installed pip version {!r}'.format('.'.join(map(str, desired_pip_version)), '.'.join(map(str, installed_pip_version)))) - sys.exit(0) -except ImportError: - print('Failed to import pip') - sys.exit(1) -EOM -) - if ! ${_py_exe} -c "$CHECK_PIP_VERSION_SCRIPT"; then - # Upgrade pip to at least 1.2 which is when we can start using "python3 -m pip" - echodebug "Running '${_pip_cmd} install ${_PIP_INSTALL_ARGS} pip>=${_MINIMUM_PIP_VERSION}'" - ${_pip_cmd} install ${_PIP_INSTALL_ARGS} -v "pip>=${_MINIMUM_PIP_VERSION}" - sleep 1 - echodebug "PATH: ${PATH}" - _pip_cmd="pip${_py_version}" - if ! __check_command_exists "${_pip_cmd}"; then - echodebug "The pip binary '${_pip_cmd}' was not found in PATH" - _pip_cmd="pip$(echo "${_py_version}" | cut -c -1)" - if ! __check_command_exists "${_pip_cmd}"; then - echodebug "The pip binary '${_pip_cmd}' was not found in PATH" - _pip_cmd="pip" - if ! __check_command_exists "${_pip_cmd}"; then - echoerror "Unable to find a pip binary" - return 1 - fi - fi - fi - echodebug "Installed pip version: $(${_pip_cmd} --version)" - fi +## DGM CHECK_PIP_VERSION_SCRIPT=$(cat << EOM +## DGM import sys +## DGM try: +## DGM import pip +## DGM installed_pip_version=tuple([int(part.strip()) for part in pip.__version__.split('.') if part.isdigit()]) +## DGM desired_pip_version=($(echo ${_MINIMUM_PIP_VERSION} | sed 's/\./, /g' )) +## DGM if installed_pip_version < desired_pip_version: +## DGM print('Desired pip version {!r} > Installed pip version {!r}'.format('.'.join(map(str, desired_pip_version)), '.'.join(map(str, installed_pip_version)))) +## DGM sys.exit(1) +## DGM print('Desired pip version {!r} < Installed pip version {!r}'.format('.'.join(map(str, desired_pip_version)), '.'.join(map(str, installed_pip_version)))) +## DGM sys.exit(0) +## DGM except ImportError: +## DGM print('Failed to import pip') +## DGM sys.exit(1) +## DGM EOM +## DGM ) +## DGM if ! ${_py_exe} -c "$CHECK_PIP_VERSION_SCRIPT"; then +## DGM # Upgrade pip to at least 1.2 which is when we can start using "python3 -m pip" +## DGM echodebug "Running '${_pip_cmd} install ${_PIP_INSTALL_ARGS} pip>=${_MINIMUM_PIP_VERSION}'" +## DGM ${_pip_cmd} install ${_PIP_INSTALL_ARGS} -v "pip>=${_MINIMUM_PIP_VERSION}" +## DGM sleep 1 +## DGM echodebug "PATH: ${PATH}" +## DGM _pip_cmd="pip${_py_version}" +## DGM if ! __check_command_exists "${_pip_cmd}"; then +## DGM echodebug "The pip binary '${_pip_cmd}' was not found in PATH" +## DGM _pip_cmd="pip$(echo "${_py_version}" | cut -c -1)" +## DGM if ! __check_command_exists "${_pip_cmd}"; then +## DGM echodebug "The pip binary '${_pip_cmd}' was not found in PATH" +## DGM _pip_cmd="pip" +## DGM if ! __check_command_exists "${_pip_cmd}"; then +## DGM echoerror "Unable to find a pip binary" +## DGM return 1 +## DGM fi +## DGM fi +## DGM fi +## DGM echodebug "Installed pip version: $(${_pip_cmd} --version)" +## DGM fi _setuptools_dep="setuptools>=${_MINIMUM_SETUPTOOLS_VERSION},<${_MAXIMUM_SETUPTOOLS_VERSION}" if [ "$_PY_MAJOR_VERSION" -ne 3 ]; then @@ -2816,8 +2816,13 @@ EOM fi echoinfo "Downloading Salt Dependencies from PyPi" - echodebug "Running '${_pip_cmd} download -d /tmp/git/deps ${_PIP_DOWNLOAD_ARGS} .'" - ${_pip_cmd} download -d /tmp/git/deps ${_PIP_DOWNLOAD_ARGS} . + if [ "${OS_NAME}" = "Linux" ]; then + echodebug "Running '${_pip_cmd} download -d /tmp/git/deps ${_PIP_DOWNLOAD_ARGS} -r requirements/static/ci/py${_py_version}/linux.txt" + ${_pip_cmd} download -d /tmp/git/deps ${_PIP_DOWNLOAD_ARGS} -r "requirements/static/ci/py${_py_version}/linux.txt" + else + echodebug "Running '${_pip_cmd} download -d /tmp/git/deps ${_PIP_DOWNLOAD_ARGS} .'" + ${_pip_cmd} download -d /tmp/git/deps ${_PIP_DOWNLOAD_ARGS} . + fi # shellcheck disable=SC2181 if [ $? -ne 0 ]; then echo "Failed to download salt dependencies" From a79cd05fa070f32d23e397d291beb18592ff6741 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 26 Nov 2024 10:45:49 -0700 Subject: [PATCH 134/173] Updated to install jaraco.functools v4.0.0 for splat on Linux --- .github/workflows/ci.yml | 52 ++++++++++++++++++++++++- .github/workflows/templates/generate.py | 29 +++++++------- bootstrap-salt.sh | 2 + 3 files changed, 66 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 980dc7e15..d7fc1acf7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -180,6 +180,51 @@ jobs: + amazonlinux-2: + name: Amazon 2 + if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' + uses: ./.github/workflows/test-linux.yml + needs: + - lint + - generate-actions-workflow + with: + distro-slug: amazonlinux-2 + display-name: Amazon 2 + container-slug: systemd-amazonlinux-2 + timeout: 20 + instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' + + + amazonlinux-2023: + name: Amazon 2023 + if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' + uses: ./.github/workflows/test-linux.yml + needs: + - lint + - generate-actions-workflow + with: + distro-slug: amazonlinux-2023 + display-name: Amazon 2023 + container-slug: systemd-amazonlinux-2023 + timeout: 20 + instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' + + + debian-11: + name: Debian 11 + if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' + uses: ./.github/workflows/test-linux.yml + needs: + - lint + - generate-actions-workflow + with: + distro-slug: debian-11 + display-name: Debian 11 + container-slug: systemd-debian-11 + timeout: 20 + instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' + + debian-12: name: Debian 12 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' @@ -192,7 +237,7 @@ jobs: display-name: Debian 12 container-slug: systemd-debian-12 timeout: 20 - instances: '["stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' + instances: '["stable-3007", "git-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' photon-5: @@ -237,7 +282,7 @@ jobs: display-name: Rocky Linux 9 container-slug: systemd-rockylinux-9 timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' + instances: '["stable-3006", "git-3006", "onedir-3006", "stable-3006-8", "stable-3007", "git-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' ubuntu-2204: @@ -267,6 +312,9 @@ jobs: - macos-13 - macos-14 - windows-2022 + - amazonlinux-2 + - amazonlinux-2023 + - debian-11 - debian-12 - photon-5 - rockylinux-8 diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index 0207d9888..cab372d80 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -8,16 +8,15 @@ # only test against current containers with systemd # will add these when they become available with systemd -# "amazonlinux-2", -# "amazonlinux-2023", -# "debian-11", -# "debian-12", # "debian-13", # "fedora-40", # "photon-4", # "ubuntu-2004", # "ubuntu-2404", LINUX_DISTROS = [ + "amazonlinux-2", + "amazonlinux-2023", + "debian-11", "debian-12", "photon-5", "rockylinux-8", @@ -37,15 +36,15 @@ # only test against current containers with systemd # will add these when they become available with systemd -# "amazonlinux-2", -# "amazonlinux-2023", -# "debian-11", # "debian-13", # "fedora-40", # "photon-4", # "ubuntu-2004", # "ubuntu-2404", STABLE_DISTROS = [ + "amazonlinux-2", + "amazonlinux-2023", + "debian-11", "debian-12", "photon-5", "rockylinux-8", @@ -55,15 +54,15 @@ # only test against current containers with systemd # will add these when they become available with systemd -# "amazonlinux-2", -# "amazonlinux-2023", -# "debian-11", # "debian-13", # "fedora-40", # "photon-4", # "ubuntu-2004", # "ubuntu-2404", ONEDIR_DISTROS = [ + "amazonlinux-2", + "amazonlinux-2023", + "debian-11", "debian-12", "photon-5", "rockylinux-8", @@ -97,6 +96,7 @@ "photon-5", ] +# "rockylinux-9", # "ubuntu-2204", BLACKLIST_GIT_3006 = [ "amazonlinux-2", @@ -106,36 +106,35 @@ "fedora-40", "photon-4", "photon-5", - "rockylinux-9", "ubuntu-2004", "ubuntu-2404", ] +# "debian-12", +# "rockylinux-9", # "ubuntu-2204", BLACKLIST_GIT_3007 = [ "amazonlinux-2", "amazonlinux-2023", "debian-11", - "debian-12", "debian-13", "fedora-40", "photon-4", "photon-5", - "rockylinux-9", "ubuntu-2004", "ubuntu-2404", ] +# "debian-12", +# "rockylinux-9", BLACKLIST_GIT_MASTER = [ "amazonlinux-2", "amazonlinux-2023", "debian-11", - "debian-12", "debian-13", "fedora-40", "photon-4", "photon-5", - "rockylinux-9", ] SALT_VERSIONS = [ diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index b41c25688..f094907b9 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -2817,6 +2817,8 @@ __install_salt_from_repo() { echoinfo "Downloading Salt Dependencies from PyPi" if [ "${OS_NAME}" = "Linux" ]; then + echodebug "Solving jaraco.functools splat issue, installing jaraco.functools v4.0.0" + ${_pip_cmd} install "jaraco.functools==4.0.0" || return 1 echodebug "Running '${_pip_cmd} download -d /tmp/git/deps ${_PIP_DOWNLOAD_ARGS} -r requirements/static/ci/py${_py_version}/linux.txt" ${_pip_cmd} download -d /tmp/git/deps ${_PIP_DOWNLOAD_ARGS} -r "requirements/static/ci/py${_py_version}/linux.txt" else From e4b316413634295557f769ea7178b920b6be7da1 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 26 Nov 2024 11:40:09 -0700 Subject: [PATCH 135/173] Updated containers, and added back in AlmaLinux --- .github/workflows/ci.yml | 48 ------------------------- .github/workflows/templates/generate.py | 18 +++++----- bootstrap-salt.sh | 32 ++++++++--------- 3 files changed, 25 insertions(+), 73 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d7fc1acf7..3af9085ab 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -180,51 +180,6 @@ jobs: - amazonlinux-2: - name: Amazon 2 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: amazonlinux-2 - display-name: Amazon 2 - container-slug: systemd-amazonlinux-2 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' - - - amazonlinux-2023: - name: Amazon 2023 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: amazonlinux-2023 - display-name: Amazon 2023 - container-slug: systemd-amazonlinux-2023 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' - - - debian-11: - name: Debian 11 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: debian-11 - display-name: Debian 11 - container-slug: systemd-debian-11 - timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' - - debian-12: name: Debian 12 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' @@ -312,9 +267,6 @@ jobs: - macos-13 - macos-14 - windows-2022 - - amazonlinux-2 - - amazonlinux-2023 - - debian-11 - debian-12 - photon-5 - rockylinux-8 diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index cab372d80..68ee54ee5 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -8,15 +8,15 @@ # only test against current containers with systemd # will add these when they become available with systemd +# "amazonlinux-2", +# "amazonlinux-2023", +# "debian-11", # "debian-13", # "fedora-40", # "photon-4", # "ubuntu-2004", # "ubuntu-2404", LINUX_DISTROS = [ - "amazonlinux-2", - "amazonlinux-2023", - "debian-11", "debian-12", "photon-5", "rockylinux-8", @@ -36,15 +36,15 @@ # only test against current containers with systemd # will add these when they become available with systemd +# "amazonlinux-2", +# "amazonlinux-2023", +# "debian-11", # "debian-13", # "fedora-40", # "photon-4", # "ubuntu-2004", # "ubuntu-2404", STABLE_DISTROS = [ - "amazonlinux-2", - "amazonlinux-2023", - "debian-11", "debian-12", "photon-5", "rockylinux-8", @@ -54,15 +54,15 @@ # only test against current containers with systemd # will add these when they become available with systemd +# "amazonlinux-2", +# "amazonlinux-2023", +# "debian-11", # "debian-13", # "fedora-40", # "photon-4", # "ubuntu-2004", # "ubuntu-2404", ONEDIR_DISTROS = [ - "amazonlinux-2", - "amazonlinux-2023", - "debian-11", "debian-12", "photon-5", "rockylinux-8", diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index f094907b9..4e39521f6 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -5146,79 +5146,79 @@ install_oracle_linux_check_services() { ####################################################################################################################### # -# RockyLinux Install Functions +# ALmaLinux Install Functions # -install_rockylinux_stable_deps() { +install_almalinux_stable_deps() { install_centos_stable_deps || return 1 return 0 } -install_rockylinux_git_deps() { +install_almalinux_git_deps() { install_centos_git_deps || return 1 return 0 } -install_rockylinux_onedir_deps() { +install_almalinux_onedir_deps() { install_centos_onedir_deps || return 1 return 0 } -install_rockylinux_testing_deps() { +install_almalinux_testing_deps() { install_centos_testing_deps || return 1 return 0 } -install_rockylinux_stable() { +install_almalinux_stable() { install_centos_stable || return 1 return 0 } -install_rockylinux_git() { +install_almalinux_git() { install_centos_git || return 1 return 0 } -install_rockylinux_onedir() { +install_almalinux_onedir() { install_centos_onedir || return 1 return 0 } -install_rockylinux_testing() { +install_almalinux_testing() { install_centos_testing || return 1 return 0 } -install_rockylinux_stable_post() { +install_almalinux_stable_post() { install_centos_stable_post || return 1 return 0 } -install_rockylinux_git_post() { +install_almalinux_git_post() { install_centos_git_post || return 1 return 0 } -install_rockylinux_onedir_post() { +install_almalinux_onedir_post() { install_centos_onedir_post || return 1 return 0 } -install_rockylinux_testing_post() { +install_almalinux_testing_post() { install_centos_testing_post || return 1 return 0 } -install_rockylinux_restart_daemons() { +install_almalinux_restart_daemons() { install_centos_restart_daemons || return 1 return 0 } -install_rockylinux_check_services() { +install_almalinux_check_services() { install_centos_check_services || return 1 return 0 } # -# Ended RockyLinux Install Functions +# Ended AlmaLinux Install Functions # ####################################################################################################################### From 9b3055b1bd2c619d979dfce29edc80f756671c4d Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 26 Nov 2024 15:06:24 -0700 Subject: [PATCH 136/173] Testing fixes for PyPI requirements clashes --- bootstrap-salt.sh | 50 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 15 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 4e39521f6..49c006f3c 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -2815,27 +2815,47 @@ __install_salt_from_repo() { fi fi - echoinfo "Downloading Salt Dependencies from PyPi" - if [ "${OS_NAME}" = "Linux" ]; then - echodebug "Solving jaraco.functools splat issue, installing jaraco.functools v4.0.0" - ${_pip_cmd} install "jaraco.functools==4.0.0" || return 1 - echodebug "Running '${_pip_cmd} download -d /tmp/git/deps ${_PIP_DOWNLOAD_ARGS} -r requirements/static/ci/py${_py_version}/linux.txt" - ${_pip_cmd} download -d /tmp/git/deps ${_PIP_DOWNLOAD_ARGS} -r "requirements/static/ci/py${_py_version}/linux.txt" - else - echodebug "Running '${_pip_cmd} download -d /tmp/git/deps ${_PIP_DOWNLOAD_ARGS} .'" - ${_pip_cmd} download -d /tmp/git/deps ${_PIP_DOWNLOAD_ARGS} . - fi + ## DGM echoinfo "Downloading Salt Dependencies from PyPi" + ## DGM if [ "${OS_NAME}" = "Linux" ]; then + ## DGM echodebug "Solving jaraco.functools splat issue, installing jaraco.functools v4.0.0" + ## DGM ${_pip_cmd} install "jaraco.functools==4.0.0" || return 1 + ## DGM echodebug "Running '${_pip_cmd} download -d /tmp/git/deps ${_PIP_DOWNLOAD_ARGS} -r requirements/static/ci/py${_py_version}/linux.txt" + ## DGM ${_pip_cmd} download -d /tmp/git/deps ${_PIP_DOWNLOAD_ARGS} -r "requirements/static/ci/py${_py_version}/linux.txt" + ## DGM else + ## DGM echodebug "Running '${_pip_cmd} download -d /tmp/git/deps ${_PIP_DOWNLOAD_ARGS} .'" + ## DGM ${_pip_cmd} download -d /tmp/git/deps ${_PIP_DOWNLOAD_ARGS} . + ## DGM fi + ## DGM # shellcheck disable=SC2181 + ## DGM if [ $? -ne 0 ]; then + ## DGM echo "Failed to download salt dependencies" + ## DGM return 1 + ## DGM fi + + + ## DGM echoinfo "Installing Downloaded Salt Dependencies" + ## DGM echodebug "Running '${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --ignore-installed ${_PIP_INSTALL_ARGS} /tmp/git/deps/*'" + ## DGM ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --ignore-installed ${_PIP_INSTALL_ARGS} /tmp/git/deps/* || return 1 + + rm -f /tmp/git/deps/* + + ## DGM different attempt, try installing requirements + echoinfo "DGM Installing Salt reuqirements from PyPi, ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --ignore-installed ${_PIP_INSTALL_ARGS} -r requirements/static/ci/py${_py_version}/linux.txt" + ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --ignore-installed ${_PIP_INSTALL_ARGS} -r "requirements/static/ci/py${_py_version}/linux.txt" # shellcheck disable=SC2181 if [ $? -ne 0 ]; then - echo "Failed to download salt dependencies" + echo "Failed to install salt requirements for the version of Python ${_py_version}" return 1 fi + if [ "${OS_NAME}" = "Linux" ]; then + ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --ignore-installed --upgrade ${_PIP_INSTALL_ARGS} "jaraco.functools==4.1.0" || return 1 + ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --ignore-installed --upgrade ${_PIP_INSTALL_ARGS} "jaraco.text==4.0.0" || return 1 + ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --ignore-installed --upgrade ${_PIP_INSTALL_ARGS} "jaraco.collections==5.1.0" || return 1 + ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --ignore-installed --upgrade ${_PIP_INSTALL_ARGS} "jaraco.context==6.0.1" || return 1 + ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --ignore-installed --upgrade ${_PIP_INSTALL_ARGS} "jaraco.classes==3.4.0" || return 1 - echoinfo "Installing Downloaded Salt Dependencies" - echodebug "Running '${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --ignore-installed ${_PIP_INSTALL_ARGS} /tmp/git/deps/*'" - ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --ignore-installed ${_PIP_INSTALL_ARGS} /tmp/git/deps/* || return 1 - rm -f /tmp/git/deps/* + ${_pip_cmd} list + fi echoinfo "Building Salt Python Wheel" if [ "$_ECHO_DEBUG" -eq $BS_TRUE ]; then From d0bd4612a69b2075c73f4bfdb94d307181c56dd9 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 26 Nov 2024 17:32:02 -0700 Subject: [PATCH 137/173] Testing pip installs from PyPI --- bootstrap-salt.sh | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 49c006f3c..5dacb8828 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -26,7 +26,7 @@ #====================================================================================================================== set -o nounset # Treat unset variables as an error -__ScriptVersion="2024.11.26" +__ScriptVersion="2024.11.27" __ScriptName="bootstrap-salt.sh" __ScriptFullName="$0" @@ -2797,6 +2797,9 @@ __install_salt_from_repo() { echodebug "Running '${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --upgrade ${_PIP_INSTALL_ARGS} wheel ${_setuptools_dep}" ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --upgrade ${_PIP_INSTALL_ARGS} wheel "${_setuptools_dep}" + echodebug "DGM doing pip list" + ${_pip_cmd} list + echoinfo "Installing salt using ${_py_exe}, $(${_py_exe} --version)" cd "${_SALT_GIT_CHECKOUT_DIR}" || return 1 @@ -2836,10 +2839,16 @@ __install_salt_from_repo() { ## DGM echodebug "Running '${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --ignore-installed ${_PIP_INSTALL_ARGS} /tmp/git/deps/*'" ## DGM ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --ignore-installed ${_PIP_INSTALL_ARGS} /tmp/git/deps/* || return 1 + echodebug "DGM doing pip list" + ${_pip_cmd} list + rm -f /tmp/git/deps/* + echodebug "DGM doing pip list" + ${_pip_cmd} list + ## DGM different attempt, try installing requirements - echoinfo "DGM Installing Salt reuqirements from PyPi, ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --ignore-installed ${_PIP_INSTALL_ARGS} -r requirements/static/ci/py${_py_version}/linux.txt" + echoinfo "DGM Installing Salt requirements from PyPi, ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --ignore-installed ${_PIP_INSTALL_ARGS} -r requirements/static/ci/py${_py_version}/linux.txt" ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --ignore-installed ${_PIP_INSTALL_ARGS} -r "requirements/static/ci/py${_py_version}/linux.txt" # shellcheck disable=SC2181 if [ $? -ne 0 ]; then @@ -2854,6 +2863,7 @@ __install_salt_from_repo() { ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --ignore-installed --upgrade ${_PIP_INSTALL_ARGS} "jaraco.context==6.0.1" || return 1 ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --ignore-installed --upgrade ${_PIP_INSTALL_ARGS} "jaraco.classes==3.4.0" || return 1 + echodebug "DGM doing pip list" ${_pip_cmd} list fi @@ -6400,7 +6410,7 @@ install_photon_deps() { __PACKAGES="${__PACKAGES} libyaml procps-ng python${PY_PKG_VER}-crypto python${PY_PKG_VER}-jinja2" __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-msgpack python${PY_PKG_VER}-requests python${PY_PKG_VER}-zmq" __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-pip python${PY_PKG_VER}-m2crypto python${PY_PKG_VER}-pyyaml" - __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-systemd" + __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-systemd sudo shadow" if [ "${_EXTRA_PACKAGES}" != "" ]; then echoinfo "Installing the following extra packages as requested: ${_EXTRA_PACKAGES}" @@ -6450,6 +6460,14 @@ install_photon_git_deps() { __PACKAGES="${__PACKAGES} git" fi + if ! __check_command_exists sudo; then + __PACKAGES="${__PACKAGES} sudo" + fi + + if ! __check_command_exists usermod; then + __PACKAGES="${__PACKAGES} shadow" + fi + if [ -n "${__PACKAGES}" ]; then # shellcheck disable=SC2086 __tdnf_install_noinput ${__PACKAGES} || return 1 @@ -6498,6 +6516,8 @@ install_photon_git() { return 1 fi + install_photon_git_deps + if [ -f "${_SALT_GIT_CHECKOUT_DIR}/salt/syspaths.py" ]; then ${_PYEXE} setup.py --salt-config-dir="$_SALT_ETC_DIR" --salt-cache-dir="${_SALT_CACHE_DIR}" ${SETUP_PY_INSTALL_ARGS} install --prefix=/usr || return 1 else @@ -6606,7 +6626,7 @@ install_photon_onedir_deps() { __install_saltstack_photon_onedir_repository || return 1 fi - __PACKAGES="procps-ng" + __PACKAGES="procps-ng sudo shadow" # shellcheck disable=SC2086 __tdnf_install_noinput ${__PACKAGES} || return 1 From 3c0f302b77f603b4abace42d0b164c789222bcbc Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 27 Nov 2024 11:26:48 -0700 Subject: [PATCH 138/173] Added support for AlmaLinux, removed Rocky Linux 9 from git testing --- .github/workflows/ci.yml | 4 +- .github/workflows/templates/generate.py | 8 +-- bootstrap-salt.sh | 75 +------------------------ 3 files changed, 7 insertions(+), 80 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3af9085ab..e88f30f52 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -207,7 +207,7 @@ jobs: display-name: Photon OS 5 container-slug: systemd-photon-5 timeout: 20 - instances: '["stable-3006", "onedir-3006", "stable-3006-8", "latest", "default"]' + instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' rockylinux-8: @@ -237,7 +237,7 @@ jobs: display-name: Rocky Linux 9 container-slug: systemd-rockylinux-9 timeout: 20 - instances: '["stable-3006", "git-3006", "onedir-3006", "stable-3006-8", "stable-3007", "git-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' + instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' ubuntu-2204: diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index 68ee54ee5..ca31b4cfa 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -91,12 +91,11 @@ "ubuntu-2404", ] +# "photon-5", BLACKLIST_3007 = [ "photon-4", - "photon-5", ] -# "rockylinux-9", # "ubuntu-2204", BLACKLIST_GIT_3006 = [ "amazonlinux-2", @@ -106,12 +105,12 @@ "fedora-40", "photon-4", "photon-5", + "rockylinux-9", "ubuntu-2004", "ubuntu-2404", ] # "debian-12", -# "rockylinux-9", # "ubuntu-2204", BLACKLIST_GIT_3007 = [ "amazonlinux-2", @@ -121,12 +120,12 @@ "fedora-40", "photon-4", "photon-5", + "rockylinux-9", "ubuntu-2004", "ubuntu-2404", ] # "debian-12", -# "rockylinux-9", BLACKLIST_GIT_MASTER = [ "amazonlinux-2", "amazonlinux-2023", @@ -135,6 +134,7 @@ "fedora-40", "photon-4", "photon-5", + "rockylinux-9", ] SALT_VERSIONS = [ diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 5dacb8828..f43acdfbe 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -2742,44 +2742,6 @@ __install_salt_from_repo() { echodebug "Installed pip version: $(${_pip_cmd} --version)" -## DGM CHECK_PIP_VERSION_SCRIPT=$(cat << EOM -## DGM import sys -## DGM try: -## DGM import pip -## DGM installed_pip_version=tuple([int(part.strip()) for part in pip.__version__.split('.') if part.isdigit()]) -## DGM desired_pip_version=($(echo ${_MINIMUM_PIP_VERSION} | sed 's/\./, /g' )) -## DGM if installed_pip_version < desired_pip_version: -## DGM print('Desired pip version {!r} > Installed pip version {!r}'.format('.'.join(map(str, desired_pip_version)), '.'.join(map(str, installed_pip_version)))) -## DGM sys.exit(1) -## DGM print('Desired pip version {!r} < Installed pip version {!r}'.format('.'.join(map(str, desired_pip_version)), '.'.join(map(str, installed_pip_version)))) -## DGM sys.exit(0) -## DGM except ImportError: -## DGM print('Failed to import pip') -## DGM sys.exit(1) -## DGM EOM -## DGM ) -## DGM if ! ${_py_exe} -c "$CHECK_PIP_VERSION_SCRIPT"; then -## DGM # Upgrade pip to at least 1.2 which is when we can start using "python3 -m pip" -## DGM echodebug "Running '${_pip_cmd} install ${_PIP_INSTALL_ARGS} pip>=${_MINIMUM_PIP_VERSION}'" -## DGM ${_pip_cmd} install ${_PIP_INSTALL_ARGS} -v "pip>=${_MINIMUM_PIP_VERSION}" -## DGM sleep 1 -## DGM echodebug "PATH: ${PATH}" -## DGM _pip_cmd="pip${_py_version}" -## DGM if ! __check_command_exists "${_pip_cmd}"; then -## DGM echodebug "The pip binary '${_pip_cmd}' was not found in PATH" -## DGM _pip_cmd="pip$(echo "${_py_version}" | cut -c -1)" -## DGM if ! __check_command_exists "${_pip_cmd}"; then -## DGM echodebug "The pip binary '${_pip_cmd}' was not found in PATH" -## DGM _pip_cmd="pip" -## DGM if ! __check_command_exists "${_pip_cmd}"; then -## DGM echoerror "Unable to find a pip binary" -## DGM return 1 -## DGM fi -## DGM fi -## DGM fi -## DGM echodebug "Installed pip version: $(${_pip_cmd} --version)" -## DGM fi - _setuptools_dep="setuptools>=${_MINIMUM_SETUPTOOLS_VERSION},<${_MAXIMUM_SETUPTOOLS_VERSION}" if [ "$_PY_MAJOR_VERSION" -ne 3 ]; then echoerror "Python version is no longer supported, only Python 3" @@ -2797,15 +2759,11 @@ __install_salt_from_repo() { echodebug "Running '${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --upgrade ${_PIP_INSTALL_ARGS} wheel ${_setuptools_dep}" ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --upgrade ${_PIP_INSTALL_ARGS} wheel "${_setuptools_dep}" - echodebug "DGM doing pip list" - ${_pip_cmd} list - echoinfo "Installing salt using ${_py_exe}, $(${_py_exe} --version)" cd "${_SALT_GIT_CHECKOUT_DIR}" || return 1 mkdir -p /tmp/git/deps echodebug "Created directory /tmp/git/deps" - echodebug "Installing Salt dependencies for Salt version $(python3 salt/version.py)" if [ ${DISTRO_NAME_L} = "ubuntu" ] && [ "$DISTRO_MAJOR_VERSION" -eq 22 ]; then echodebug "Ubuntu 22.04 has problem with base.txt requirements file, not parsing sys_platform == 'win32', upgrading from default pip works" @@ -2818,37 +2776,9 @@ __install_salt_from_repo() { fi fi - ## DGM echoinfo "Downloading Salt Dependencies from PyPi" - ## DGM if [ "${OS_NAME}" = "Linux" ]; then - ## DGM echodebug "Solving jaraco.functools splat issue, installing jaraco.functools v4.0.0" - ## DGM ${_pip_cmd} install "jaraco.functools==4.0.0" || return 1 - ## DGM echodebug "Running '${_pip_cmd} download -d /tmp/git/deps ${_PIP_DOWNLOAD_ARGS} -r requirements/static/ci/py${_py_version}/linux.txt" - ## DGM ${_pip_cmd} download -d /tmp/git/deps ${_PIP_DOWNLOAD_ARGS} -r "requirements/static/ci/py${_py_version}/linux.txt" - ## DGM else - ## DGM echodebug "Running '${_pip_cmd} download -d /tmp/git/deps ${_PIP_DOWNLOAD_ARGS} .'" - ## DGM ${_pip_cmd} download -d /tmp/git/deps ${_PIP_DOWNLOAD_ARGS} . - ## DGM fi - ## DGM # shellcheck disable=SC2181 - ## DGM if [ $? -ne 0 ]; then - ## DGM echo "Failed to download salt dependencies" - ## DGM return 1 - ## DGM fi - - - ## DGM echoinfo "Installing Downloaded Salt Dependencies" - ## DGM echodebug "Running '${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --ignore-installed ${_PIP_INSTALL_ARGS} /tmp/git/deps/*'" - ## DGM ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --ignore-installed ${_PIP_INSTALL_ARGS} /tmp/git/deps/* || return 1 - - echodebug "DGM doing pip list" - ${_pip_cmd} list - rm -f /tmp/git/deps/* - echodebug "DGM doing pip list" - ${_pip_cmd} list - - ## DGM different attempt, try installing requirements - echoinfo "DGM Installing Salt requirements from PyPi, ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --ignore-installed ${_PIP_INSTALL_ARGS} -r requirements/static/ci/py${_py_version}/linux.txt" + echodebug "Installing Salt requirements from PyPi, ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --ignore-installed ${_PIP_INSTALL_ARGS} -r requirements/static/ci/py${_py_version}/linux.txt" ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --ignore-installed ${_PIP_INSTALL_ARGS} -r "requirements/static/ci/py${_py_version}/linux.txt" # shellcheck disable=SC2181 if [ $? -ne 0 ]; then @@ -2862,9 +2792,6 @@ __install_salt_from_repo() { ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --ignore-installed --upgrade ${_PIP_INSTALL_ARGS} "jaraco.collections==5.1.0" || return 1 ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --ignore-installed --upgrade ${_PIP_INSTALL_ARGS} "jaraco.context==6.0.1" || return 1 ${_pip_cmd} install ${_USE_BREAK_SYSTEM_PACKAGES} --ignore-installed --upgrade ${_PIP_INSTALL_ARGS} "jaraco.classes==3.4.0" || return 1 - - echodebug "DGM doing pip list" - ${_pip_cmd} list fi echoinfo "Building Salt Python Wheel" From 7492ff27f307ffd6068c5c6f4abbb640c3df522e Mon Sep 17 00:00:00 2001 From: Salt Project Packaging Date: Wed, 27 Nov 2024 18:56:47 +0000 Subject: [PATCH 139/173] Update develop branch for the v2024.11.27 release --- CHANGELOG.md | 8 ++++++++ bootstrap-salt.ps1 | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a4d9ac944..ae589d07b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +# v2024.11.27 + +## What's Changed + +- Testing git install methods by @dmurphy18 in https://github.com/saltstack/salt-bootstrap/pull/2057 + +**Full Changelog**: https://github.com/saltstack/salt-bootstrap/compare/v2024.11.26...v2024.11.27 + # v2024.11.26 ## What's Changed diff --git a/bootstrap-salt.ps1 b/bootstrap-salt.ps1 index cd70eebad..4a0d30fda 100644 --- a/bootstrap-salt.ps1 +++ b/bootstrap-salt.ps1 @@ -108,7 +108,7 @@ if ($help) { exit 0 } -$__ScriptVersion = "2024.11.26" +$__ScriptVersion = "2024.11.27" $ScriptName = $myInvocation.MyCommand.Name # We'll check for the Version next, because it also has no requirements From fdcfec993ba75f8631dc1a5ccc6697c65b9ade26 Mon Sep 17 00:00:00 2001 From: Salt Project Packaging Date: Wed, 27 Nov 2024 18:57:26 +0000 Subject: [PATCH 140/173] Update develop branch for the v2024.11.27 release --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae589d07b..3b75f96ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,14 @@ **Full Changelog**: https://github.com/saltstack/salt-bootstrap/compare/v2024.11.26...v2024.11.27 +# v2024.11.27 + +## What's Changed + +- Testing git install methods by @dmurphy18 in https://github.com/saltstack/salt-bootstrap/pull/2057 + +**Full Changelog**: https://github.com/saltstack/salt-bootstrap/compare/v2024.11.26...v2024.11.27 + # v2024.11.26 ## What's Changed From 5339251d5938fa466b2f6fe8ba5b807b9b0456b5 Mon Sep 17 00:00:00 2001 From: Salt Project Packaging Date: Wed, 27 Nov 2024 18:57:41 +0000 Subject: [PATCH 141/173] Update README.rst with 2024.11.27 release sha256sum --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index 8a2310760..745b41917 100644 --- a/README.rst +++ b/README.rst @@ -37,6 +37,7 @@ sum** of the downloaded ``bootstrap-salt.sh`` file. The SHA256 sum of the ``bootstrap-salt.sh`` file, per release, is: +- 2024.11.27: ``e972bd1ef01d09cd1d9294374ef974c9e3dd9a2aee37cf3859144585fd8bf1d0`` - 2024.11.26: ``832c7a20b96e1df171d715323df9afff8a11aef42d15598c007f240bc89d723c`` - 2024.11.21: ``ddf624c3a94d721da3f7629402a6c7ecc9dd96d13c1ead2a626314e97cee982a`` - 2024.11.07: ``70a9783649e129985563d1a86cf28b8984499643e62ae1dc47dc008bd204fcbb`` From 535a53ea369b40a6219afd14df6cd6bec6633c78 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 27 Nov 2024 12:01:16 -0700 Subject: [PATCH 142/173] Updated date to overcome issue with existing tag for 2024.11.27 --- bootstrap-salt.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index f43acdfbe..08a513ffa 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -26,7 +26,7 @@ #====================================================================================================================== set -o nounset # Treat unset variables as an error -__ScriptVersion="2024.11.27" +__ScriptVersion="2024.11.28" __ScriptName="bootstrap-salt.sh" __ScriptFullName="$0" From 677ecee80198595ea58ce21405086b020ce49869 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 27 Nov 2024 12:01:16 -0700 Subject: [PATCH 143/173] Updated date to overcome issue with existing tag for 2024.11.27 --- bootstrap-salt.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index f43acdfbe..08a513ffa 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -26,7 +26,7 @@ #====================================================================================================================== set -o nounset # Treat unset variables as an error -__ScriptVersion="2024.11.27" +__ScriptVersion="2024.11.28" __ScriptName="bootstrap-salt.sh" __ScriptFullName="$0" From 578ee0eeeb9e13eaa5bfbb5c20ff037a44595333 Mon Sep 17 00:00:00 2001 From: Salt Project Packaging Date: Wed, 27 Nov 2024 19:20:52 +0000 Subject: [PATCH 144/173] Update develop branch for the v2024.11.27 release --- CHANGELOG.md | 4 ++++ bootstrap-salt.sh | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b75f96ea..4804c4edc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # v2024.11.27 +**Full Changelog**: https://github.com/saltstack/salt-bootstrap/compare/v2024.11.27...v2024.11.27 + +# v2024.11.27 + ## What's Changed - Testing git install methods by @dmurphy18 in https://github.com/saltstack/salt-bootstrap/pull/2057 diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 08a513ffa..f43acdfbe 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -26,7 +26,7 @@ #====================================================================================================================== set -o nounset # Treat unset variables as an error -__ScriptVersion="2024.11.28" +__ScriptVersion="2024.11.27" __ScriptName="bootstrap-salt.sh" __ScriptFullName="$0" From 72254edd67aac6d08e14f73c9a8e7653b95d62ad Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 27 Nov 2024 12:31:39 -0700 Subject: [PATCH 145/173] Forcing git tag for release 2024.11.27 due to existing tag which was set by accident --- .github/workflows/release.yml | 2 +- bootstrap-salt.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f31d6c470..8e54b67b5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -159,7 +159,7 @@ jobs: - name: Tag The ${{ needs.update-develop.outputs.release-version }} Release run: | - git tag --no-sign -m "Release ${{ needs.update-develop.outputs.release-version }}" -a ${{ needs.update-develop.outputs.release-version }} + git tag -f --no-sign -m "Release ${{ needs.update-develop.outputs.release-version }}" -a ${{ needs.update-develop.outputs.release-version }} - name: Update bootstrap-salt.sh sha256sum's run: | diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 08a513ffa..f43acdfbe 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -26,7 +26,7 @@ #====================================================================================================================== set -o nounset # Treat unset variables as an error -__ScriptVersion="2024.11.28" +__ScriptVersion="2024.11.27" __ScriptName="bootstrap-salt.sh" __ScriptFullName="$0" From ab523a74dc360dadd0d7f629105d4afedcfbd6cd Mon Sep 17 00:00:00 2001 From: Salt Project Packaging Date: Wed, 27 Nov 2024 19:40:25 +0000 Subject: [PATCH 146/173] Update develop branch for the v2024.11.27 release --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4804c4edc..cb299377b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ # v2024.11.27 +**Full Changelog**: https://github.com/saltstack/salt-bootstrap/compare/v2024.11.27...v2024.11.27 + +# v2024.11.27 + ## What's Changed - Testing git install methods by @dmurphy18 in https://github.com/saltstack/salt-bootstrap/pull/2057 From 777c3ad055a6f5d29641c1c150935e1f093bf93c Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 27 Nov 2024 12:47:45 -0700 Subject: [PATCH 147/173] Revert forcing git tag, other issues with check sums calculated, easy solution is wait a day to release --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8e54b67b5..f31d6c470 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -159,7 +159,7 @@ jobs: - name: Tag The ${{ needs.update-develop.outputs.release-version }} Release run: | - git tag -f --no-sign -m "Release ${{ needs.update-develop.outputs.release-version }}" -a ${{ needs.update-develop.outputs.release-version }} + git tag --no-sign -m "Release ${{ needs.update-develop.outputs.release-version }}" -a ${{ needs.update-develop.outputs.release-version }} - name: Update bootstrap-salt.sh sha256sum's run: | From 5689a93c798d054e6dff2fe8d0403678a1fa3552 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Fri, 29 Nov 2024 10:41:05 -0700 Subject: [PATCH 148/173] Updated date --- bootstrap-salt.ps1 | 2 +- bootstrap-salt.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bootstrap-salt.ps1 b/bootstrap-salt.ps1 index 4a0d30fda..8d0c3c628 100644 --- a/bootstrap-salt.ps1 +++ b/bootstrap-salt.ps1 @@ -108,7 +108,7 @@ if ($help) { exit 0 } -$__ScriptVersion = "2024.11.27" +$__ScriptVersion = "2024.11.29" $ScriptName = $myInvocation.MyCommand.Name # We'll check for the Version next, because it also has no requirements diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index f43acdfbe..49c71d3cd 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -26,7 +26,7 @@ #====================================================================================================================== set -o nounset # Treat unset variables as an error -__ScriptVersion="2024.11.27" +__ScriptVersion="2024.11.29" __ScriptName="bootstrap-salt.sh" __ScriptFullName="$0" From ce687f4066f1a69a944a6c3f00473b2ecbfa2406 Mon Sep 17 00:00:00 2001 From: Salt Project Packaging Date: Fri, 29 Nov 2024 17:58:13 +0000 Subject: [PATCH 149/173] Update develop branch for the v2024.11.29 release --- CHANGELOG.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cb299377b..a2b85d2d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,14 @@ +# v2024.11.29 + +## What's Changed + +- Updated date to overcome issue with existing tag for 2024.11.27 by @dmurphy18 in https://github.com/saltstack/salt-bootstrap/pull/2058 +- Fix date tag by @dmurphy18 in https://github.com/saltstack/salt-bootstrap/pull/2059 +- Revert forcing git tag, other issues with check sums calculated, easiest solution is wait a day to release by @dmurphy18 in https://github.com/saltstack/salt-bootstrap/pull/2060 +- Updated date by @dmurphy18 in https://github.com/saltstack/salt-bootstrap/pull/2061 + +**Full Changelog**: https://github.com/saltstack/salt-bootstrap/compare/v2024.11.27...v2024.11.29 + # v2024.11.27 **Full Changelog**: https://github.com/saltstack/salt-bootstrap/compare/v2024.11.27...v2024.11.27 From bbaa32a888b4def23a38832c4003d13c8f754e37 Mon Sep 17 00:00:00 2001 From: Salt Project Packaging Date: Fri, 29 Nov 2024 17:58:54 +0000 Subject: [PATCH 150/173] Update README.rst with 2024.11.29 release sha256sum --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index 745b41917..cf80f3943 100644 --- a/README.rst +++ b/README.rst @@ -37,6 +37,7 @@ sum** of the downloaded ``bootstrap-salt.sh`` file. The SHA256 sum of the ``bootstrap-salt.sh`` file, per release, is: +- 2024.11.29: ``0ac87384dee051aceded69704485a5de0e4a308551a462b10c262111b57acff0`` - 2024.11.27: ``e972bd1ef01d09cd1d9294374ef974c9e3dd9a2aee37cf3859144585fd8bf1d0`` - 2024.11.26: ``832c7a20b96e1df171d715323df9afff8a11aef42d15598c007f240bc89d723c`` - 2024.11.21: ``ddf624c3a94d721da3f7629402a6c7ecc9dd96d13c1ead2a626314e97cee982a`` From c4fdafaef3bc5a435810a702c62c646d4701eb25 Mon Sep 17 00:00:00 2001 From: twangboy Date: Wed, 4 Dec 2024 08:40:04 -0700 Subject: [PATCH 151/173] Fix quickstart for Windows with new repo --- salt-quick-start.ps1 | 251 ++++++++++++++++++++++++++++++------------- 1 file changed, 178 insertions(+), 73 deletions(-) diff --git a/salt-quick-start.ps1 b/salt-quick-start.ps1 index 16c705554..f6f79c724 100644 --- a/salt-quick-start.ps1 +++ b/salt-quick-start.ps1 @@ -1,39 +1,31 @@ -function Convert-PSObjectToHashtable { - param ( - [Parameter(ValueFromPipeline)] - $InputObject - ) - if ($null -eq $InputObject) { return $null } - - $is_enum = $InputObject -is [System.Collections.IEnumerable] - $not_string = $InputObject -isnot [string] - if ($is_enum -and $not_string) { - $collection = @( - foreach ($object in $InputObject) { - Convert-PSObjectToHashtable $object - } - ) +<# +.SYNOPSIS + A simple Powershell script to quickly start using Salt. - Write-Host -NoEnumerate $collection - } elseif ($InputObject -is [PSObject]) { - $hash = @{} +.DESCRIPTION + This script will download the latest onedir version of Salt and extract it + into the same directory where the script is run. The script sets up an + environment that will allow you to run salt-call commands. To remove, just + delete the `salt` directory. The environment variables will only be set for + the current powershell session. - foreach ($property in $InputObject.PSObject.Properties) { - $hash[$property.Name] = Convert-PSObjectToHashtable $property.Value - } +.EXAMPLE + ./salt-quick-start.ps1 - $hash - } else { - $InputObject - } -} +.LINK + Salt Bootstrap GitHub Project (script home) - https://github.com/saltstack/salt-bootstrap + Original Vagrant Provisioner Project - https://github.com/saltstack/salty-vagrant + Vagrant Project (utilizes this script) - https://github.com/mitchellh/vagrant + Salt Download Location - https://packages.broadcom.com/artifactory/saltproject-generic/windows/ + Salt Manual Install Directions (Windows) - https://docs.saltproject.io/salt/install-guide/en/latest/topics/install-by-operating-system/windows.html +#> + +# This is so the -Verbose parameter will work +[CmdletBinding()] param() function Expand-ZipFile { # Extract a zip file # - # Used by: - # - Install-SaltMinion - # # Args: # ZipFile (string): The file to extract # Destination (string): The location to extract to @@ -81,60 +73,151 @@ function Expand-ZipFile { Write-Debug "Finished unzipping '$ZipFile' to '$Destination'" } +function Get-FileHash { + # Get-FileHash is a built-in cmdlet in powershell 5+ but we need to support + # powershell 3. This will overwrite the powershell 5 commandlet only for + # this script. But it will provide the missing cmdlet for powershell 3 + [CmdletBinding()] + param( + [Parameter(Mandatory=$true)] + [String] $Path, -[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]'Tls12' + [Parameter(Mandatory=$false)] + [ValidateSet( + "SHA1", + "SHA256", + "SHA384", + "SHA512", + # https://serverfault.com/questions/820300/ + # why-isnt-mactripledes-algorithm-output-in-powershell-stable + "MACTripleDES", # don't use + "MD5", + "RIPEMD160", + IgnoreCase=$true)] + [String] $Algorithm = "SHA256" + ) -$global:ProgressPreference = 'SilentlyContinue' + if ( !(Test-Path $Path) ) { + Write-Verbose "Invalid path for hashing: $Path" + return @{} + } -$RepoUrl = "https://repo.saltproject.io/salt/py3/onedir" + if ( (Get-Item -Path $Path) -isnot [System.IO.FileInfo]) { + Write-Verbose "Not a file for hashing: $Path" + return @{} + } -if ([IntPtr]::Size -eq 4) { - $arch = "x86" -} else { - $arch = "amd64" -} -$enc = [System.Text.Encoding]::UTF8 -try { - $response = Invoke-WebRequest -Uri "$RepoUrl/repo.json" -UseBasicParsing - if ($response.Content.GetType().Name -eq "Byte[]") { - $psobj = $enc.GetString($response.Content) | ConvertFrom-Json + $Path = Resolve-Path -Path $Path - } else { - $psobj = $response.Content | ConvertFrom-Json - } - $hash = Convert-PSObjectToHashtable $psobj -} catch { - Write-Host "repo.json not found at: $RepoUrl" - $hash = @{} -} -$searchVersion = "latest" -if ( $hash.Contains($searchVersion)) { - foreach ($item in $hash.($searchVersion).Keys) { - if ( $item.EndsWith(".zip") ) { - if ( $item.Contains($arch) ) { - $saltFileName = $hash.($searchVersion).($item).name - $saltVersion = $hash.($searchVersion).($item).version - $saltSha512 = $hash.($searchVersion).($item).SHA512 - } + Switch ($Algorithm) { + SHA1 { + $hasher = [System.Security.Cryptography.SHA1CryptoServiceProvider]::Create() + } + SHA256 { + $hasher = [System.Security.Cryptography.SHA256]::Create() + } + SHA384 { + $hasher = [System.Security.Cryptography.SHA384]::Create() + } + SHA512 { + $hasher = [System.Security.Cryptography.SHA512]::Create() + } + MACTripleDES { + $hasher = [System.Security.Cryptography.MACTripleDES]::Create() + } + MD5 { + $hasher = [System.Security.Cryptography.MD5]::Create() + } + RIPEMD160 { + $hasher = [System.Security.Cryptography.RIPEMD160]::Create() } } -} -if ( $saltFileName -and $saltVersion -and $saltSha512 ) { - if ( $RepoUrl.Contains("minor") ) { - $saltFileUrl = @($RepoUrl, $saltVersion, $saltFileName) -join "/" - } else { - $saltFileUrl = @($RepoUrl, "minor", $saltVersion, $saltFileName) -join "/" + + Write-Verbose "Hashing using $Algorithm algorithm" + try { + $data = [System.IO.File]::OpenRead($Path) + $hash = $hasher.ComputeHash($data) + $hash = [System.BitConverter]::ToString($hash) -replace "-","" + return @{ + Path = $Path; + Algorithm = $Algorithm.ToUpper(); + Hash = $hash + } + } catch { + Write-Verbose "Error hashing: $Path" + Write-Verbose "ERROR: $_" + return @{} + } finally { + if ($null -ne $data) { + $data.Close() + } } } -Write-Host "* INFO: Downloading Salt" +#=============================================================================== +# Script settings +#=============================================================================== +[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]'Tls12' +$global:ProgressPreference = 'SilentlyContinue' + +#=============================================================================== +# Declare Variables +#=============================================================================== +$ApiUrl = "https://packages.broadcom.com/artifactory/api/storage/saltproject-generic/onedir" +# Detect architecture ($arch) +if ([IntPtr]::Size -eq 4) { $arch = "x86" } else { $arch = "amd64" } + +#=============================================================================== +# Setting up quickstart environment +#=============================================================================== +Write-Host "" +Write-Host "Setting up quickstart environment for Salt" -ForegroundColor Cyan + +Write-Verbose "Getting version information from Artifactory" +$response = Invoke-WebRequest $ApiUrl -UseBasicParsing +# Convert the output to a powershell object +$psobj = $response.ToString() | ConvertFrom-Json +$Version = $psobj.children[-1].uri.Trim("/") + +Write-Verbose "Getting sha256 hash and download url from Artifactory" +$saltFileName = "salt-$Version-onedir-windows-$arch.zip" +$response = Invoke-WebRequest "$ApiUrl/$Version/$saltFileName" -UseBasicParsing +$psobj = $response.ToString() | ConvertFrom-Json +$saltFileUrl = $psobj.downloadUri +$saltSha256 = $psobj.checksums.sha256 + +Write-Verbose "URL: $saltFileUrl" +Write-Host "* INFO: Downloading Salt: " -NoNewline Invoke-WebRequest -Uri $saltFileUrl -OutFile .\salt.zip +if ( Test-Path -Path .\salt.zip ) { + Write-Host "Success" -ForegroundColor Green +} else { + Write-Host "Failed" -ForegroundColor Red + exit 1 +} +$localSha256 = (Get-FileHash -Path .\salt.zip -Algorithm SHA256).Hash +Write-Verbose "Local Hash: $localSha256" +Write-Verbose "Remote Hash: $saltSha256" + +Write-Host "* INFO: Comparing Hash: " -NoNewline +if ( $localSha256 -eq $saltSha256 ) { + Write-Host "Success" -ForegroundColor Green +} else { + Write-Host "Failed" -ForegroundColor Red + exit 1 +} -Write-Host "* INFO: Extracting Salt" +Write-Host "* INFO: Extracting Salt: " -NoNewline Expand-ZipFile -ZipFile .\salt.zip -Destination . +if ( Test-Path -Path .\salt\Scripts\python.exe ) { + Write-Host "Success" -ForegroundColor Green +} else { + Write-Host "Failed" -ForegroundColor Red + exit 1 +} +Write-Host "* INFO: Creating Saltfile: " -NoNewline $PATH = $(Get-Location).Path - $saltfile_contents = @" salt-call: local: True @@ -143,20 +226,42 @@ salt-call: cachedir: $PATH\salt\var\cache\salt file_root: $PATH\salt\srv\salt "@ - Set-Content -Path .\salt\Saltfile -Value $saltfile_contents +if ( Test-Path -Path .\salt\Saltfile ) { + Write-Host "Success" -ForegroundColor Green +} else { + Write-Host "Failed" -ForegroundColor Red + exit 1 +} New-Item -Path "$PATH\salt\var\log\salt" -Type Directory -Force | Out-Null New-Item -Path "$PATH\salt\conf" -Type Directory -Force | Out-Null New-Item -Path "$PATH\salt\var\cache\salt" -Type Directory -Force | Out-Null New-Item -Path "$PATH\salt\srv\salt" -Type Directory -Force | Out-Null -Write-Host "* INFO: Adding Salt to current path" -Write-Host "* INFO: $PATH\salt" +Write-Host "* INFO: Adding Salt to current path: " -NoNewline $env:Path = "$PATH\salt;$env:PATH" +Write-Verbose $env:Path +if ( $env:PATH -Like "*$PATH\salt*" ) { + Write-Host "Success" -ForegroundColor Green +} else { + Write-Host "Failed" -ForegroundColor Red + exit 1 +} +Write-Host "* INFO: $PATH\salt" -Write-Host "* INFO: Setting the SALT_SALTFILE environment variable" -Write-Host "* INFO: $PATH\salt\Saltfile" +Write-Host "* INFO: Setting the SALT_SALTFILE environment variable: "-NoNewline $env:SALT_SALTFILE="$PATH\salt\Saltfile" +if ( Test-Path -Path $env:SALT_SALTFILE ) { + Write-Host "Success" -ForegroundColor Green +} else { + Write-Host "Failed" -ForegroundColor Red + exit 1 +} +Write-Host "* INFO: $PATH\salt\Saltfile" +Write-Host "" +Write-Host "You can now run simple salt-call commands" -ForegroundColor Cyan Write-Host "* INFO: Create Salt states in $PATH\salt\srv\salt" +Write-Host "* INFO: Try running salt-call test.ping" +Write-Host "" From 1b98100152462314aa4c75c64314f1dabb6eae9c Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 4 Dec 2024 09:51:13 -0700 Subject: [PATCH 152/173] Removed unused variable --- bootstrap-salt.sh | 41 ++++++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 49c71d3cd..06a8355a0 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -26,7 +26,7 @@ #====================================================================================================================== set -o nounset # Treat unset variables as an error -__ScriptVersion="2024.11.29" +__ScriptVersion="2024.12.03" __ScriptName="bootstrap-salt.sh" __ScriptFullName="$0" @@ -6634,36 +6634,39 @@ __ZYPPER_REQUIRES_REPLACE_FILES=-1 __set_suse_pkg_repo() { - # Set distro repo variable - if [ "${DISTRO_MAJOR_VERSION}" -gt 2015 ]; then - DISTRO_REPO="openSUSE_Tumbleweed" - elif [ "${DISTRO_MAJOR_VERSION}" -eq 15 ] && [ "${DISTRO_MINOR_VERSION}" -ge 4 ]; then - DISTRO_REPO="${DISTRO_MAJOR_VERSION}.${DISTRO_MINOR_VERSION}" - elif [ "${DISTRO_MAJOR_VERSION}" -ge 42 ] || [ "${DISTRO_MAJOR_VERSION}" -eq 15 ]; then - DISTRO_REPO="openSUSE_Leap_${DISTRO_MAJOR_VERSION}.${DISTRO_MINOR_VERSION}" - else - DISTRO_REPO="SLE_${DISTRO_MAJOR_VERSION}_SP${SUSE_PATCHLEVEL}" - fi + ## DGM # Set distro repo variable + ## DGM if [ "${DISTRO_MAJOR_VERSION}" -gt 2015 ]; then + ## DGM DISTRO_REPO="openSUSE_Tumbleweed" + ## DGM elif [ "${DISTRO_MAJOR_VERSION}" -eq 15 ] && [ "${DISTRO_MINOR_VERSION}" -ge 4 ]; then + ## DGM DISTRO_REPO="${DISTRO_MAJOR_VERSION}.${DISTRO_MINOR_VERSION}" + ## DGM elif [ "${DISTRO_MAJOR_VERSION}" -ge 42 ] || [ "${DISTRO_MAJOR_VERSION}" -eq 15 ]; then + ## DGM DISTRO_REPO="openSUSE_Leap_${DISTRO_MAJOR_VERSION}.${DISTRO_MINOR_VERSION}" + ## DGM else + ## DGM DISTRO_REPO="SLE_${DISTRO_MAJOR_VERSION}_SP${SUSE_PATCHLEVEL}" + ## DGM fi - suse_pkg_url_base="https://download.opensuse.org/repositories/systemsmanagement:/saltstack" - suse_pkg_url_path="${DISTRO_REPO}/systemsmanagement:saltstack.repo" - SUSE_PKG_URL="$suse_pkg_url_base/$suse_pkg_url_path" + ## DGM suse_pkg_url_base="https://download.opensuse.org/repositories/systemsmanagement:/saltstack" + ## DGM suse_pkg_url_path="${DISTRO_REPO}/systemsmanagement:saltstack.repo" + ## DGM SUSE_PKG_URL="$suse_pkg_url_base/$suse_pkg_url_path" + SUSE_PKG_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" } __check_and_refresh_suse_pkg_repo() { # Check to see if systemsmanagement_saltstack exists - __zypper repos | grep -q systemsmanagement_saltstack + ## DGM __zypper repos | grep -q systemsmanagement_saltstack + __zypper repos | grep -q 'salt.repo' if [ $? -eq 1 ]; then - # zypper does not yet know anything about systemsmanagement_saltstack + ## DGM # zypper does not yet know anything about systemsmanagement_saltstack + # zypper does not yet know anything about salt.repo __zypper addrepo --refresh "${SUSE_PKG_URL}" || return 1 fi } __version_lte() { - if ! __check_command_exists python; then - zypper --non-interactive install --replacefiles --auto-agree-with-licenses python || \ - zypper --non-interactive install --auto-agree-with-licenses python || return 1 + if ! __check_command_exists python3; then + zypper --non-interactive install --replacefiles --auto-agree-with-licenses python3 || \ + zypper --non-interactive install --auto-agree-with-licenses python3 || return 1 fi if [ "$(${_PY_EXE} -c 'import sys; V1=tuple([int(i) for i in sys.argv[1].split(".")]); V2=tuple([int(i) for i in sys.argv[2].split(".")]); print(V1<=V2)' "$1" "$2")" = "True" ]; then From 644dfd1c57d20defdb8ded1c546675c3358081c5 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Mon, 9 Dec 2024 13:59:19 -0700 Subject: [PATCH 153/173] Updates to correctly support SLES 15 (sled), with allowance for minor versions --- bootstrap-salt.sh | 227 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 183 insertions(+), 44 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 06a8355a0..0a6398d75 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -26,7 +26,7 @@ #====================================================================================================================== set -o nounset # Treat unset variables as an error -__ScriptVersion="2024.12.03" +__ScriptVersion="2024.12.09" __ScriptName="bootstrap-salt.sh" __ScriptFullName="$0" @@ -3053,8 +3053,9 @@ __install_saltstack_ubuntu_onedir_repository() { echo "Pin: version $ONEDIR_REV.*" >> /etc/apt/preferences.d/salt-pin-1001 echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + ONEDIR_REV_DOT=$(echo "$ONEDIR_REV" | sed 's/-/\./') echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - echo "Pin: version $ONEDIR_REV" >> /etc/apt/preferences.d/salt-pin-1001 + echo "Pin: version $ONEDIR_REV_DOT" >> /etc/apt/preferences.d/salt-pin-1001 echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 fi fi @@ -3506,8 +3507,10 @@ __install_saltstack_debian_repository() { echo "Pin: version $STABLE_REV.*" >> /etc/apt/preferences.d/salt-pin-1001 echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + STABLE_REV_DOT=$(echo "$STABLE_REV" | sed 's/-/\./') + MINOR_VER_STRG="-$STABLE_REV_DOT" echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - echo "Pin: version $STABLE_REV" >> /etc/apt/preferences.d/salt-pin-1001 + echo "Pin: version $STABLE_REV_DOT" >> /etc/apt/preferences.d/salt-pin-1001 echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 fi fi @@ -3549,8 +3552,9 @@ __install_saltstack_debian_onedir_repository() { echo "Pin: version $ONEDIR_REV.*" >> /etc/apt/preferences.d/salt-pin-1001 echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + ONEDIR_REV_DOT=$(echo "$ONEDIR_REV" | sed 's/-/\./') echo "Package: salt-*" > /etc/apt/preferences.d/salt-pin-1001 - echo "Pin: version $ONEDIR_REV" >> /etc/apt/preferences.d/salt-pin-1001 + echo "Pin: version $ONEDIR_REV_DOT" >> /etc/apt/preferences.d/salt-pin-1001 echo "Pin-Priority: 1001" >> /etc/apt/preferences.d/salt-pin-1001 fi fi @@ -3895,9 +3899,10 @@ __install_saltstack_fedora_onedir_repository() { fi elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then # using minor version - echo "[salt-repo-${ONEDIR_REV}-lts]" > "${YUM_REPO_FILE}" + ONEDIR_REV_DOT=$(echo "$ONEDIR_REV" | sed 's/-/\./') + echo "[salt-repo-${ONEDIR_REV_DOT}-lts]" > "${YUM_REPO_FILE}" # shellcheck disable=SC2129 - echo "name=Salt Repo for Salt v${ONEDIR_REV} LTS" >> "${YUM_REPO_FILE}" + echo "name=Salt Repo for Salt v${ONEDIR_REV_DOT} LTS" >> "${YUM_REPO_FILE}" echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" echo "priority=10" >> "${YUM_REPO_FILE}" @@ -4140,7 +4145,8 @@ install_fedora_onedir() { MINOR_VER_STRG="" elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then # Minor version Salt, need to add specific minor version - MINOR_VER_STRG="-$STABLE_REV" + STABLE_REV_DOT=$(echo "$STABLE_REV" | sed 's/-/\./') + MINOR_VER_STRG="-$STABLE_REV_DOT" else MINOR_VER_STRG="" fi @@ -4223,9 +4229,10 @@ __install_saltstack_rhel_onedir_repository() { fi elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then # using minor version - echo "[salt-repo-${ONEDIR_REV}-lts]" > "${YUM_REPO_FILE}" + ONEDIR_REV_DOT=$(echo "$ONEDIR_REV" | sed 's/-/\./') + echo "[salt-repo-${ONEDIR_REV_DOT}-lts]" > "${YUM_REPO_FILE}" # shellcheck disable=SC2129 - echo "name=Salt Repo for Salt v${ONEDIR_REV} LTS" >> "${YUM_REPO_FILE}" + echo "name=Salt Repo for Salt v${ONEDIR_REV_DOT} LTS" >> "${YUM_REPO_FILE}" echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" echo "priority=10" >> "${YUM_REPO_FILE}" @@ -4297,7 +4304,8 @@ install_centos_stable() { MINOR_VER_STRG="" elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then # Minor version Salt, need to add specific minor version - MINOR_VER_STRG="-$STABLE_REV" + STABLE_REV_DOT=$(echo "$STABLE_REV" | sed 's/-/\./') + MINOR_VER_STRG="-$STABLE_REV_DOT" else MINOR_VER_STRG="" fi @@ -4525,7 +4533,8 @@ install_centos_onedir() { MINOR_VER_STRG="" elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then # Minor version Salt, need to add specific minor version - MINOR_VER_STRG="-$ONEDIR_REV" + ONEDIR_REV_DOT=$(echo "$ONEDIR_REV" | sed 's/-/\./') + MINOR_VER_STRG="-$ONEDIR_REV_DOT" else MINOR_VER_STRG="" fi @@ -5657,9 +5666,10 @@ install_amazon_linux_ami_2_deps() { fi elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then # using minor version - echo "[salt-repo-${STABLE_REV}-lts]" > "${YUM_REPO_FILE}" + STABLE_REV_DOT=$(echo "$STABLE_REV" | sed 's/-/\./') + echo "[salt-repo-${STABLE_REV_DOT}-lts]" > "${YUM_REPO_FILE}" # shellcheck disable=SC2129 - echo "name=Salt Repo for Salt v${STABLE_REV} LTS" >> "${YUM_REPO_FILE}" + echo "name=Salt Repo for Salt v${STABLE_REV_DOT} LTS" >> "${YUM_REPO_FILE}" echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" echo "priority=10" >> "${YUM_REPO_FILE}" @@ -5721,9 +5731,10 @@ install_amazon_linux_ami_2_onedir_deps() { fi elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then # using minor version - echo "[salt-repo-${ONEDIR_REV}-lts]" > "${YUM_REPO_FILE}" + ONEDIR_REV_DOT=$(echo "$ONEDIR_REV" | sed 's/-/\./') + echo "[salt-repo-${ONEDIR_REV_DOT}-lts]" > "${YUM_REPO_FILE}" # shellcheck disable=SC2129 - echo "name=Salt Repo for Salt v${ONEDIR_REV} LTS" >> "${YUM_REPO_FILE}" + echo "name=Salt Repo for Salt v${ONEDIR_REV_DOT} LTS" >> "${YUM_REPO_FILE}" echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" echo "priority=10" >> "${YUM_REPO_FILE}" @@ -5873,9 +5884,10 @@ install_amazon_linux_ami_2023_onedir_deps() { fi elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then # using minor version - echo "[salt-repo-${ONEDIR_REV}-lts]" > "${YUM_REPO_FILE}" + ONEDIR_REV_DOT=$(echo "$ONEDIR_REV" | sed 's/-/\./') + echo "[salt-repo-${ONEDIR_REV_DOT}-lts]" > "${YUM_REPO_FILE}" # shellcheck disable=SC2129 - echo "name=Salt Repo for Salt v${ONEDIR_REV} LTS" >> "${YUM_REPO_FILE}" + echo "name=Salt Repo for Salt v${ONEDIR_REV_DOT} LTS" >> "${YUM_REPO_FILE}" echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" echo "priority=10" >> "${YUM_REPO_FILE}" @@ -6286,8 +6298,9 @@ __install_saltstack_photon_onedir_repository() { fi elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then # using minor version - echo "[salt-repo-${ONEDIR_REV}-lts]" > "${YUM_REPO_FILE}" - echo "name=Salt Repo for Salt v${ONEDIR_REV} LTS" >> "${YUM_REPO_FILE}" + ONEDIR_REV_DOT=$(echo "$ONEDIR_REV" | sed 's/-/\./') + echo "[salt-repo-${ONEDIR_REV_DOT}-lts]" > "${YUM_REPO_FILE}" + echo "name=Salt Repo for Salt v${ONEDIR_REV_DOT} LTS" >> "${YUM_REPO_FILE}" echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" echo "priority=10" >> "${YUM_REPO_FILE}" @@ -6582,7 +6595,8 @@ install_photon_onedir() { MINOR_VER_STRG="-$_GENERIC_PKG_VERSION" elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then # Minor version Salt, need to add specific minor version - MINOR_VER_STRG="-$STABLE_REV" + STABLE_REV_DOT=$(echo "$STABLE_REV" | sed 's/-/\./') + MINOR_VER_STRG="-$STABLE_REV_DOT" else # default to latest version Salt, config and repo already setup __get_packagesite_onedir_latest @@ -6632,34 +6646,78 @@ install_photon_onedir_post() { # __ZYPPER_REQUIRES_REPLACE_FILES=-1 -__set_suse_pkg_repo() { - - ## DGM # Set distro repo variable - ## DGM if [ "${DISTRO_MAJOR_VERSION}" -gt 2015 ]; then - ## DGM DISTRO_REPO="openSUSE_Tumbleweed" - ## DGM elif [ "${DISTRO_MAJOR_VERSION}" -eq 15 ] && [ "${DISTRO_MINOR_VERSION}" -ge 4 ]; then - ## DGM DISTRO_REPO="${DISTRO_MAJOR_VERSION}.${DISTRO_MINOR_VERSION}" - ## DGM elif [ "${DISTRO_MAJOR_VERSION}" -ge 42 ] || [ "${DISTRO_MAJOR_VERSION}" -eq 15 ]; then - ## DGM DISTRO_REPO="openSUSE_Leap_${DISTRO_MAJOR_VERSION}.${DISTRO_MINOR_VERSION}" - ## DGM else - ## DGM DISTRO_REPO="SLE_${DISTRO_MAJOR_VERSION}_SP${SUSE_PATCHLEVEL}" - ## DGM fi - - ## DGM suse_pkg_url_base="https://download.opensuse.org/repositories/systemsmanagement:/saltstack" - ## DGM suse_pkg_url_path="${DISTRO_REPO}/systemsmanagement:saltstack.repo" - ## DGM SUSE_PKG_URL="$suse_pkg_url_base/$suse_pkg_url_path" - SUSE_PKG_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" -} __check_and_refresh_suse_pkg_repo() { # Check to see if systemsmanagement_saltstack exists - ## DGM __zypper repos | grep -q systemsmanagement_saltstack __zypper repos | grep -q 'salt.repo' if [ $? -eq 1 ]; then - ## DGM # zypper does not yet know anything about systemsmanagement_saltstack # zypper does not yet know anything about salt.repo - __zypper addrepo --refresh "${SUSE_PKG_URL}" || return 1 + # zypper does not support exclude similar to Photon, hence have to do following + ZYPPER_REPO_FILE="/etc/zypp/repos.d/salt.repo" + # shellcheck disable=SC2129 + if [ "$ONEDIR_REV" != "latest" ]; then + # 3006.x is default, and latest for 3006.x branch + if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then + # latest version for branch 3006 | 3007 + REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) + if [ "$REPO_REV_MAJOR" -eq "3007" ]; then + # Enable the Salt 3007 STS repo + echo "[salt-repo-3007-sts]" > "${ZYPPER_REPO_FILE}" + echo "name=Salt Repo for Salt v3007 STS" >> "${ZYPPER_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${ZYPPER_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${ZYPPER_REPO_FILE}" + echo "priority=10" >> "${ZYPPER_REPO_FILE}" + echo "enabled=1" >> "${ZYPPER_REPO_FILE}" + echo "enabled_metadata=1" >> "${ZYPPER_REPO_FILE}" + echo "exclude=*3006* *3008* *3009* *3010*" >> "${ZYPPER_REPO_FILE}" + echo "gpgcheck=1" >> "${ZYPPER_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${ZYPPER_REPO_FILE}" + zypper addlock "salt-* < 3007" && zypper addlock "salt-* >= 3008" + else + # Salt 3006 repo + echo "[salt-repo-3006-lts]" > "${ZYPPER_REPO_FILE}" + echo "name=Salt Repo for Salt v3006 LTS" >> "${ZYPPER_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${ZYPPER_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${ZYPPER_REPO_FILE}" + echo "priority=10" >> "${ZYPPER_REPO_FILE}" + echo "enabled=1" >> "${ZYPPER_REPO_FILE}" + echo "enabled_metadata=1" >> "${ZYPPER_REPO_FILE}" + echo "exclude=*3007* *3008* *3009* *3010*" >> "${ZYPPER_REPO_FILE}" + echo "gpgcheck=1" >> "${ZYPPER_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${ZYPPER_REPO_FILE}" + zypper addlock "salt-* < 3006" && zypper addlock "salt-* >= 3007" + fi + elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then + # using minor version + ONEDIR_REV_DOT=$(echo "$ONEDIR_REV" | sed 's/-/\./') + echo "[salt-repo-${ONEDIR_REV_DOT}-lts]" > "${ZYPPER_REPO_FILE}" + echo "name=Salt Repo for Salt v${ONEDIR_REV_DOT} LTS" >> "${ZYPPER_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${ZYPPER_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${ZYPPER_REPO_FILE}" + echo "priority=10" >> "${ZYPPER_REPO_FILE}" + echo "enabled=1" >> "${ZYPPER_REPO_FILE}" + echo "enabled_metadata=1" >> "${ZYPPER_REPO_FILE}" + echo "gpgcheck=1" >> "${ZYPPER_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${ZYPPER_REPO_FILE}"a + ONEDIR_MAJ_VER=$(echo "${ONEDIR_REV_DOT}" | awk -F '.' '{print $1}') + # shellcheck disable=SC2004 + ONEDIR_MAJ_VER_PLUS=$((${ONEDIR_MAJ_VER} + 1)) + zypper addlock "salt-* < ${ONEDIR_MAJ_VER}" && zypper addlock "salt-* >= ${ONEDIR_MAJ_VER_PLUS}" + fi + else + # Enable the Salt LATEST repo + echo "[salt-repo-latest]" > "${ZYPPER_REPO_FILE}" + echo "name=Salt Repo for Salt LATEST release" >> "${ZYPPER_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${ZYPPER_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${ZYPPER_REPO_FILE}" + echo "priority=10" >> "${ZYPPER_REPO_FILE}" + echo "enabled=1" >> "${ZYPPER_REPO_FILE}" + echo "enabled_metadata=1" >> "${ZYPPER_REPO_FILE}" + echo "gpgcheck=1" >> "${ZYPPER_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${ZYPPER_REPO_FILE}" + fi + __zypper addrepo --refresh "${ZYPPER_REPO_FILE}" || return 1 fi } @@ -6710,8 +6768,6 @@ __zypper_install() { __opensuse_prep_install() { # DRY function for common installation preparatory steps for SUSE if [ "$_DISABLE_REPOS" -eq $BS_FALSE ]; then - # Is the repository already known - __set_suse_pkg_repo # Check zypper repos and refresh if necessary __check_and_refresh_suse_pkg_repo fi @@ -6798,7 +6854,8 @@ install_opensuse_stable() { MINOR_VER_STRG="" elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then # Minor version Salt, need to add specific minor version - MINOR_VER_STRG="-$STABLE_REV" + STABLE_REV_DOT=$(echo "$STABLE_REV" | sed 's/-/\./') + MINOR_VER_STRG="-$STABLE_REV_DOT" else MINOR_VER_STRG="" fi @@ -7113,12 +7170,94 @@ install_suse_15_restart_daemons() { return 0 } +install_suse_15_check_services() { + install_opensuse_check_services || return 1 + return 0 +} + # # End of SUSE Enterprise 15 # ####################################################################################################################### +####################################################################################################################### +# +# SUSE Enterprise 15, now has ID sled +# + +install_sled_15_stable_deps() { + __opensuse_prep_install || return 1 + install_opensuse_15_stable_deps || return 1 + + return 0 +} + +install_sled_15_git_deps() { + install_suse_15_stable_deps || return 1 + + if ! __check_command_exists git; then + __zypper_install git-core || return 1 + fi + + install_opensuse_15_git_deps || return 1 + + return 0 +} + +install_sled_15_onedir_deps() { + __opensuse_prep_install || return 1 + install_opensuse_15_onedir_deps || return 1 + + return 0 +} + +install_sled_15_stable() { + install_opensuse_stable || return 1 + return 0 +} + +install_sled_15_git() { + install_opensuse_15_git || return 1 + return 0 +} + +install_sled_15_onedir() { + install_opensuse_stable || return 1 + return 0 +} + +install_sled_15_stable_post() { + install_opensuse_stable_post || return 1 + return 0 +} + +install_sled_15_git_post() { + install_opensuse_git_post || return 1 + return 0 +} + +install_sled_15_onedir_post() { + install_opensuse_stable_post || return 1 + return 0 +} + +install_sled_15_restart_daemons() { + install_opensuse_restart_daemons || return 1 + return 0 +} + +install_sled_15_check_services() { + install_opensuse_check_services || return 1 + return 0 +} + +# +# End of SUSE Enterprise 15 aka sled +# +####################################################################################################################### + + ####################################################################################################################### # # Gentoo Install Functions. From 0bd577de501d72d1425233efddec18edbcd80904 Mon Sep 17 00:00:00 2001 From: Salt Project Packaging Date: Mon, 9 Dec 2024 23:10:33 +0000 Subject: [PATCH 154/173] Update develop branch for the v2024.12.09 release --- CHANGELOG.md | 9 +++++++++ bootstrap-salt.ps1 | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a2b85d2d6..16e7ac943 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +# v2024.12.09 + +## What's Changed + +- Fix quickstart for Windows with new repo by @twangboy in https://github.com/saltstack/salt-bootstrap/pull/2065 +- Utilize salt-project packages with SUSE by @dmurphy18 in https://github.com/saltstack/salt-bootstrap/pull/2066 + +**Full Changelog**: https://github.com/saltstack/salt-bootstrap/compare/v2024.11.29...v2024.12.09 + # v2024.11.29 ## What's Changed diff --git a/bootstrap-salt.ps1 b/bootstrap-salt.ps1 index 8d0c3c628..f355e0cd1 100644 --- a/bootstrap-salt.ps1 +++ b/bootstrap-salt.ps1 @@ -108,7 +108,7 @@ if ($help) { exit 0 } -$__ScriptVersion = "2024.11.29" +$__ScriptVersion = "2024.12.09" $ScriptName = $myInvocation.MyCommand.Name # We'll check for the Version next, because it also has no requirements From 54243fea9cdb20c214fd0f434c7b218e63df8768 Mon Sep 17 00:00:00 2001 From: Salt Project Packaging Date: Mon, 9 Dec 2024 23:11:16 +0000 Subject: [PATCH 155/173] Update README.rst with 2024.12.09 release sha256sum --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index cf80f3943..233b3f671 100644 --- a/README.rst +++ b/README.rst @@ -37,6 +37,7 @@ sum** of the downloaded ``bootstrap-salt.sh`` file. The SHA256 sum of the ``bootstrap-salt.sh`` file, per release, is: +- 2024.12.09: ``44f9405a6d9622ad8fa7c93e83a52e01ca328f27e4e9dea4a52268c6a22dbe6d`` - 2024.11.29: ``0ac87384dee051aceded69704485a5de0e4a308551a462b10c262111b57acff0`` - 2024.11.27: ``e972bd1ef01d09cd1d9294374ef974c9e3dd9a2aee37cf3859144585fd8bf1d0`` - 2024.11.26: ``832c7a20b96e1df171d715323df9afff8a11aef42d15598c007f240bc89d723c`` From e5c03a53aa2929e08e11df8107090b45eb5e8ecf Mon Sep 17 00:00:00 2001 From: David Murphy Date: Tue, 10 Dec 2024 11:26:38 -0700 Subject: [PATCH 156/173] Restrict use of dnf to Fedora only, otherwise use yum when dealing with RedHat family --- bootstrap-salt.sh | 52 +++++++++++++++++++++++------------------------ 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 0a6398d75..25731b9d3 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -4224,8 +4224,8 @@ __install_saltstack_rhel_onedir_repository() { REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) if [ "$REPO_REV_MAJOR" -eq "3007" ]; then # Enable the Salt 3007 STS repo - dnf config-manager --set-disable salt-repo-* - dnf config-manager --set-enabled salt-repo-3007-sts + yum config-manager --set-disable salt-repo-* + yum config-manager --set-enabled salt-repo-3007-sts fi elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then # using minor version @@ -4243,11 +4243,11 @@ __install_saltstack_rhel_onedir_repository() { fi else # Enable the Salt LATEST repo - dnf config-manager --set-disable salt-repo-* - dnf config-manager --set-enabled salt-repo-latest + yum config-manager --set-disable salt-repo-* + yum config-manager --set-enabled salt-repo-latest fi - dnf clean expire-cache || return 1 - dnf makecache || return 1 + yum clean expire-cache || return 1 + yum makecache || return 1 elif [ "$ONEDIR_REV" != "latest" ]; then echowarn "salt.repo already exists, ignoring salt version argument." echowarn "Use -F (forced overwrite) to install $ONEDIR_REV." @@ -4559,8 +4559,8 @@ install_centos_onedir() { fi # shellcheck disable=SC2086 - dnf makecache || return 1 - dnf list salt-minion || return 1 + yum makecache || return 1 + yum list salt-minion || return 1 __yum_install_noinput ${__PACKAGES} || return 1 return 0 @@ -5661,8 +5661,8 @@ install_amazon_linux_ami_2_deps() { REPO_REV_MAJOR=$(echo "$STABLE_REV" | cut -d '.' -f 1) if [ "$REPO_REV_MAJOR" -eq "3007" ]; then # Enable the Salt 3007 STS repo - dnf config-manager --set-disable salt-repo-* - dnf config-manager --set-enabled salt-repo-3007-sts + yum config-manager --set-disable salt-repo-* + yum config-manager --set-enabled salt-repo-3007-sts fi elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then # using minor version @@ -5680,11 +5680,11 @@ install_amazon_linux_ami_2_deps() { fi else # Enable the Salt LATEST repo - dnf config-manager --set-disable salt-repo-* - dnf config-manager --set-enabled salt-repo-latest + yum config-manager --set-disable salt-repo-* + yum config-manager --set-enabled salt-repo-latest fi - dnf clean expire-cache || return 1 - dnf makecache || return 1 + yum clean expire-cache || return 1 + yum makecache || return 1 fi fi @@ -5726,8 +5726,8 @@ install_amazon_linux_ami_2_onedir_deps() { REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) if [ "$REPO_REV_MAJOR" -eq "3007" ]; then # Enable the Salt 3007 STS repo - dnf config-manager --set-disable salt-repo-* - dnf config-manager --set-enabled salt-repo-3007-sts + yum config-manager --set-disable salt-repo-* + yum config-manager --set-enabled salt-repo-3007-sts fi elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then # using minor version @@ -5745,11 +5745,11 @@ install_amazon_linux_ami_2_onedir_deps() { fi else # Enable the Salt LATEST repo - dnf config-manager --set-disable salt-repo-* - dnf config-manager --set-enabled salt-repo-latest + yum config-manager --set-disable salt-repo-* + yum config-manager --set-enabled salt-repo-latest fi - dnf clean expire-cache || return 1 - dnf makecache || return 1 + yum clean expire-cache || return 1 + yum makecache || return 1 fi fi @@ -5879,8 +5879,8 @@ install_amazon_linux_ami_2023_onedir_deps() { REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) if [ "$REPO_REV_MAJOR" -eq "3007" ]; then # Enable the Salt 3007 STS repo - dnf config-manager --set-disable salt-repo-* - dnf config-manager --set-enabled salt-repo-3007-sts + yum config-manager --set-disable salt-repo-* + yum config-manager --set-enabled salt-repo-3007-sts fi elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then # using minor version @@ -5898,11 +5898,11 @@ install_amazon_linux_ami_2023_onedir_deps() { fi else # Enable the Salt LATEST repo - dnf config-manager --set-disable salt-repo-* - dnf config-manager --set-enabled salt-repo-latest + yum config-manager --set-disable salt-repo-* + yum config-manager --set-enabled salt-repo-latest fi - dnf clean expire-cache || return 1 - dnf makecache || return 1 + yum clean expire-cache || return 1 + yum makecache || return 1 fi fi From a4c84236d8e93e0cd48908aef1aa5a30e071181f Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 11 Dec 2024 11:08:54 -0700 Subject: [PATCH 157/173] Adjust Amazon support since their yum (v3) doesn't support config-manager --- bootstrap-salt.sh | 133 ++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 112 insertions(+), 21 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 25731b9d3..893205d1d 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -5652,8 +5652,10 @@ install_amazon_linux_ami_2_deps() { if [ $_DISABLE_REPOS -eq $BS_FALSE ] || [ "$_CUSTOM_REPO_URL" != "null" ]; then if [ ! -s "${YUM_REPO_FILE}" ]; then - FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" - __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" + ## Amazon Linux yum (v3) doesn't support config-manager + ## FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" + ## __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" + # shellcheck disable=SC2129 if [ "$STABLE_REV" != "latest" ]; then # 3006.x is default, and latest for 3006.x branch if [ "$(echo "$STABLE_REV" | grep -E '^(3006|3007)$')" != "" ]; then @@ -5661,14 +5663,33 @@ install_amazon_linux_ami_2_deps() { REPO_REV_MAJOR=$(echo "$STABLE_REV" | cut -d '.' -f 1) if [ "$REPO_REV_MAJOR" -eq "3007" ]; then # Enable the Salt 3007 STS repo - yum config-manager --set-disable salt-repo-* - yum config-manager --set-enabled salt-repo-3007-sts + echo "[salt-repo-3007-sts]" > "${YUM_REPO_FILE}" + echo "name=Salt Repo for Salt v3007 STS" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "exclude=*3006* *3008* *3009* *3010*" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" + else + # Salt 3006 repo + echo "[salt-repo-3006-lts]" > "${YUM_REPO_FILE}" + echo "name=Salt Repo for Salt v3006 LTS" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "exclude=*3007* *3008* *3009* *3010*" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" fi elif [ "$(echo "$STABLE_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then # using minor version STABLE_REV_DOT=$(echo "$STABLE_REV" | sed 's/-/\./') echo "[salt-repo-${STABLE_REV_DOT}-lts]" > "${YUM_REPO_FILE}" - # shellcheck disable=SC2129 echo "name=Salt Repo for Salt v${STABLE_REV_DOT} LTS" >> "${YUM_REPO_FILE}" echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" @@ -5680,8 +5701,15 @@ install_amazon_linux_ami_2_deps() { fi else # Enable the Salt LATEST repo - yum config-manager --set-disable salt-repo-* - yum config-manager --set-enabled salt-repo-latest + echo "[salt-repo-latest]" > "${YUM_REPO_FILE}" + echo "name=Salt Repo for Salt LATEST release" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" fi yum clean expire-cache || return 1 yum makecache || return 1 @@ -5717,8 +5745,10 @@ install_amazon_linux_ami_2_onedir_deps() { if [ $_DISABLE_REPOS -eq $BS_FALSE ] || [ "$_CUSTOM_REPO_URL" != "null" ]; then if [ ! -s "${YUM_REPO_FILE}" ]; then - FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" - __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" + ## Amazon Linux yum (v3) doesn't support config-manager + ## FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" + ## __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" + # shellcheck disable=SC2129 if [ "$ONEDIR_REV" != "latest" ]; then # 3006.x is default, and latest for 3006.x branch if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then @@ -5726,14 +5756,33 @@ install_amazon_linux_ami_2_onedir_deps() { REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) if [ "$REPO_REV_MAJOR" -eq "3007" ]; then # Enable the Salt 3007 STS repo - yum config-manager --set-disable salt-repo-* - yum config-manager --set-enabled salt-repo-3007-sts + echo "[salt-repo-3007-sts]" > "${YUM_REPO_FILE}" + echo "name=Salt Repo for Salt v3007 STS" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "exclude=*3006* *3008* *3009* *3010*" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" + else + # Salt 3006 repo + echo "[salt-repo-3006-lts]" > "${YUM_REPO_FILE}" + echo "name=Salt Repo for Salt v3006 LTS" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "exclude=*3007* *3008* *3009* *3010*" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" fi elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then # using minor version ONEDIR_REV_DOT=$(echo "$ONEDIR_REV" | sed 's/-/\./') echo "[salt-repo-${ONEDIR_REV_DOT}-lts]" > "${YUM_REPO_FILE}" - # shellcheck disable=SC2129 echo "name=Salt Repo for Salt v${ONEDIR_REV_DOT} LTS" >> "${YUM_REPO_FILE}" echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" @@ -5745,8 +5794,15 @@ install_amazon_linux_ami_2_onedir_deps() { fi else # Enable the Salt LATEST repo - yum config-manager --set-disable salt-repo-* - yum config-manager --set-enabled salt-repo-latest + echo "[salt-repo-latest]" > "${YUM_REPO_FILE}" + echo "name=Salt Repo for Salt LATEST release" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" fi yum clean expire-cache || return 1 yum makecache || return 1 @@ -5852,6 +5908,13 @@ install_amazon_linux_ami_2023_git_deps() { return 0 } +install_amazon_linux_ami_2023_deps() { + + # Set ONEDIR_REV to STABLE_REV + ONEDIR_REV=${STABLE_REV} + install_amazon_linux_ami_2023_onedir_deps || return 1 +} + install_amazon_linux_ami_2023_onedir_deps() { # We need to install yum-utils before doing anything else when installing on @@ -5870,8 +5933,10 @@ install_amazon_linux_ami_2023_onedir_deps() { if [ $_DISABLE_REPOS -eq $BS_FALSE ] || [ "$_CUSTOM_REPO_URL" != "null" ]; then if [ ! -s "${YUM_REPO_FILE}" ]; then - FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" - __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" + ## Amazon Linux yum (v3) doesn't support config-manager + ## FETCH_URL="https://github.com/saltstack/salt-install-guide/releases/latest/download/salt.repo" + ## __fetch_url "${YUM_REPO_FILE}" "${FETCH_URL}" + # shellcheck disable=SC2129 if [ "$ONEDIR_REV" != "latest" ]; then # 3006.x is default, and latest for 3006.x branch if [ "$(echo "$ONEDIR_REV" | grep -E '^(3006|3007)$')" != "" ]; then @@ -5879,14 +5944,33 @@ install_amazon_linux_ami_2023_onedir_deps() { REPO_REV_MAJOR=$(echo "$ONEDIR_REV" | cut -d '.' -f 1) if [ "$REPO_REV_MAJOR" -eq "3007" ]; then # Enable the Salt 3007 STS repo - yum config-manager --set-disable salt-repo-* - yum config-manager --set-enabled salt-repo-3007-sts + echo "[salt-repo-3007-sts]" > "${YUM_REPO_FILE}" + echo "name=Salt Repo for Salt v3007 STS" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "exclude=*3006* *3008* *3009* *3010*" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" + else + # Salt 3006 repo + echo "[salt-repo-3006-lts]" > "${YUM_REPO_FILE}" + echo "name=Salt Repo for Salt v3006 LTS" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "exclude=*3007* *3008* *3009* *3010*" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" fi elif [ "$(echo "$ONEDIR_REV" | grep -E '^([3-9][0-5]{2}[6-9](\.[0-9]*)?)')" != "" ]; then # using minor version ONEDIR_REV_DOT=$(echo "$ONEDIR_REV" | sed 's/-/\./') echo "[salt-repo-${ONEDIR_REV_DOT}-lts]" > "${YUM_REPO_FILE}" - # shellcheck disable=SC2129 echo "name=Salt Repo for Salt v${ONEDIR_REV_DOT} LTS" >> "${YUM_REPO_FILE}" echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" @@ -5898,8 +5982,15 @@ install_amazon_linux_ami_2023_onedir_deps() { fi else # Enable the Salt LATEST repo - yum config-manager --set-disable salt-repo-* - yum config-manager --set-enabled salt-repo-latest + echo "[salt-repo-latest]" > "${YUM_REPO_FILE}" + echo "name=Salt Repo for Salt LATEST release" >> "${YUM_REPO_FILE}" + echo "baseurl=https://${_REPO_URL}/saltproject-rpm/" >> "${YUM_REPO_FILE}" + echo "skip_if_unavailable=True" >> "${YUM_REPO_FILE}" + echo "priority=10" >> "${YUM_REPO_FILE}" + echo "enabled=1" >> "${YUM_REPO_FILE}" + echo "enabled_metadata=1" >> "${YUM_REPO_FILE}" + echo "gpgcheck=1" >> "${YUM_REPO_FILE}" + echo "gpgkey=https://${_REPO_URL}/api/security/keypair/SaltProjectKey/public" >> "${YUM_REPO_FILE}" fi yum clean expire-cache || return 1 yum makecache || return 1 From ac14d1d0fedd800b475fffde710b8f96055ee1ba Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 11 Dec 2024 11:31:59 -0700 Subject: [PATCH 158/173] Updating container usage to reflect that used in new Salt pipelines for CI/CD --- .github/workflows/ci.yml | 10 ++-- .github/workflows/templates/generate.py | 39 ++++++++++----- .github/workflows/test-linux.yml | 63 +++++++++++++++++++++---- 3 files changed, 86 insertions(+), 26 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e88f30f52..cfc4d8444 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -190,7 +190,7 @@ jobs: with: distro-slug: debian-12 display-name: Debian 12 - container-slug: systemd-debian-12 + container-slug: debian-12 timeout: 20 instances: '["stable-3007", "git-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' @@ -205,7 +205,7 @@ jobs: with: distro-slug: photon-5 display-name: Photon OS 5 - container-slug: systemd-photon-5 + container-slug: photon-5 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' @@ -220,7 +220,7 @@ jobs: with: distro-slug: rockylinux-8 display-name: Rocky Linux 8 - container-slug: systemd-rockylinux-8 + container-slug: rockylinux-8 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' @@ -235,7 +235,7 @@ jobs: with: distro-slug: rockylinux-9 display-name: Rocky Linux 9 - container-slug: systemd-rockylinux-9 + container-slug: rockylinux-9 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' @@ -250,7 +250,7 @@ jobs: with: distro-slug: ubuntu-2204 display-name: Ubuntu 22.04 - container-slug: systemd-ubuntu-22.04 + container-slug: ubuntu-22.04 timeout: 20 instances: '["stable-3006", "git-3006", "onedir-3006", "stable-3006-8", "stable-3007", "git-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index ca31b4cfa..e0ffa3567 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -216,20 +216,33 @@ "windows-2022": "Windows 2022", } +## "amazonlinux-2": "systemd-amazonlinux-2", +## "amazonlinux-2023": "systemd-amazonlinux-2023", +## "debian-11": "systemd-debian-11", +## "debian-12": "systemd-debian-12", +## "debian-13": "systemd-debian-13", +## "fedora-40": "systemd-fedora-40", +## "photon-4": "systemd-photon-4", +## "photon-5": "systemd-photon-5", +## "rockylinux-8": "systemd-rockylinux-8", +## "rockylinux-9": "systemd-rockylinux-9", +## "ubuntu-2004": "systemd-ubuntu-20.04", +## "ubuntu-2204": "systemd-ubuntu-22.04", +## "ubuntu-2404": "systemd-ubuntu-24.04", CONTAINER_SLUG_NAMES = { - "amazonlinux-2": "systemd-amazonlinux-2", - "amazonlinux-2023": "systemd-amazonlinux-2023", - "debian-11": "systemd-debian-11", - "debian-12": "systemd-debian-12", - "debian-13": "systemd-debian-13", - "fedora-40": "systemd-fedora-40", - "photon-4": "systemd-photon-4", - "photon-5": "systemd-photon-5", - "rockylinux-8": "systemd-rockylinux-8", - "rockylinux-9": "systemd-rockylinux-9", - "ubuntu-2004": "systemd-ubuntu-20.04", - "ubuntu-2204": "systemd-ubuntu-22.04", - "ubuntu-2404": "systemd-ubuntu-24.04", + "amazonlinux-2": "amazonlinux-2", + "amazonlinux-2023": "amazonlinux-2023", + "debian-11": "debian-11", + "debian-12": "debian-12", + "debian-13": "debian-13", + "fedora-40": "fedora-40", + "photon-4": "photon-4", + "photon-5": "photon-5", + "rockylinux-8": "rockylinux-8", + "rockylinux-9": "rockylinux-9", + "ubuntu-2004": "ubuntu-20.04", + "ubuntu-2204": "ubuntu-22.04", + "ubuntu-2404": "ubuntu-24.04", "macos-12": "macos-12", "macos-13": "macos-13", "macos-14": "macOS 14", diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 120f23eca..0c44d68c6 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -32,9 +32,9 @@ jobs: Test: name: ${{ matrix.instance }} runs-on: ubuntu-latest - container: - image: ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} - options: --privileged + ## DGM container: + ## DGM image: ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} + ## DGM options: --privileged timeout-minutes: ${{ inputs.timeout }} strategy: @@ -46,15 +46,58 @@ jobs: steps: - uses: actions/checkout@v4 - - name: Install Python Dependencies with pip breakage + ## DGM - name: Install Python Dependencies with pip breakage + ## DGM if: ${{ ( inputs.distro-slug == 'debian-11' ) || ( inputs.distro-slug == 'debian-12' ) || ( inputs.distro-slug == 'debian-13' ) || ( inputs.distro-slug == 'ubuntu-2404' ) }} + ## DGM run: | + ## DGM python3 -m pip install --break-system-packages -r tests/requirements.txt + + ## DGM - name: Install Python Dependencies without pip breakage + ## DGM if: ${{ ( inputs.distro-slug != 'debian-11' ) && ( inputs.distro-slug != 'debian-12' ) && ( inputs.distro-slug != 'debian-13' ) && ( inputs.distro-slug != 'ubuntu-2404' ) }} + ## DGM run: | + ## DGM python3 -m pip install -r tests/requirements.txt + + ## DGM - name: Get Version + ## DGM run: | + ## DGM # We need to get the version here and make it an environment variable + ## DGM # It is used to install via bootstrap and in the test + ## DGM # The version is in the instance name + ## DGM # sed 1st - becomes space, 2nd - becomes dot + ## DGM vt_parm_ver=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./' | awk -F ' ' '{print $2}') + ## DGM echo "SaltVersion=$vt_parm_ver" >> $GITHUB_ENV + + ## DGM - name: Bootstrap Salt + ## DGM run: | + ## DGM # sed 1st - becomes space, 2nd - becomes dot + ## DGM bt_arg1=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./' | awk -F ' ' '{print $1}') + ## DGM bt_arg2=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./' | awk -F ' ' '{print $2}') + ## DGM echo "bt parms ,$bt_parms, bt_arg1 ,$bt_arg1, bt_arg2 ,$bt_arg2," + ## DGM sh -x ./bootstrap-salt.sh "$bt_arg1" "$bt_arg2" + + ## DGM - name: Test Bootstrap + ## DGM run: | + ## DGM pytest --cache-clear -v -s -ra --log-cli-level=debug tests/integration/ + + - name: "Pull container ${{ inputs.container-slug }}" + run: | + docker pull ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} + + - name: "Create container ${{ inputs.container-slug }}" + run: | + /usr/bin/docker create --name ${{ github.run_id }}_salt-test --workdir /__w/salt/salt --privileged -e "HOME=/github/home" -e GITHUB_ACTIONS=true -e CI=true -v "/var/run/docker.sock":"/var/run/docker.sock" -v "/home/runner/work":"/__w" -v "/home/runner/work/_temp":"/__w/_temp" -v "/home/runner/work/_actions":"/__w/_actions" -v "/opt/hostedtoolcache":"/__t" -v "/home/runner/work/_temp/_github_home":"/github/home" -v "/home/runner/work/_temp/_github_workflow":"/github/workflow" --entrypoint "/usr/lib/systemd/systemd" ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} --systemd --unit rescue.target + + - name: "Start container ${{ inputs.container-slug }}" + run: | + /usr/bin/docker start ${{ github.run_id }}_salt-test + + - name: "Install Python Dependencies with pip breakage in container ${{ inputs.container-slug }}" if: ${{ ( inputs.distro-slug == 'debian-11' ) || ( inputs.distro-slug == 'debian-12' ) || ( inputs.distro-slug == 'debian-13' ) || ( inputs.distro-slug == 'ubuntu-2404' ) }} run: | python3 -m pip install --break-system-packages -r tests/requirements.txt - - name: Install Python Dependencies without pip breakage + - name: "Install Python Dependencies without pip breakage in container ${{ inputs.container-slug }}" if: ${{ ( inputs.distro-slug != 'debian-11' ) && ( inputs.distro-slug != 'debian-12' ) && ( inputs.distro-slug != 'debian-13' ) && ( inputs.distro-slug != 'ubuntu-2404' ) }} run: | - python3 -m pip install -r tests/requirements.txt + docker exec ${{ github.run_id}}_salt-test python3 -m pip install -r tests/requirements.txt - name: Get Version run: | @@ -65,17 +108,21 @@ jobs: vt_parm_ver=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./' | awk -F ' ' '{print $2}') echo "SaltVersion=$vt_parm_ver" >> $GITHUB_ENV + - name: Current Directory + run: | + pwd + - name: Bootstrap Salt run: | # sed 1st - becomes space, 2nd - becomes dot bt_arg1=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./' | awk -F ' ' '{print $1}') bt_arg2=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./' | awk -F ' ' '{print $2}') echo "bt parms ,$bt_parms, bt_arg1 ,$bt_arg1, bt_arg2 ,$bt_arg2," - sh -x ./bootstrap-salt.sh "$bt_arg1" "$bt_arg2" + docker exec ${{ github.run_id}}_salt-test sh -x ./bootstrap-salt.sh "$bt_arg1" "$bt_arg2" - name: Test Bootstrap run: | - pytest --cache-clear -v -s -ra --log-cli-level=debug tests/integration/ + docker exec ${{ github.run_id}}_salt-test pytest --cache-clear -v -s -ra --log-cli-level=debug tests/integration/ - name: Set Exit Status if: always() From 852a98fce4ebce668365d491537aff3702e4abb8 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 11 Dec 2024 11:51:45 -0700 Subject: [PATCH 159/173] Updating github actions used --- .github/workflows/test-linux.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 0c44d68c6..541707e47 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -89,10 +89,15 @@ jobs: run: | /usr/bin/docker start ${{ github.run_id }}_salt-test + - name: Download requirements file tests/requirements.txt + uses: actions/download-artifact@v4 + with: + - name: tests/requirements.txt + - name: "Install Python Dependencies with pip breakage in container ${{ inputs.container-slug }}" if: ${{ ( inputs.distro-slug == 'debian-11' ) || ( inputs.distro-slug == 'debian-12' ) || ( inputs.distro-slug == 'debian-13' ) || ( inputs.distro-slug == 'ubuntu-2404' ) }} run: | - python3 -m pip install --break-system-packages -r tests/requirements.txt + docker exec ${{ github.run_id}}_salt-test python3 -m pip install --break-system-packages -r tests/requirements.txt - name: "Install Python Dependencies without pip breakage in container ${{ inputs.container-slug }}" if: ${{ ( inputs.distro-slug != 'debian-11' ) && ( inputs.distro-slug != 'debian-12' ) && ( inputs.distro-slug != 'debian-13' ) && ( inputs.distro-slug != 'ubuntu-2404' ) }} From a0285f3e6e008018d2b01173a927cef08e38add0 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 11 Dec 2024 12:57:24 -0700 Subject: [PATCH 160/173] Fixing actions --- .github/workflows/test-linux.yml | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 541707e47..886821301 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -89,20 +89,27 @@ jobs: run: | /usr/bin/docker start ${{ github.run_id }}_salt-test - - name: Download requirements file tests/requirements.txt + - name: Current Directory + run: | + pwd + ls -alh + + - name: Download requirements file tests/requirements.txt uses: actions/download-artifact@v4 with: - - name: tests/requirements.txt + - name: requirements.txt - name: "Install Python Dependencies with pip breakage in container ${{ inputs.container-slug }}" if: ${{ ( inputs.distro-slug == 'debian-11' ) || ( inputs.distro-slug == 'debian-12' ) || ( inputs.distro-slug == 'debian-13' ) || ( inputs.distro-slug == 'ubuntu-2404' ) }} run: | - docker exec ${{ github.run_id}}_salt-test python3 -m pip install --break-system-packages -r tests/requirements.txt + ## docker exec ${{ github.run_id}}_salt-test python3 -m pip install --break-system-packages -r tests/requirements.txt + docker exec ${{ github.run_id}}_salt-test python3 -m pip install --break-system-packages -r requirements.txt - name: "Install Python Dependencies without pip breakage in container ${{ inputs.container-slug }}" if: ${{ ( inputs.distro-slug != 'debian-11' ) && ( inputs.distro-slug != 'debian-12' ) && ( inputs.distro-slug != 'debian-13' ) && ( inputs.distro-slug != 'ubuntu-2404' ) }} run: | - docker exec ${{ github.run_id}}_salt-test python3 -m pip install -r tests/requirements.txt + ## docker exec ${{ github.run_id}}_salt-test python3 -m pip install -r tests/requirements.txt + docker exec ${{ github.run_id}}_salt-test python3 -m pip install -r requirements.txt - name: Get Version run: | @@ -113,10 +120,6 @@ jobs: vt_parm_ver=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./' | awk -F ' ' '{print $2}') echo "SaltVersion=$vt_parm_ver" >> $GITHUB_ENV - - name: Current Directory - run: | - pwd - - name: Bootstrap Salt run: | # sed 1st - becomes space, 2nd - becomes dot From 9d69e8a49d45d6d357e17e0ee39539d0dfbf45df Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 11 Dec 2024 13:19:58 -0700 Subject: [PATCH 161/173] Working on actions --- .github/workflows/test-linux.yml | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 886821301..65be11f97 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -94,22 +94,20 @@ jobs: pwd ls -alh - - name: Download requirements file tests/requirements.txt - uses: actions/download-artifact@v4 - with: - - name: requirements.txt + ## DGM - name: Download requirements file tests/requirements.txt + ## DGM uses: actions/download-artifact@v4 + ## DGM with: + ## DGM - name: requirements.txt - name: "Install Python Dependencies with pip breakage in container ${{ inputs.container-slug }}" if: ${{ ( inputs.distro-slug == 'debian-11' ) || ( inputs.distro-slug == 'debian-12' ) || ( inputs.distro-slug == 'debian-13' ) || ( inputs.distro-slug == 'ubuntu-2404' ) }} run: | - ## docker exec ${{ github.run_id}}_salt-test python3 -m pip install --break-system-packages -r tests/requirements.txt - docker exec ${{ github.run_id}}_salt-test python3 -m pip install --break-system-packages -r requirements.txt + docker exec ${{ github.run_id}}_salt-test python3 -m pip install --break-system-packages -r tests/requirements.txt - name: "Install Python Dependencies without pip breakage in container ${{ inputs.container-slug }}" if: ${{ ( inputs.distro-slug != 'debian-11' ) && ( inputs.distro-slug != 'debian-12' ) && ( inputs.distro-slug != 'debian-13' ) && ( inputs.distro-slug != 'ubuntu-2404' ) }} run: | - ## docker exec ${{ github.run_id}}_salt-test python3 -m pip install -r tests/requirements.txt - docker exec ${{ github.run_id}}_salt-test python3 -m pip install -r requirements.txt + docker exec ${{ github.run_id}}_salt-test python3 -m pip install -r tests/requirements.txt - name: Get Version run: | From b6683d5e5e3d01a5202b1ed83af60eeb5dea802f Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 11 Dec 2024 14:04:57 -0700 Subject: [PATCH 162/173] More actions work --- .github/workflows/test-linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 65be11f97..576c3e8da 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -83,7 +83,7 @@ jobs: - name: "Create container ${{ inputs.container-slug }}" run: | - /usr/bin/docker create --name ${{ github.run_id }}_salt-test --workdir /__w/salt/salt --privileged -e "HOME=/github/home" -e GITHUB_ACTIONS=true -e CI=true -v "/var/run/docker.sock":"/var/run/docker.sock" -v "/home/runner/work":"/__w" -v "/home/runner/work/_temp":"/__w/_temp" -v "/home/runner/work/_actions":"/__w/_actions" -v "/opt/hostedtoolcache":"/__t" -v "/home/runner/work/_temp/_github_home":"/github/home" -v "/home/runner/work/_temp/_github_workflow":"/github/workflow" --entrypoint "/usr/lib/systemd/systemd" ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} --systemd --unit rescue.target + /usr/bin/docker create --name ${{ github.run_id }}_salt-test --workdir /home/runner/work/salt-bootstrap/salt-bootstrap --privileged -e "HOME=/github/home" -e GITHUB_ACTIONS=true -e CI=true -v "/var/run/docker.sock":"/var/run/docker.sock" -v "/home/runner/work":"/__w" -v "/home/runner/work/_temp":"/__w/_temp" -v "/home/runner/work/_actions":"/__w/_actions" -v "/opt/hostedtoolcache":"/__t" -v "/home/runner/work/_temp/_github_home":"/github/home" -v "/home/runner/work/_temp/_github_workflow":"/github/workflow" --entrypoint "/usr/lib/systemd/systemd" ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} --systemd --unit rescue.target - name: "Start container ${{ inputs.container-slug }}" run: | From a3971ee9938d46f0eda8e918eec38eb8055797e9 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 11 Dec 2024 14:12:47 -0700 Subject: [PATCH 163/173] More actions usage --- .github/workflows/test-linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 576c3e8da..0d3bbd09d 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -83,7 +83,7 @@ jobs: - name: "Create container ${{ inputs.container-slug }}" run: | - /usr/bin/docker create --name ${{ github.run_id }}_salt-test --workdir /home/runner/work/salt-bootstrap/salt-bootstrap --privileged -e "HOME=/github/home" -e GITHUB_ACTIONS=true -e CI=true -v "/var/run/docker.sock":"/var/run/docker.sock" -v "/home/runner/work":"/__w" -v "/home/runner/work/_temp":"/__w/_temp" -v "/home/runner/work/_actions":"/__w/_actions" -v "/opt/hostedtoolcache":"/__t" -v "/home/runner/work/_temp/_github_home":"/github/home" -v "/home/runner/work/_temp/_github_workflow":"/github/workflow" --entrypoint "/usr/lib/systemd/systemd" ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} --systemd --unit rescue.target + /usr/bin/docker create --name ${{ github.run_id }}_salt-test --workdir /_w/ --privileged -e "HOME=/github/home" -e GITHUB_ACTIONS=true -e CI=true -v "/var/run/docker.sock":"/var/run/docker.sock" -v "/home/runner/work":"/__w" -v "/home/runner/work/_temp":"/__w/_temp" -v "/home/runner/work/_actions":"/__w/_actions" -v "/opt/hostedtoolcache":"/__t" -v "/home/runner/work/_temp/_github_home":"/github/home" -v "/home/runner/work/_temp/_github_workflow":"/github/workflow" -v "/home/runner/work/salt-bootstrap/salt-bootstrap":"/_w/btstrap" --entrypoint "/usr/lib/systemd/systemd" ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} --systemd --unit rescue.target - name: "Start container ${{ inputs.container-slug }}" run: | From d4cc7c11847ca554e2ad7974513bbbdd1d04a7ef Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 11 Dec 2024 14:19:50 -0700 Subject: [PATCH 164/173] Updating actions --- .github/workflows/test-linux.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 0d3bbd09d..6caeb090e 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -77,13 +77,22 @@ jobs: ## DGM run: | ## DGM pytest --cache-clear -v -s -ra --log-cli-level=debug tests/integration/ + - name: Get Version + run: | + # We need to get the version here and make it an environment variable + # It is used to install via bootstrap and in the test + # The version is in the instance name + # sed 1st - becomes space, 2nd - becomes dot + vt_parm_ver=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./' | awk -F ' ' '{print $2}') + echo "SaltVersion=$vt_parm_ver" >> $GITHUB_ENV + - name: "Pull container ${{ inputs.container-slug }}" run: | docker pull ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} - name: "Create container ${{ inputs.container-slug }}" run: | - /usr/bin/docker create --name ${{ github.run_id }}_salt-test --workdir /_w/ --privileged -e "HOME=/github/home" -e GITHUB_ACTIONS=true -e CI=true -v "/var/run/docker.sock":"/var/run/docker.sock" -v "/home/runner/work":"/__w" -v "/home/runner/work/_temp":"/__w/_temp" -v "/home/runner/work/_actions":"/__w/_actions" -v "/opt/hostedtoolcache":"/__t" -v "/home/runner/work/_temp/_github_home":"/github/home" -v "/home/runner/work/_temp/_github_workflow":"/github/workflow" -v "/home/runner/work/salt-bootstrap/salt-bootstrap":"/_w/btstrap" --entrypoint "/usr/lib/systemd/systemd" ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} --systemd --unit rescue.target + /usr/bin/docker create --name ${{ github.run_id }}_salt-test --workdir /_w/ --privileged -e "HOME=/github/home" -e GITHUB_ACTIONS=true -e CI=true -e ${{ GITHUB_ENV }} -v "/var/run/docker.sock":"/var/run/docker.sock" -v "/home/runner/work":"/__w" -v "/home/runner/work/_temp":"/__w/_temp" -v "/home/runner/work/_actions":"/__w/_actions" -v "/opt/hostedtoolcache":"/__t" -v "/home/runner/work/_temp/_github_home":"/github/home" -v "/home/runner/work/_temp/_github_workflow":"/github/workflow" -v "/home/runner/work/salt-bootstrap/salt-bootstrap":"/_w/btstrap" --entrypoint "/usr/lib/systemd/systemd" ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} --systemd --unit rescue.target - name: "Start container ${{ inputs.container-slug }}" run: | @@ -102,21 +111,12 @@ jobs: - name: "Install Python Dependencies with pip breakage in container ${{ inputs.container-slug }}" if: ${{ ( inputs.distro-slug == 'debian-11' ) || ( inputs.distro-slug == 'debian-12' ) || ( inputs.distro-slug == 'debian-13' ) || ( inputs.distro-slug == 'ubuntu-2404' ) }} run: | - docker exec ${{ github.run_id}}_salt-test python3 -m pip install --break-system-packages -r tests/requirements.txt + docker exec ${{ github.run_id}}_salt-test python3 -m pip install --break-system-packages -r /_w/btstrap/tests/requirements.txt - name: "Install Python Dependencies without pip breakage in container ${{ inputs.container-slug }}" if: ${{ ( inputs.distro-slug != 'debian-11' ) && ( inputs.distro-slug != 'debian-12' ) && ( inputs.distro-slug != 'debian-13' ) && ( inputs.distro-slug != 'ubuntu-2404' ) }} run: | - docker exec ${{ github.run_id}}_salt-test python3 -m pip install -r tests/requirements.txt - - - name: Get Version - run: | - # We need to get the version here and make it an environment variable - # It is used to install via bootstrap and in the test - # The version is in the instance name - # sed 1st - becomes space, 2nd - becomes dot - vt_parm_ver=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./' | awk -F ' ' '{print $2}') - echo "SaltVersion=$vt_parm_ver" >> $GITHUB_ENV + docker exec ${{ github.run_id}}_salt-test python3 -m pip install -r /_w/btstrap/tests/requirements.txt - name: Bootstrap Salt run: | @@ -124,11 +124,11 @@ jobs: bt_arg1=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./' | awk -F ' ' '{print $1}') bt_arg2=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./' | awk -F ' ' '{print $2}') echo "bt parms ,$bt_parms, bt_arg1 ,$bt_arg1, bt_arg2 ,$bt_arg2," - docker exec ${{ github.run_id}}_salt-test sh -x ./bootstrap-salt.sh "$bt_arg1" "$bt_arg2" + docker exec ${{ github.run_id}}_salt-test sh -x /_w/btstrap/bootstrap-salt.sh "$bt_arg1" "$bt_arg2" - name: Test Bootstrap run: | - docker exec ${{ github.run_id}}_salt-test pytest --cache-clear -v -s -ra --log-cli-level=debug tests/integration/ + docker exec ${{ github.run_id}}_salt-test pytest --cache-clear -v -s -ra --log-cli-level=debug /_w/btstrap/tests/integration/ - name: Set Exit Status if: always() From dc01eea317404aaee2137993fdb50a25a01f1bd7 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 11 Dec 2024 14:27:59 -0700 Subject: [PATCH 165/173] More actions work --- .github/workflows/test-linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 6caeb090e..5ed8719fe 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -92,7 +92,7 @@ jobs: - name: "Create container ${{ inputs.container-slug }}" run: | - /usr/bin/docker create --name ${{ github.run_id }}_salt-test --workdir /_w/ --privileged -e "HOME=/github/home" -e GITHUB_ACTIONS=true -e CI=true -e ${{ GITHUB_ENV }} -v "/var/run/docker.sock":"/var/run/docker.sock" -v "/home/runner/work":"/__w" -v "/home/runner/work/_temp":"/__w/_temp" -v "/home/runner/work/_actions":"/__w/_actions" -v "/opt/hostedtoolcache":"/__t" -v "/home/runner/work/_temp/_github_home":"/github/home" -v "/home/runner/work/_temp/_github_workflow":"/github/workflow" -v "/home/runner/work/salt-bootstrap/salt-bootstrap":"/_w/btstrap" --entrypoint "/usr/lib/systemd/systemd" ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} --systemd --unit rescue.target + /usr/bin/docker create --name ${{ github.run_id }}_salt-test --workdir /_w/ --privileged -e "HOME=/github/home" -e GITHUB_ACTIONS=true -e CI=true -e $GITHUB_ENV -v "/var/run/docker.sock":"/var/run/docker.sock" -v "/home/runner/work":"/__w" -v "/home/runner/work/_temp":"/__w/_temp" -v "/home/runner/work/_actions":"/__w/_actions" -v "/opt/hostedtoolcache":"/__t" -v "/home/runner/work/_temp/_github_home":"/github/home" -v "/home/runner/work/_temp/_github_workflow":"/github/workflow" -v "/home/runner/work/salt-bootstrap/salt-bootstrap":"/_w/btstrap" --entrypoint "/usr/lib/systemd/systemd" ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} --systemd --unit rescue.target - name: "Start container ${{ inputs.container-slug }}" run: | From f47986ffd5576f8f243c933667c175d2aaec3d23 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 11 Dec 2024 15:24:10 -0700 Subject: [PATCH 166/173] Update tests - debugging --- tests/integration/test_installation.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index 38e6e814a..c1e6bc99e 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -35,10 +35,12 @@ def run_salt_call(cmd): log.error(f"failed to produce output result, '{result}'") else: - cmdl = ["sudo"] + ## DGM cmdl = ["sudo"] + cmdl = [] cmdl.extend(cmd) cmdl.append("--out=json") result = subprocess.run(cmdl, capture_output=True, text=True) + print(f"DGM run_salt_call result '{result}'", flush=True) if 0 == result.returncode: json_data = json.loads(result.stdout) else: @@ -50,6 +52,7 @@ def run_salt_call(cmd): def test_ping(path): cmd = ["salt-call", "--local", "test.ping"] result = run_salt_call(cmd) + print(f"DGM test_ping result '{result}'", flush=True) assert result == True @@ -57,6 +60,7 @@ def test_target_python_version(path, target_python_version): cmd = ["salt-call", "--local", "grains.item", "pythonversion", "--timeout=120"] result = run_salt_call(cmd) # Returns: {'pythonversion': [3, 10, 11, 'final', 0]} + print(f"DGM test_target_python_version result '{result}'", flush=True) py_maj_ver = result["pythonversion"][0] assert py_maj_ver == target_python_version @@ -67,5 +71,6 @@ def test_target_salt_version(path, target_salt_version): cmd = ["salt-call", "--local", "grains.item", "saltversion", "--timeout=120"] result = run_salt_call(cmd) # Returns: {'saltversion': '3006.9+217.g53cfa53040'} + print(f"DGM test_target_salt_version result '{result}'", flush=True) adj_saltversion = result["saltversion"].split("+")[0] assert adj_saltversion == target_salt_version From 651d50aa05dd91b32e6eee3c2891841bcbb10c3b Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 11 Dec 2024 16:01:54 -0700 Subject: [PATCH 167/173] Updated tests to allow for older Python versions, add Amazon containers --- .github/workflows/ci.yml | 32 +++++++++++++++++++++++++ .github/workflows/templates/generate.py | 12 +++++----- tests/integration/test_installation.py | 11 +++++++-- 3 files changed, 47 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cfc4d8444..253a76931 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -180,6 +180,36 @@ jobs: + amazonlinux-2: + name: Amazon 2 + if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' + uses: ./.github/workflows/test-linux.yml + needs: + - lint + - generate-actions-workflow + with: + distro-slug: amazonlinux-2 + display-name: Amazon 2 + container-slug: amazonlinux-2 + timeout: 20 + instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' + + + amazonlinux-2023: + name: Amazon 2023 + if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' + uses: ./.github/workflows/test-linux.yml + needs: + - lint + - generate-actions-workflow + with: + distro-slug: amazonlinux-2023 + display-name: Amazon 2023 + container-slug: amazonlinux-2023 + timeout: 20 + instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' + + debian-12: name: Debian 12 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' @@ -267,6 +297,8 @@ jobs: - macos-13 - macos-14 - windows-2022 + - amazonlinux-2 + - amazonlinux-2023 - debian-12 - photon-5 - rockylinux-8 diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index e0ffa3567..3a4ae7125 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -8,8 +8,6 @@ # only test against current containers with systemd # will add these when they become available with systemd -# "amazonlinux-2", -# "amazonlinux-2023", # "debian-11", # "debian-13", # "fedora-40", @@ -17,6 +15,8 @@ # "ubuntu-2004", # "ubuntu-2404", LINUX_DISTROS = [ + "amazonlinux-2", + "amazonlinux-2023", "debian-12", "photon-5", "rockylinux-8", @@ -36,8 +36,6 @@ # only test against current containers with systemd # will add these when they become available with systemd -# "amazonlinux-2", -# "amazonlinux-2023", # "debian-11", # "debian-13", # "fedora-40", @@ -45,6 +43,8 @@ # "ubuntu-2004", # "ubuntu-2404", STABLE_DISTROS = [ + "amazonlinux-2", + "amazonlinux-2023", "debian-12", "photon-5", "rockylinux-8", @@ -54,8 +54,6 @@ # only test against current containers with systemd # will add these when they become available with systemd -# "amazonlinux-2", -# "amazonlinux-2023", # "debian-11", # "debian-13", # "fedora-40", @@ -63,6 +61,8 @@ # "ubuntu-2004", # "ubuntu-2404", ONEDIR_DISTROS = [ + "amazonlinux-2", + "amazonlinux-2023", "debian-12", "photon-5", "rockylinux-8", diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index c1e6bc99e..e8ec2a7d8 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -35,11 +35,18 @@ def run_salt_call(cmd): log.error(f"failed to produce output result, '{result}'") else: - ## DGM cmdl = ["sudo"] - cmdl = [] + print(f"DGM run_salt_call platform.system '{platform.system()}'", flush=True) + if platform.system() == "Darwin": + cmdl = ["sudo"] + else: + cmdl = [] cmdl.extend(cmd) cmdl.append("--out=json") result = subprocess.run(cmdl, capture_output=True, text=True) + try: + result = subprocess.run(cmd, capture_output=True, text=True) + except TypeError: + result = subprocess.run(cmd, text=True) print(f"DGM run_salt_call result '{result}'", flush=True) if 0 == result.returncode: json_data = json.loads(result.stdout) From 0075aa99027e611b00cb4a0dd350617b223e334c Mon Sep 17 00:00:00 2001 From: David Murphy Date: Wed, 11 Dec 2024 16:56:41 -0700 Subject: [PATCH 168/173] Updated Amazon dependencies, and tests --- bootstrap-salt.sh | 4 ++-- tests/integration/test_installation.py | 5 ++--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 893205d1d..59688d092 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -5731,7 +5731,7 @@ install_amazon_linux_ami_2_onedir_deps() { # We need to install yum-utils before doing anything else when installing on # Amazon Linux ECS-optimized images. See issue #974. - __PACKAGES="yum-utils" + __PACKAGES="yum-utils chkconfig procps-ng findutils" # ensure sudo installed __PACKAGES="${__PACKAGES} sudo" @@ -5919,7 +5919,7 @@ install_amazon_linux_ami_2023_onedir_deps() { # We need to install yum-utils before doing anything else when installing on # Amazon Linux ECS-optimized images. See issue #974. - __PACKAGES="yum-utils" + __PACKAGES="yum-utils chkconfig procps-ng findutils" # ensure sudo installed __PACKAGES="${__PACKAGES} sudo" diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index e8ec2a7d8..2bc791aa9 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -42,11 +42,10 @@ def run_salt_call(cmd): cmdl = [] cmdl.extend(cmd) cmdl.append("--out=json") - result = subprocess.run(cmdl, capture_output=True, text=True) try: - result = subprocess.run(cmd, capture_output=True, text=True) + result = subprocess.run(cmdl, capture_output=True, text=True) except TypeError: - result = subprocess.run(cmd, text=True) + result = subprocess.run(cmdl, text=True) print(f"DGM run_salt_call result '{result}'", flush=True) if 0 == result.returncode: json_data = json.loads(result.stdout) From 316e65a97927f4daa2e96aba46fc4846538d3989 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Thu, 12 Dec 2024 10:24:19 -0700 Subject: [PATCH 169/173] Updated tests, and debugging, min Python version 3.6 --- .pre-commit-config.yaml | 6 ++++-- bootstrap-salt.sh | 8 +++++++- tests/integration/test_installation.py | 4 +++- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 85e4d1787..04a74c18c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -41,8 +41,10 @@ repos: rev: v3.15.0 hooks: - id: pyupgrade - name: Rewrite Code to be Py3.9+ - args: [--py39-plus] + name: Rewrite Code to be Py3.6+ + args: [--py36-plus] + ## DGM name: Rewrite Code to be Py3.9+ + ## DGM args: [--py39-plus] - repo: https://github.com/asottile/reorder_python_imports rev: v3.12.0 diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 59688d092..17646579b 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -26,7 +26,7 @@ #====================================================================================================================== set -o nounset # Treat unset variables as an error -__ScriptVersion="2024.12.09" +__ScriptVersion="2024.12.12" __ScriptName="bootstrap-salt.sh" __ScriptFullName="$0" @@ -2468,6 +2468,12 @@ __check_services_systemd() { servicename=$1 echodebug "Checking if service ${servicename} is enabled" + ## DGM + if [ "${DISTRO_NAME_L}" = "amazon_linux_ami" ] && [ "${DISTRO_VERSION}" = "2" ]; then + echoinfo "DGM skipping the is-enabled test on AMZN2, since command 'systemctl is-enabled salt-minion' is generating error msg 'Failed to get D-Bus connection: No such file or directory' on AMZN2 container" + return 0 + fi + if [ "$(systemctl is-enabled "${servicename}")" = "enabled" ]; then echodebug "Service ${servicename} is enabled" return 0 diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index 2bc791aa9..993f24b0e 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -45,7 +45,9 @@ def run_salt_call(cmd): try: result = subprocess.run(cmdl, capture_output=True, text=True) except TypeError: - result = subprocess.run(cmdl, text=True) + result = subprocess.run( + cmdl, stdout=PIPE, stderr=PIPE, universal_newlines=True + ) print(f"DGM run_salt_call result '{result}'", flush=True) if 0 == result.returncode: json_data = json.loads(result.stdout) From 1040775f56f66e1aa501655eec793e400de0044b Mon Sep 17 00:00:00 2001 From: David Murphy Date: Thu, 12 Dec 2024 10:55:42 -0700 Subject: [PATCH 170/173] Added Debian 11 and 13 containers, removed Amazon Linux 2 container, updated test --- .github/workflows/ci.yml | 38 +++++++---- .github/workflows/templates/generate.py | 39 +++--------- bootstrap-salt.sh | 85 +++++-------------------- tests/integration/test_installation.py | 5 +- 4 files changed, 57 insertions(+), 110 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 253a76931..1df1bd8de 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -180,32 +180,32 @@ jobs: - amazonlinux-2: - name: Amazon 2 + amazonlinux-2023: + name: Amazon 2023 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' uses: ./.github/workflows/test-linux.yml needs: - lint - generate-actions-workflow with: - distro-slug: amazonlinux-2 - display-name: Amazon 2 - container-slug: amazonlinux-2 + distro-slug: amazonlinux-2023 + display-name: Amazon 2023 + container-slug: amazonlinux-2023 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' - amazonlinux-2023: - name: Amazon 2023 + debian-11: + name: Debian 11 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' uses: ./.github/workflows/test-linux.yml needs: - lint - generate-actions-workflow with: - distro-slug: amazonlinux-2023 - display-name: Amazon 2023 - container-slug: amazonlinux-2023 + distro-slug: debian-11 + display-name: Debian 11 + container-slug: debian-11 timeout: 20 instances: '["stable-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' @@ -225,6 +225,21 @@ jobs: instances: '["stable-3007", "git-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' + debian-13: + name: Debian 13 + if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' + uses: ./.github/workflows/test-linux.yml + needs: + - lint + - generate-actions-workflow + with: + distro-slug: debian-13 + display-name: Debian 13 + container-slug: debian-13 + timeout: 20 + instances: '["stable-3006", "git-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' + + photon-5: name: Photon OS 5 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' @@ -297,9 +312,10 @@ jobs: - macos-13 - macos-14 - windows-2022 - - amazonlinux-2 - amazonlinux-2023 + - debian-11 - debian-12 + - debian-13 - photon-5 - rockylinux-8 - rockylinux-9 diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index 3a4ae7125..78f1d8216 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -6,18 +6,16 @@ os.chdir(os.path.abspath(os.path.dirname(__file__))) -# only test against current containers with systemd -# will add these when they become available with systemd -# "debian-11", -# "debian-13", +# "amazonlinux-2", # "fedora-40", # "photon-4", # "ubuntu-2004", # "ubuntu-2404", LINUX_DISTROS = [ - "amazonlinux-2", "amazonlinux-2023", + "debian-11", "debian-12", + "debian-13", "photon-5", "rockylinux-8", "rockylinux-9", @@ -34,44 +32,38 @@ "macos-14", ] -# only test against current containers with systemd -# will add these when they become available with systemd -# "debian-11", -# "debian-13", +# "amazonlinux-2", # "fedora-40", # "photon-4", # "ubuntu-2004", # "ubuntu-2404", STABLE_DISTROS = [ - "amazonlinux-2", "amazonlinux-2023", + "debian-11", "debian-12", + "debian-13", "photon-5", "rockylinux-8", "rockylinux-9", "ubuntu-2204", ] -# only test against current containers with systemd -# will add these when they become available with systemd -# "debian-11", -# "debian-13", +# "amazonlinux-2", # "fedora-40", # "photon-4", # "ubuntu-2004", # "ubuntu-2404", ONEDIR_DISTROS = [ - "amazonlinux-2", "amazonlinux-2023", + "debian-11", "debian-12", + "debian-13", "photon-5", "rockylinux-8", "rockylinux-9", "ubuntu-2204", ] -# only test against current containers with systemd -# will add these when they become available with systemd # "amazonlinux-2", # "amazonlinux-2023", # "photon-4", @@ -216,19 +208,6 @@ "windows-2022": "Windows 2022", } -## "amazonlinux-2": "systemd-amazonlinux-2", -## "amazonlinux-2023": "systemd-amazonlinux-2023", -## "debian-11": "systemd-debian-11", -## "debian-12": "systemd-debian-12", -## "debian-13": "systemd-debian-13", -## "fedora-40": "systemd-fedora-40", -## "photon-4": "systemd-photon-4", -## "photon-5": "systemd-photon-5", -## "rockylinux-8": "systemd-rockylinux-8", -## "rockylinux-9": "systemd-rockylinux-9", -## "ubuntu-2004": "systemd-ubuntu-20.04", -## "ubuntu-2204": "systemd-ubuntu-22.04", -## "ubuntu-2404": "systemd-ubuntu-24.04", CONTAINER_SLUG_NAMES = { "amazonlinux-2": "amazonlinux-2", "amazonlinux-2023": "amazonlinux-2023", diff --git a/bootstrap-salt.sh b/bootstrap-salt.sh index 17646579b..bd29313b8 100755 --- a/bootstrap-salt.sh +++ b/bootstrap-salt.sh @@ -2468,12 +2468,6 @@ __check_services_systemd() { servicename=$1 echodebug "Checking if service ${servicename} is enabled" - ## DGM - if [ "${DISTRO_NAME_L}" = "amazon_linux_ami" ] && [ "${DISTRO_VERSION}" = "2" ]; then - echoinfo "DGM skipping the is-enabled test on AMZN2, since command 'systemctl is-enabled salt-minion' is generating error msg 'Failed to get D-Bus connection: No such file or directory' on AMZN2 container" - return 0 - fi - if [ "$(systemctl is-enabled "${servicename}")" = "enabled" ]; then echodebug "Service ${servicename} is enabled" return 0 @@ -3195,11 +3189,8 @@ install_ubuntu_git_deps() { __PACKAGES="${__PACKAGES} util-linux-extra" fi - # Additionally install procps and pciutils which allows for Docker bootstraps. See 366#issuecomment-39666813 - __PACKAGES="${__PACKAGES} procps pciutils" - - # ensure sudo installed - __PACKAGES="${__PACKAGES} sudo" + # Additionally install procps pciutils and sudo which allows for Docker bootstraps. See 366#issuecomment-39666813 + __PACKAGES="${__PACKAGES} procps pciutils sudo" # shellcheck disable=SC2086 __apt_get_install_noinput ${__PACKAGES} || return 1 @@ -3598,15 +3589,12 @@ install_debian_onedir_deps() { return 1 fi - # Additionally install procps and pciutils which allows for Docker bootstraps. See 366#issuecomment-39666813 - __PACKAGES='procps pciutils' + # Additionally install procps, pciutils and sudo which allows for Docker bootstraps. See 366#issuecomment-39666813 + __PACKAGES='procps pciutils sudo' # YAML module is used for generating custom master/minion configs __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-yaml" - # ensure sudo installed - __PACKAGES="${__PACKAGES} sudo" - # shellcheck disable=SC2086 __apt_get_install_noinput ${__PACKAGES} || return 1 @@ -3649,11 +3637,8 @@ install_debian_git_deps() { __PACKAGES="python${PY_PKG_VER}-dev python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools gcc" echodebug "install_debian_git_deps() Installing ${__PACKAGES}" - # Additionally install procps and pciutils which allows for Docker bootstraps. See 366#issuecomment-39666813 - __PACKAGES="${__PACKAGES} procps pciutils" - - # ensure sudo installed - __PACKAGES="${__PACKAGES} sudo" + # Additionally install procps, pciutils and sudo which allows for Docker bootstraps. See 366#issuecomment-39666813 + __PACKAGES="${__PACKAGES} procps pciutils sudo" # shellcheck disable=SC2086 __apt_get_install_noinput ${__PACKAGES} || return 1 @@ -3953,14 +3938,11 @@ install_fedora_deps() { __PACKAGES="${__PACKAGES} dnf-utils libyaml procps-ng python${PY_PKG_VER}-crypto python${PY_PKG_VER}-jinja2" __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-msgpack python${PY_PKG_VER}-requests python${PY_PKG_VER}-zmq" __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-pip python${PY_PKG_VER}-m2crypto python${PY_PKG_VER}-pyyaml" - __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-systemd" + __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-systemd sudo" if [ "${_EXTRA_PACKAGES}" != "" ]; then echoinfo "Installing the following extra packages as requested: ${_EXTRA_PACKAGES}" fi - # ensure sudo installed - __PACKAGES="${__PACKAGES} sudo" - # shellcheck disable=SC2086 __dnf_install_noinput ${__PACKAGES} ${_EXTRA_PACKAGES} || return 1 @@ -3991,10 +3973,7 @@ install_fedora_git_deps() { # shellcheck disable=SC2119 __git_clone_and_checkout || return 1 - __PACKAGES="python${PY_PKG_VER}-devel python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools gcc gcc-c++" - - # ensure sudo installed - __PACKAGES="${__PACKAGES} sudo" + __PACKAGES="python${PY_PKG_VER}-devel python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools gcc gcc-c++ sudo" # shellcheck disable=SC2086 __dnf_install_noinput ${__PACKAGES} || return 1 @@ -4123,10 +4102,7 @@ install_fedora_onedir_deps() { __install_saltstack_fedora_onedir_repository || return 1 fi - __PACKAGES="dnf-utils chkconfig procps-ng" - - # ensure sudo installed - __PACKAGES="${__PACKAGES} sudo" + __PACKAGES="dnf-utils chkconfig procps-ng sudo" # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -4286,10 +4262,7 @@ install_centos_stable_deps() { __install_saltstack_rhel_onedir_repository || return 1 fi - __PACKAGES="yum-utils chkconfig procps-ng findutils" - - # ensure sudo installed - __PACKAGES="${__PACKAGES} sudo" + __PACKAGES="yum-utils chkconfig procps-ng findutils sudo" # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -4335,9 +4308,6 @@ install_centos_stable() { __PACKAGES="${__PACKAGES} salt-api$MINOR_VER_STRG" fi - # ensure sudo installed - __PACKAGES="${__PACKAGES} sudo" - # shellcheck disable=SC2086 yum makecache || return 1 __yum_install_noinput ${__PACKAGES} || return 1 @@ -4416,10 +4386,7 @@ install_centos_git_deps() { return 1 fi - __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-devel python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools gcc" - - # ensure sudo installed - __PACKAGES="${__PACKAGES} sudo" + __PACKAGES="${__PACKAGES} python${PY_PKG_VER}-devel python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools gcc sudo" # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -4515,10 +4482,7 @@ install_centos_onedir_deps() { __install_saltstack_rhel_onedir_repository || return 1 fi - __PACKAGES="yum-utils chkconfig procps-ng findutils" - - # ensure sudo installed - __PACKAGES="${__PACKAGES} sudo" + __PACKAGES="yum-utils chkconfig procps-ng findutils sudo" # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -5619,10 +5583,7 @@ install_amazon_linux_ami_2_git_deps() { # shellcheck disable=SC2119 __git_clone_and_checkout || return 1 - __PACKAGES="python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools python${PY_PKG_VER}-devel gcc" - - # ensure sudo installed - __PACKAGES="${__PACKAGES} sudo" + __PACKAGES="python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools python${PY_PKG_VER}-devel gcc sudo" # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -5644,10 +5605,7 @@ install_amazon_linux_ami_2_deps() { # We need to install yum-utils before doing anything else when installing on # Amazon Linux ECS-optimized images. See issue #974. - __PACKAGES="yum-utils" - - # ensure sudo installed - __PACKAGES="${__PACKAGES} sudo" + __PACKAGES="yum-utils sudo" __yum_install_noinput ${__PACKAGES} @@ -5737,10 +5695,7 @@ install_amazon_linux_ami_2_onedir_deps() { # We need to install yum-utils before doing anything else when installing on # Amazon Linux ECS-optimized images. See issue #974. - __PACKAGES="yum-utils chkconfig procps-ng findutils" - - # ensure sudo installed - __PACKAGES="${__PACKAGES} sudo" + __PACKAGES="yum-utils chkconfig procps-ng findutils sudo" __yum_install_noinput ${__PACKAGES} @@ -5897,10 +5852,7 @@ install_amazon_linux_ami_2023_git_deps() { # shellcheck disable=SC2119 __git_clone_and_checkout || return 1 - __PACKAGES="python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools python${PY_PKG_VER}-devel gcc" - - # ensure sudo installed - __PACKAGES="${__PACKAGES} sudo" + __PACKAGES="python${PY_PKG_VER}-pip python${PY_PKG_VER}-setuptools python${PY_PKG_VER}-devel gcc sudo" # shellcheck disable=SC2086 __yum_install_noinput ${__PACKAGES} || return 1 @@ -5925,10 +5877,7 @@ install_amazon_linux_ami_2023_onedir_deps() { # We need to install yum-utils before doing anything else when installing on # Amazon Linux ECS-optimized images. See issue #974. - __PACKAGES="yum-utils chkconfig procps-ng findutils" - - # ensure sudo installed - __PACKAGES="${__PACKAGES} sudo" + __PACKAGES="yum-utils chkconfig procps-ng findutils sudo" __yum_install_noinput ${__PACKAGES} diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index 993f24b0e..94a9d416e 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -46,7 +46,10 @@ def run_salt_call(cmd): result = subprocess.run(cmdl, capture_output=True, text=True) except TypeError: result = subprocess.run( - cmdl, stdout=PIPE, stderr=PIPE, universal_newlines=True + cmdl, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, ) print(f"DGM run_salt_call result '{result}'", flush=True) if 0 == result.returncode: From 73475d2505d7648bceed428817df056677eb2325 Mon Sep 17 00:00:00 2001 From: David Murphy Date: Thu, 12 Dec 2024 12:24:29 -0700 Subject: [PATCH 171/173] Removed Debian-13 container and cleaned up debug statements --- .github/workflows/ci.yml | 16 -------- .github/workflows/templates/generate.py | 7 ++-- .github/workflows/test-linux.yml | 49 +------------------------ tests/integration/test_installation.py | 5 --- 4 files changed, 6 insertions(+), 71 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1df1bd8de..71d622007 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -225,21 +225,6 @@ jobs: instances: '["stable-3007", "git-3007", "onedir-3007", "stable-3007-1", "git-master", "latest", "default"]' - debian-13: - name: Debian 13 - if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' - uses: ./.github/workflows/test-linux.yml - needs: - - lint - - generate-actions-workflow - with: - distro-slug: debian-13 - display-name: Debian 13 - container-slug: debian-13 - timeout: 20 - instances: '["stable-3006", "git-3006", "onedir-3006", "stable-3006-8", "stable-3007", "onedir-3007", "stable-3007-1", "latest", "default"]' - - photon-5: name: Photon OS 5 if: github.event_name == 'push' || needs.collect-changed-files.outputs.run-tests == 'true' @@ -315,7 +300,6 @@ jobs: - amazonlinux-2023 - debian-11 - debian-12 - - debian-13 - photon-5 - rockylinux-8 - rockylinux-9 diff --git a/.github/workflows/templates/generate.py b/.github/workflows/templates/generate.py index 78f1d8216..cb0e693ac 100755 --- a/.github/workflows/templates/generate.py +++ b/.github/workflows/templates/generate.py @@ -7,6 +7,7 @@ os.chdir(os.path.abspath(os.path.dirname(__file__))) # "amazonlinux-2", +# "debian-13", # "fedora-40", # "photon-4", # "ubuntu-2004", @@ -15,7 +16,6 @@ "amazonlinux-2023", "debian-11", "debian-12", - "debian-13", "photon-5", "rockylinux-8", "rockylinux-9", @@ -33,6 +33,7 @@ ] # "amazonlinux-2", +# "debian-13", # "fedora-40", # "photon-4", # "ubuntu-2004", @@ -41,7 +42,6 @@ "amazonlinux-2023", "debian-11", "debian-12", - "debian-13", "photon-5", "rockylinux-8", "rockylinux-9", @@ -49,6 +49,7 @@ ] # "amazonlinux-2", +# "debian-13", # "fedora-40", # "photon-4", # "ubuntu-2004", @@ -57,7 +58,6 @@ "amazonlinux-2023", "debian-11", "debian-12", - "debian-13", "photon-5", "rockylinux-8", "rockylinux-9", @@ -94,6 +94,7 @@ "amazonlinux-2023", "debian-11", "debian-12", + "debian-13", "fedora-40", "photon-4", "photon-5", diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index 5ed8719fe..b9c5662ba 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -32,10 +32,6 @@ jobs: Test: name: ${{ matrix.instance }} runs-on: ubuntu-latest - ## DGM container: - ## DGM image: ghcr.io/saltstack/salt-ci-containers/testing:${{ inputs.container-slug }} - ## DGM options: --privileged - timeout-minutes: ${{ inputs.timeout }} strategy: max-parallel: 2 @@ -46,37 +42,6 @@ jobs: steps: - uses: actions/checkout@v4 - ## DGM - name: Install Python Dependencies with pip breakage - ## DGM if: ${{ ( inputs.distro-slug == 'debian-11' ) || ( inputs.distro-slug == 'debian-12' ) || ( inputs.distro-slug == 'debian-13' ) || ( inputs.distro-slug == 'ubuntu-2404' ) }} - ## DGM run: | - ## DGM python3 -m pip install --break-system-packages -r tests/requirements.txt - - ## DGM - name: Install Python Dependencies without pip breakage - ## DGM if: ${{ ( inputs.distro-slug != 'debian-11' ) && ( inputs.distro-slug != 'debian-12' ) && ( inputs.distro-slug != 'debian-13' ) && ( inputs.distro-slug != 'ubuntu-2404' ) }} - ## DGM run: | - ## DGM python3 -m pip install -r tests/requirements.txt - - ## DGM - name: Get Version - ## DGM run: | - ## DGM # We need to get the version here and make it an environment variable - ## DGM # It is used to install via bootstrap and in the test - ## DGM # The version is in the instance name - ## DGM # sed 1st - becomes space, 2nd - becomes dot - ## DGM vt_parm_ver=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./' | awk -F ' ' '{print $2}') - ## DGM echo "SaltVersion=$vt_parm_ver" >> $GITHUB_ENV - - ## DGM - name: Bootstrap Salt - ## DGM run: | - ## DGM # sed 1st - becomes space, 2nd - becomes dot - ## DGM bt_arg1=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./' | awk -F ' ' '{print $1}') - ## DGM bt_arg2=$(echo "${{ matrix.instance }}" | sed 's/-/ /' | sed 's/-/./' | awk -F ' ' '{print $2}') - ## DGM echo "bt parms ,$bt_parms, bt_arg1 ,$bt_arg1, bt_arg2 ,$bt_arg2," - ## DGM sh -x ./bootstrap-salt.sh "$bt_arg1" "$bt_arg2" - - ## DGM - name: Test Bootstrap - ## DGM run: | - ## DGM pytest --cache-clear -v -s -ra --log-cli-level=debug tests/integration/ - - name: Get Version run: | # We need to get the version here and make it an environment variable @@ -98,23 +63,13 @@ jobs: run: | /usr/bin/docker start ${{ github.run_id }}_salt-test - - name: Current Directory - run: | - pwd - ls -alh - - ## DGM - name: Download requirements file tests/requirements.txt - ## DGM uses: actions/download-artifact@v4 - ## DGM with: - ## DGM - name: requirements.txt - - name: "Install Python Dependencies with pip breakage in container ${{ inputs.container-slug }}" - if: ${{ ( inputs.distro-slug == 'debian-11' ) || ( inputs.distro-slug == 'debian-12' ) || ( inputs.distro-slug == 'debian-13' ) || ( inputs.distro-slug == 'ubuntu-2404' ) }} + if: ${{ ( inputs.distro-slug == 'debian-12' ) || ( inputs.distro-slug == 'debian-13' ) || ( inputs.distro-slug == 'ubuntu-2404' ) }} run: | docker exec ${{ github.run_id}}_salt-test python3 -m pip install --break-system-packages -r /_w/btstrap/tests/requirements.txt - name: "Install Python Dependencies without pip breakage in container ${{ inputs.container-slug }}" - if: ${{ ( inputs.distro-slug != 'debian-11' ) && ( inputs.distro-slug != 'debian-12' ) && ( inputs.distro-slug != 'debian-13' ) && ( inputs.distro-slug != 'ubuntu-2404' ) }} + if: ${{ ( inputs.distro-slug != 'debian-12' ) && ( inputs.distro-slug != 'debian-13' ) && ( inputs.distro-slug != 'ubuntu-2404' ) }} run: | docker exec ${{ github.run_id}}_salt-test python3 -m pip install -r /_w/btstrap/tests/requirements.txt diff --git a/tests/integration/test_installation.py b/tests/integration/test_installation.py index 94a9d416e..2bc739489 100644 --- a/tests/integration/test_installation.py +++ b/tests/integration/test_installation.py @@ -35,7 +35,6 @@ def run_salt_call(cmd): log.error(f"failed to produce output result, '{result}'") else: - print(f"DGM run_salt_call platform.system '{platform.system()}'", flush=True) if platform.system() == "Darwin": cmdl = ["sudo"] else: @@ -51,7 +50,6 @@ def run_salt_call(cmd): stderr=subprocess.PIPE, universal_newlines=True, ) - print(f"DGM run_salt_call result '{result}'", flush=True) if 0 == result.returncode: json_data = json.loads(result.stdout) else: @@ -63,7 +61,6 @@ def run_salt_call(cmd): def test_ping(path): cmd = ["salt-call", "--local", "test.ping"] result = run_salt_call(cmd) - print(f"DGM test_ping result '{result}'", flush=True) assert result == True @@ -71,7 +68,6 @@ def test_target_python_version(path, target_python_version): cmd = ["salt-call", "--local", "grains.item", "pythonversion", "--timeout=120"] result = run_salt_call(cmd) # Returns: {'pythonversion': [3, 10, 11, 'final', 0]} - print(f"DGM test_target_python_version result '{result}'", flush=True) py_maj_ver = result["pythonversion"][0] assert py_maj_ver == target_python_version @@ -82,6 +78,5 @@ def test_target_salt_version(path, target_salt_version): cmd = ["salt-call", "--local", "grains.item", "saltversion", "--timeout=120"] result = run_salt_call(cmd) # Returns: {'saltversion': '3006.9+217.g53cfa53040'} - print(f"DGM test_target_salt_version result '{result}'", flush=True) adj_saltversion = result["saltversion"].split("+")[0] assert adj_saltversion == target_salt_version From db07164315a7c1dd384a59e2fd78cba67031f845 Mon Sep 17 00:00:00 2001 From: Salt Project Packaging Date: Thu, 12 Dec 2024 20:16:55 +0000 Subject: [PATCH 172/173] Update develop branch for the v2024.12.12 release --- CHANGELOG.md | 9 +++++++++ bootstrap-salt.ps1 | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 16e7ac943..7b87b2d2a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +# v2024.12.12 + +## What's Changed + +- Restrict use of dnf to Fedora only, otherwise use yum when dealing with RedHat family by @dmurphy18 in https://github.com/saltstack/salt-bootstrap/pull/2070 +- Updating container usage to that used in new Salt pipelines for CI/CD, fix Amazon Linux by @dmurphy18 in https://github.com/saltstack/salt-bootstrap/pull/2071 + +**Full Changelog**: https://github.com/saltstack/salt-bootstrap/compare/v2024.12.09...v2024.12.12 + # v2024.12.09 ## What's Changed diff --git a/bootstrap-salt.ps1 b/bootstrap-salt.ps1 index f355e0cd1..8b1bb3a38 100644 --- a/bootstrap-salt.ps1 +++ b/bootstrap-salt.ps1 @@ -108,7 +108,7 @@ if ($help) { exit 0 } -$__ScriptVersion = "2024.12.09" +$__ScriptVersion = "2024.12.12" $ScriptName = $myInvocation.MyCommand.Name # We'll check for the Version next, because it also has no requirements From 2058991403a843ba9ac43de913097e92d4475750 Mon Sep 17 00:00:00 2001 From: Salt Project Packaging Date: Thu, 12 Dec 2024 20:17:42 +0000 Subject: [PATCH 173/173] Update README.rst with 2024.12.12 release sha256sum --- README.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/README.rst b/README.rst index 233b3f671..db1baf02e 100644 --- a/README.rst +++ b/README.rst @@ -37,6 +37,7 @@ sum** of the downloaded ``bootstrap-salt.sh`` file. The SHA256 sum of the ``bootstrap-salt.sh`` file, per release, is: +- 2024.12.12: ``7cc91adfa5a15ff57d203dc2b79608c773efc639d4e9bf03861198903e11becd`` - 2024.12.09: ``44f9405a6d9622ad8fa7c93e83a52e01ca328f27e4e9dea4a52268c6a22dbe6d`` - 2024.11.29: ``0ac87384dee051aceded69704485a5de0e4a308551a462b10c262111b57acff0`` - 2024.11.27: ``e972bd1ef01d09cd1d9294374ef974c9e3dd9a2aee37cf3859144585fd8bf1d0``