update packman

tools/packman/bootstrap/configure.bat

@@ -1,4 +1,4 @@
-:: Copyright 2019 NVIDIA CORPORATION
+:: Copyright 2019-2023 NVIDIA CORPORATION
 ::
 :: Licensed under the Apache License, Version 2.0 (the "License");
 :: you may not use this file except in compliance with the License.
@@ -12,7 +12,7 @@
 :: See the License for the specific language governing permissions and
 :: limitations under the License.
 
-set PM_PACKMAN_VERSION=6.42
+set PM_PACKMAN_VERSION=7.15.1
 
 :: Specify where packman command is rooted
 set PM_INSTALL_PATH=%~dp0..
@@ -59,7 +59,7 @@ if defined PM_PYTHON_EXT (
 	goto PACKMAN
 )
 
-set PM_PYTHON_VERSION=3.7.12-windows-x86_64
+set PM_PYTHON_VERSION=3.10.5-1-windows-x86_64
 set PM_PYTHON_BASE_DIR=%PM_PACKAGES_ROOT%\python
 set PM_PYTHON_DIR=%PM_PYTHON_BASE_DIR%\%PM_PYTHON_VERSION%
 set PM_PYTHON=%PM_PYTHON_DIR%\python.exe
@@ -95,11 +95,16 @@ if exist "%PM_PYTHON%" (
     if exist "%PM_PYTHON_DIR%" ( rd /s /q "%PM_PYTHON_DIR%" > nul )
 )
 
-:: Perform atomic rename
-rename "%TEMP_FOLDER_NAME%" "%PM_PYTHON_VERSION%" 1> nul
-:: Failure during move, need to clean up and abort
+:: Perform atomic move (allowing overwrite, /y)
+move /y "%TEMP_FOLDER_NAME%" "%PM_PYTHON_DIR%" 1> nul
+:: Verify that python.exe is now where we expect
+if exist "%PM_PYTHON%" goto PACKMAN
+
+:: Wait a second and try again (can help with access denied weirdness)
+timeout /t 1 /nobreak 1> nul
+move /y "%TEMP_FOLDER_NAME%" "%PM_PYTHON_DIR%" 1> nul
 if %errorlevel% neq 0 (
-    echo !!! Error renaming python !!!
+    echo !!! Error moving python %TEMP_FOLDER_NAME% -> %PM_PYTHON_DIR% !!!
     call :CLEAN_UP_TEMP_FOLDER
     goto ERROR
 )
@@ -114,7 +119,7 @@ if defined PM_MODULE_DIR_EXT (
 
 set PM_MODULE=%PM_MODULE_DIR%\run.py
 
-if exist "%PM_MODULE%" goto ENSURE_7ZA
+if exist "%PM_MODULE%" goto END
 
 :: Clean out broken PM_MODULE_DIR if it exists
 if exist "%PM_MODULE_DIR%" ( rd /s /q "%PM_MODULE_DIR%" > nul )
@@ -137,19 +142,6 @@ if %errorlevel% neq 0 (
 
 del "%TARGET%"
 
-:ENSURE_7ZA
-set PM_7Za_VERSION=16.02.4
-set PM_7Za_PATH=%PM_PACKAGES_ROOT%\7za\%PM_7ZA_VERSION%
-if exist "%PM_7Za_PATH%" goto END
-set PM_7Za_PATH=%PM_PACKAGES_ROOT%\chk\7za\%PM_7ZA_VERSION%
-if exist "%PM_7Za_PATH%" goto END
-
-"%PM_PYTHON%" -S -s -u -E "%PM_MODULE%" pull "%PM_MODULE_DIR%\deps.packman.xml"
-if %errorlevel% neq 0 (
-    echo !!! Error fetching packman dependencies !!!
-    goto ERROR
-)
-
 goto END
 
 :ERROR_MKDIR_PACKAGES_ROOT

tools/packman/bootstrap/install_package.py

@@ -144,4 +144,11 @@ def install_package(package_path, install_path):
 
 
 if __name__ == "__main__":
-    install_package(sys.argv[1], sys.argv[2])
+    executable_paths = os.getenv("PATH")
+    paths_list = executable_paths.split(os.path.pathsep) if executable_paths else []
+    target_path_np = os.path.normpath(sys.argv[2])
+    target_path_np_nc = os.path.normcase(target_path_np)
+    for exec_path in paths_list:
+        if os.path.normcase(os.path.normpath(exec_path)) == target_path_np_nc:
+            raise RuntimeError(f"packman will not install to executable path '{exec_path}'")
+    install_package(sys.argv[1], target_path_np)

tools/packman/packman

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-# Copyright 2019-2020 NVIDIA CORPORATION
+# Copyright 2019-2023 NVIDIA CORPORATION
 
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -18,11 +18,13 @@ set -eu
 
 if echo ${PM_VERBOSITY-} | grep -i "debug" > /dev/null ; then
 	set -x
+	PM_CURL_SILENT=""
+	PM_WGET_QUIET=""
 else
 	PM_CURL_SILENT="-s -S"
 	PM_WGET_QUIET="--quiet"
 fi
-PM_PACKMAN_VERSION=6.42
+PM_PACKMAN_VERSION=7.15.1
 
 # This is necessary for newer macOS
 if [ `uname` == 'Darwin' ]; then
@@ -42,9 +44,13 @@ export PM_INSTALL_PATH="$(get_abs_filename "$(dirname "${BASH_SOURCE}")")"
 if [ -z "${PM_PACKAGES_ROOT:-}" ]; then
 	# Set variable temporarily in this process so that the following execution will work
 	if [ `uname` == 'Darwin' ]; then
-		export PM_PACKAGES_ROOT="/Library/Caches/packman"
+		export PM_PACKAGES_ROOT="${HOME}/Library/Application Support/packman-cache"
 	else
-		export PM_PACKAGES_ROOT="/var/tmp/packman"
+		if [ -z "${XDG_CACHE_HOME:-}" ]; then
+			export PM_PACKAGES_ROOT="${HOME}/.cache/packman"
+		else
+			export PM_PACKAGES_ROOT="${XDG_CACHE_HOME}/packman"
+		fi
 	fi
 fi
 
@@ -81,7 +87,7 @@ install_python()
 {
 	PLATFORM=`uname`
 	PROCESSOR=`uname -m`
-	PYTHON_VERSION=3.7.12
+	PYTHON_VERSION=3.10.5-1
 
 	if [ $PLATFORM == 'Darwin' ]; then
 		PYTHON_PACKAGE=$PYTHON_VERSION-macos-x86_64
@@ -150,20 +156,6 @@ if [ ! -f "$PM_MODULE" ]; then
 	fi
 fi
 
-# Ensure 7za package exists:
-PM_7za_VERSION=16.02.4
-export PM_7za_PATH="$PM_PACKAGES_ROOT/7za/$PM_7za_VERSION"
-if [ ! -d "$PM_7za_PATH" ]; then
-    export PM_7za_PATH="$PM_PACKAGES_ROOT/chk/7za/$PM_7za_VERSION"
-    if [ ! -d "$PM_7za_PATH" ]; then
-        "$PM_PYTHON" -S -s -u -E "$PM_MODULE" pull "$PM_MODULE_DIR/deps.packman.xml"
-        if [ "$?" -ne 0 ]; then
-           echo "Failure while installing required 7za package"
-           exit 1
-        fi
-    fi
-fi
-
 # Generate temporary file name for environment variables:
 PM_VAR_PATH=`mktemp -u -t tmp.$$.pmvars.XXXXXX`
 

tools/packman/packman.cmd

@@ -1,23 +1,22 @@
-:: Reset errorlevel status (don't inherit from caller) [xxxxxxxxxxx]
+:: RUN_PM_MODULE must always be at the same spot for packman update to work (batch reloads file during update!) 
+:: [xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx]
+:: Reset errorlevel status (don't inherit from caller) 
 @call :ECHO_AND_RESET_ERROR
-:: You can remove the call below if you do your own manual configuration of the dev machines
-call "%~dp0\bootstrap\configure.bat"
 
+:: You can remove this section if you do your own manual configuration of the dev machines
+call :CONFIGURE
 if %errorlevel% neq 0 ( exit /b %errorlevel% )
+
 :: Everything below is mandatory
 if not defined PM_PYTHON goto :PYTHON_ENV_ERROR
 if not defined PM_MODULE goto :MODULE_ENV_ERROR
 
-:: Generate temporary path for variable file
-for /f "delims=" %%a in ('powershell -ExecutionPolicy ByPass -NoLogo -NoProfile ^
--File "%~dp0bootstrap\generate_temp_file_name.ps1"') do set PM_VAR_PATH=%%a
+set PM_VAR_PATH_ARG=
 
-if %1.==. (
-	set PM_VAR_PATH_ARG=
-) else (
-	set PM_VAR_PATH_ARG=--var-path="%PM_VAR_PATH%"
-)
+if "%1"=="pull" goto :SET_VAR_PATH
+if "%1"=="install" goto :SET_VAR_PATH
 
+:RUN_PM_MODULE
 "%PM_PYTHON%" -S -s -u -E "%PM_MODULE%" %* %PM_VAR_PATH_ARG%
 if %errorlevel% neq 0 ( exit /b %errorlevel% )
 
@@ -48,9 +47,43 @@ exit /b 1
 @echo Error while processing and setting environment variables!
 exit /b 1
 
+:: pad [xxxx]
 :ECHO_AND_RESET_ERROR
 @echo off
 if /I "%PM_VERBOSITY%"=="debug" (
 	@echo on
 )
 exit /b 0
+
+:SET_VAR_PATH
+:: Generate temporary path for variable file
+for /f "delims=" %%a in ('%PM_PYTHON% -S -s -u -E -c "import tempfile;file = tempfile.NamedTemporaryFile(mode='w+t', delete=False);print(file.name)"') do (set PM_VAR_PATH=%%a)
+set PM_VAR_PATH_ARG=--var-path="%PM_VAR_PATH%"
+goto :RUN_PM_MODULE
+
+:CONFIGURE
+:: Must capture and set code page to work around issue #279, powershell invocation mutates console font
+:: This issue only happens in Windows CMD shell when using 65001 code page. Some Git Bash implementations 
+:: don't support chcp so this workaround is a bit convoluted.
+:: Test for chcp:
+chcp > nul 2>&1
+if %errorlevel% equ 0 ( 
+	for /f "tokens=2 delims=:" %%a in ('chcp') do (set PM_OLD_CODE_PAGE=%%a)
+) else (
+	call :ECHO_AND_RESET_ERROR
+)
+:: trim leading space (this is safe even when PM_OLD_CODE_PAGE has not been set)
+set PM_OLD_CODE_PAGE=%PM_OLD_CODE_PAGE:~1%
+if "%PM_OLD_CODE_PAGE%" equ "65001" (
+	chcp 437 > nul
+	set PM_RESTORE_CODE_PAGE=1
+)
+call "%~dp0\bootstrap\configure.bat"
+set PM_CONFIG_ERRORLEVEL=%errorlevel%
+if defined PM_RESTORE_CODE_PAGE (
+	:: Restore code page
+	chcp %PM_OLD_CODE_PAGE% > nul
+)
+set PM_OLD_CODE_PAGE=
+set PM_RESTORE_CODE_PAGE=
+exit /b %PM_CONFIG_ERRORLEVEL%

tools/packman/packmanconf.py

@@ -34,9 +34,9 @@ def init():
     """
     major = sys.version_info[0]
     minor = sys.version_info[1]
-    if major != 3 or minor != 7:
+    if major != 3 or minor != 10:
         raise RuntimeError(
-            f"This version of packman requires Python 3.7.x, but {major}.{minor} was provided"
+            f"This version of packman requires Python 3.10.x, but {major}.{minor} was provided"
         )
     conf_dir = os.path.dirname(os.path.abspath(__file__))
     os.environ["PM_INSTALL_PATH"] = conf_dir
@@ -55,9 +55,15 @@ def get_packages_root(conf_dir: str) -> str:
             root = os.path.join(drive, "packman-repo")
         elif platform_name == "Darwin":
             # macOS
-            root = "/Library/Caches/packman"
+            root = os.path.join(
+                os.path.expanduser("~"), "Library/Application Support/packman-cache"
+            )
         elif platform_name == "Linux":
-            root = "/var/tmp/packman"
+            try:
+                cache_root = os.environ["XDG_HOME_CACHE"]
+            except KeyError:
+                cache_root = os.path.join(os.path.expanduser("~"), ".cache")
+            return os.path.join(cache_root, "packman")
         else:
             raise RuntimeError(f"Unsupported platform '{platform_name}'")
     # make sure the path exists:

tools/packman/python.bat

@@ -13,9 +13,20 @@
 :: limitations under the License.
 
 @echo off
-setlocal
+setlocal enableextensions
 
 call "%~dp0\packman" init
 set "PYTHONPATH=%PM_MODULE_DIR%;%PYTHONPATH%"
-set PYTHONNOUSERSITE=1
-"%PM_PYTHON%" -u %*
+
+if not defined PYTHONNOUSERSITE (
+    set PYTHONNOUSERSITE=1
+)
+
+REM For performance, default to unbuffered; however, allow overriding via
+REM PYTHONUNBUFFERED=0 since PYTHONUNBUFFERED on windows can truncate output
+REM when printing long strings
+if not defined PYTHONUNBUFFERED (
+    set PYTHONUNBUFFERED=1
+)
+
+"%PM_PYTHON%" %*

tools/packman/python.sh

@@ -22,11 +22,21 @@ if [ ! -f "$PACKMAN_CMD" ]; then
 fi
 source "$PACKMAN_CMD" init
 export PYTHONPATH="${PM_MODULE_DIR}:${PYTHONPATH}"
-export PYTHONNOUSERSITE=1
+
+if [ -z "${PYTHONNOUSERSITE:-}" ]; then
+    export PYTHONNOUSERSITE=1
+fi
+
+# For performance, default to unbuffered; however, allow overriding via
+# PYTHONUNBUFFERED=0 since PYTHONUNBUFFERED on windows can truncate output
+# when printing long strings
+if [ -z "${PYTHONUNBUFFERED:-}" ]; then
+    export PYTHONUNBUFFERED=1
+fi
 
 # workaround for our python not shipping with certs
 if [[ -z ${SSL_CERT_DIR:-} ]]; then
     export SSL_CERT_DIR=/etc/ssl/certs/
 fi
 
-"${PM_PYTHON}" -u "$@"
+"${PM_PYTHON}" "$@"