From 3336e57ca2e1d53c526ad94d062cf218ae38a653 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marc=20M=C3=B6nner=C3=A5t?= Date: Mon, 7 Oct 2024 17:43:35 +0200 Subject: [PATCH] Install and clone script (#11) --------- Co-authored-by: Monnerat Marc swisstopo --- README.md | 117 ++++++------------------------------ clone_conda_environment.cmd | 41 ++++++++----- exclude.txt | 2 + install.bat | 19 +++++- toolbox/CHANGES.txt | 9 +++ toolbox/VERSION | 2 +- 6 files changed, 73 insertions(+), 117 deletions(-) create mode 100644 exclude.txt diff --git a/README.md b/README.md index 28a5880..493c589 100644 --- a/README.md +++ b/README.md @@ -9,18 +9,24 @@ Basic ESRI ArcMap's Toolbox to perform Trace Information Extraction (TIE) Analys ![TIE analysis](images/trace_analysis.png) -Installation ------------- +## Installation Basically, this is an ESRI ArcMap/ArcGis Pro ToolBox (Python2.7 and Python 3.9) running external scripts using `Python3`. -The `install.py` script is very `swisstopo`specific, it basically: +### Automatic installation (VDI Win10 TopGis) + +Use the `install.bat` script in _\\v0t0020a.adr.admin.ch\prod\lgX\TOPGIS\TIEToolbox_ + +It will create: + + * A new network drive T:\ pointing to _\\v0t0020a.adr.admin.ch\prod\lgX\TOPGIS\TIEToolbox_ + * A new _H:\.condarc_ file (proxy setting) + * A configuration file _H:\config\geocover\geocover.ini_ with the name of the conda environment to use. By default _TIE_ + * Copy the toolbox itself to D:\Argis_Home + + +### Manual install - * Create a `.condarc` in the `$HOME`directory ( _H:\_ ) - * Set up a `geocover.ini` file in the `$HOME\local\geocover` directory. Check the variables - `tie_conda_env` and `proxy`for corectness. - * Create a `conda` environment in `D:\conda\envs\TIE` - * Map the drive `T:` to a conda environment on the `LgX` Conda ----- @@ -35,111 +41,24 @@ There are several limitations: * On the VDI, you have C:\ and D:\ and may write almost everywhere. * Calling any `conda env` from within `ArcGis Pro` will pollute the _sys-path_ with the default environment variables. -Creating a conda env from scratch (takes about 2 hours). The command will create a new env -_D:\conda\envs_ +Creating a conda env from scratch (takes about 2-4 hours). The command will create a new env in _D:\conda\envs_ H:\code\bitbuckets\arcmap-tie-toolbox> setup_conda_environment.cmd TIE -Use the network env on the _LgX_ server, mapped on _T:\_ - - H:\> activate T:\TIE - -Clone the same environment (takes 30 minutes) - - H:\code\bitbuckets\arcmap-tie-toolbox> clone_conda_environment.cmd TIE - - --- **NOTE** As there are no `arcgis` and `arcpy` modules in this environnement, it's labelled as `broken`by ArcGis Pro. --- -Additionnally, you may try to speed up to subsequent install by using `mamba`: - - conda install mamba - mamba install ipycytoscape - -You may then `activate` your conda environnement: - - C:>argis-pro-py3> deactivate - - activate ANNA - -Install modules - -Linux: - - python -m pip install 'geocover-utils @ git+https://bitbucket.org/procrastinatio/geocover-utils.git@0.1.1' - -Windows: - python -m pip install --trusted bitbucket.org https://bitbucket.org/procrastinatio/geocover-utils/get/master.zip - - python -m pip install --trusted bitbucket.org https://bitbucket.org/procrastinatio/lg-tie-lib/get/packaging.zip - -TODO: add the `geocover-utils` and `traces-extraction` libraries - -Copy the `geocover.ini` configuration to `H:\local\config\geocover` - - - "C:\Program Files\ArcGIS\Pro\bin\Python\Scripts\conda.exe" create -p D:\conda\envs\EMPTY_FROM_YAML -v --override-channels -c conda-forge -y --show-channel-urls python=3.9.18 conda pip setuptools - -### Python2 - -Install `Python2` deopnedecies - - H:\> C:\Python27\ArcGIS10.8\Scripts\pip.exe install psutil - - H:\> C:\Python27\ArcGISx6410.8\Scripts\pip.exe install psutil - - -### Create a `conda` with mayavi and arcgis - -Create a bare mininal environment for python 3.9 - - H:\code\arcmap-tie-toolbox> "C:\Program Files\ArcGIS\Pro\bin\Python\Scripts\conda.exe" create -p C:\legacySW/EMPTY -v --override-channels -c conda-forge -y --show-channel-urls mamba python=3.9.18 - -Install the ESRI stuff - - H:\code\arcmap-tie-toolbox> "C:\Program Files\ArcGIS\Pro\bin\Python\Scripts\conda.exe" install -p C:\legacySW/EMPTY -v -c conda-forge -c esri -y --show-channel-urls arcgis arcpy - -Add the usefull stuff - - H:\code\arcmap-tie-toolbox> "C:\Program Files\ArcGIS\Pro\bin\Python\Scripts\conda.exe" install -p C:\legacySW/EMPTY -v -c conda-forge -c esri geocube geopandas matplotlib mayavi numpy rasterio scipy "shapely>=2.0.0" scikit-image dask - -Extract the package list (full source url) - - H:\code\arcmap-tie-toolbox> "C:\Program Files\ArcGIS\Pro\bin\Python\Scripts\conda.exe" list --explicit -p C:\legacySW\EMPTY >h:\EMPTY__list_export.yaml - -On the *VDI* recreate the environment with: - - "C:\Program Files\ArcGIS\Pro\bin\Python\Scripts\conda.exe" create -p D:\conda\envs\EMPTY --file EMPTY__list_export.yaml - - - - - - - -### Python3 - -Modules to install in a fresh `env` - - conda install geocube geopandas matplotlib mayavi numpy rasterio scipy "shapely>=2.0.0" scikit-image dask - -Update in an `ArcGis Pro` if cloning (in theory) - - conda install --name ARGISTIE --channel conda-forge geocube geopandas mayavi rasterio "shapely>=2.0.0" scikit-image ## Build - (c:\LegacySW\build-distribute) H:\code\arcmap-tie-toolbox>conda build --debug recipe -c esri -c conda-forge - - + (c:\LegacySW\build-distribute) H:\code\arcmap-tie-toolbox>conda build --debug recipe -c swisstopo -c conda-forge -## TIE +## Traces InterfacesExtraction (TIE) Using a `dask` pipeline @@ -187,7 +106,7 @@ Takes about 10-15 minutes for 5 km2 ![](images/3D_with_TIE_analysis.png "3D View and TIE analysis") [map.geo.admin.ch (3D)](https://s.geo.admin.ch/3k4ctm9jhzdn) - + Or add manually the following KML files: https://dubious.cloud/TIE/data/Kaiseregg/2.0/tie_as_multilines_under.kml https://dubious.cloud/TIE/data/Kaiseregg/2.0/tie_as_lines_under.kml diff --git a/clone_conda_environment.cmd b/clone_conda_environment.cmd index b83bdb4..7c36e23 100644 --- a/clone_conda_environment.cmd +++ b/clone_conda_environment.cmd @@ -1,16 +1,29 @@ -REM @echo off +@echo off REM Batch script to clone a conda environement from a local drive to an UNC share +REM +REM Usage: +REM clone_conda_environment.bat SRC_ENV_NAME DST_ENV_NAME +REM +REM with D:\conda\envs\%SRC_ENV_NAME% +REM T:\conda\envs\%DST_ENV_NAME% -SET "envname=%1" +SET "srcEnvName=%1" +SET "dstEnvName=%2" -IF not defined envname ( +IF not defined srcEnvName ( echo Please, define src_env EXIT /B 3 ) ELSE ( - echo src_env=%envname% + echo srcEnvName=%srcEnvName% +) +IF not defined dstEnvName ( + echo Please, define dstEnvname + EXIT /B 3 +) ELSE ( + echo destEnvName=%dstEnvName% ) set default_conda_exe="%ProgramFiles%\ArcGIS\Pro\bin\Python\Scripts\conda.exe" @@ -19,8 +32,8 @@ set PYTHON_VERSION=3.9.18 REM ... or for the Pool VDI ??? -set src_env=D:\conda\envs\%envname% -set dest_env=\\v0t0020a.adr.admin.ch\prod\lgX\TOPGIS\conda\envs\%envname% +set src_env=D:\conda\envs\%srcEnvName% +set dest_env=\\v0t0020a.adr.admin.ch\prod\lgX\TOPGIS\TIEToolbox\conda\envs\%dstEnvName% set conda_exe=%src_env%\condabin\conda.bat echo Target %dest_env% @@ -29,20 +42,20 @@ echo %time% === Cloning conda env "%src_env%" to %dest_env% === REM cleanup if not exist "c:\temp\" mkdir "c:\temp\" -del c:\temp\%envname%_conda_clone*.txt +del c:\temp\%srcEnvName%_conda_clone*.txt del %dest_env% -ECHO %time% === Cleaning env '%envname%' === +ECHO %time% === Cleaning env '%srcEnvName%' === -CALL %conda_exe% clean -p %src_env% --all --yes 1> c:\temp\%envname%_conda_clone_clean.txt ^ - 2> c:\temp\%envname%_conda_clone_clean_err.txt +CALL %conda_exe% clean -p %src_env% --all --yes 1> c:\temp\%srcEnvName%_conda_clone_clean.txt ^ + 2> c:\temp\%srcEnvName%_conda_clone_clean_err.txt -ECHO %time% === Cloning env '%envname%' from %src_env% === +ECHO %time% === Cloning env '%srcEnvName%' from %src_env% to %dest_env% === -CALL %default_conda_exe% create --clone %src_env% -p %dest_env% -vv --no-shortcuts ^ - --pinned 1> c:\temp\%envname%_conda_clone_clone.txt ^ - 2> c:\temp\%envname%_conda_clone_clone_err.txt +CALL %default_conda_exe% create --clone %src_env% -p %dest_env% --no-shortcuts ^ + --pinned 1> c:\temp\%srcEnvName%_conda_clone_clone.txt ^ + 2> c:\temp\%srcEnvName%_conda_clone_clone_err.txt ECHO %time% === Done === diff --git a/exclude.txt b/exclude.txt new file mode 100644 index 0000000..414937e --- /dev/null +++ b/exclude.txt @@ -0,0 +1,2 @@ +*.pyc +__pycache__\ \ No newline at end of file diff --git a/install.bat b/install.bat index bf197e7..27b2fac 100644 --- a/install.bat +++ b/install.bat @@ -1,12 +1,22 @@ @echo off + +SET "condaEnvName=%1" + +IF not defined condaEnvName ( + echo Using the default value for condaEnvName + set condaEnvName=TIE +) ELSE ( + echo condaEnvName=%condaEnvName% +) + REM Define the drive letter and the network path set driveLetter=T: set networkPath=\\v0t0020a.adr.admin.ch\prod\lgX\TOPGIS\TIEToolbox set dirPath=H:\config\geocover set condarcPath=H:\.condarc SET ARCGIS_HOME=D:\ArcGIS_Home -set condaEnvName=TIE + REM Check if the drive exists and remove it if it does if exist %driveLetter%\ ( @@ -61,8 +71,11 @@ echo projectdir=T:\conda\envs\%condaEnvName%\demo echo username=%userName% ) > %dirPath%\geocover.ini -endlocal ECHO %time% === Copying toolbox to %ARCGIS_HOME% === if not exist %ARCGIS_HOME% mkdir %ARCGIS_HOME% -xcopy T:\conda\envs\%condaEnvName%\Lib\site-packages\tietoolbox\esri\toolboxes "%ARCGIS_HOME%\tietoolbox\" /S /F /R /Y /I + +REM Copy Python files while excluding .pyc files and __pycache__ directories +xcopy T:\conda\envs\%condaEnvName%\Lib\site-packages\tietoolbox "%ARCGIS_HOME%\tietoolbox\" /S /E /Y /EXCLUDE:exclude.txt + +endlocal diff --git a/toolbox/CHANGES.txt b/toolbox/CHANGES.txt index 1e6936f..b39b197 100644 --- a/toolbox/CHANGES.txt +++ b/toolbox/CHANGES.txt @@ -33,3 +33,12 @@ Version 0.4.7 - using scripts created by `entry_points (in `\Scripts`) - fix small cleanup error +Version 0.4.8 +2024-10-06 +- Removing tests (no license for arcpy) +- Building on windows + +Version 0.4.9 +2024-10-07 +- Improved install and clone batch file +- Doc diff --git a/toolbox/VERSION b/toolbox/VERSION index cb498ab..76914dd 100644 --- a/toolbox/VERSION +++ b/toolbox/VERSION @@ -1 +1 @@ -0.4.8 +0.4.9