From 80987dc4d218ecc2b7b6ed02bb730a54a913a501 Mon Sep 17 00:00:00 2001 From: Blake Caldwell <blake_caldwell@brown.edu> Date: Sun, 21 Mar 2021 05:30:05 -0400 Subject: [PATCH] MAINT: installation scripts with hnn-core --- installer/ubuntu/hnn-ubuntu.sh | 95 +++++++++++++++++++++---------- installer/windows/hnn-windows.ps1 | 51 ++++++++--------- 2 files changed, 89 insertions(+), 57 deletions(-) diff --git a/installer/ubuntu/hnn-ubuntu.sh b/installer/ubuntu/hnn-ubuntu.sh index 4af9990d8..46b1bb888 100755 --- a/installer/ubuntu/hnn-ubuntu.sh +++ b/installer/ubuntu/hnn-ubuntu.sh @@ -3,6 +3,7 @@ set -e [[ "$LOGFILE" ]] || LOGFILE="ubuntu_install.log" +[[ "$USE_CONDA" ]] || USE_CONDA=0 function start_download { echo "Downloading $2" @@ -61,17 +62,17 @@ echo "Using python version $PYTHON_VERSION" | tee -a "$LOGFILE" export DEBIAN_FRONTEND=noninteractive echo "Updating package repository..." | tee -a "$LOGFILE" -sudo -E apt-get update &>> "$LOGFILE" +sudo -E apt-get update &> "$LOGFILE" echo "Updating OS python packages..." | tee -a "$LOGFILE" if [[ "${PYTHON_VERSION}" =~ "3.7" ]] && [[ "$DISTRIB" =~ "bionic" ]]; then - sudo -E apt-get install --no-install-recommends -y python3.7 python3-pip python3.7-tk python3.7-dev &>> "$LOGFILE" && \ - sudo python3.7 -m pip install --upgrade pip setuptools &>> "$LOGFILE" + sudo -E apt-get install --no-install-recommends -y python3.7 python3-pip python3-tk python3.7-dev &> "$LOGFILE" && \ + sudo python3.7 -m pip install --upgrade pip setuptools &> "$LOGFILE" sudo ln -s /usr/lib/python3/dist-packages/apt_pkg.cpython-36m-x86_64-linux-gnu.so \ /usr/lib/python3/dist-packages/apt_pkg.so else - sudo -E apt-get install --no-install-recommends -y python3 python3-pip python3-tk python3-setuptools &>> "$LOGFILE" && \ - sudo pip3 install --upgrade pip &>> "$LOGFILE" + sudo -E apt-get install --no-install-recommends -y python3 python3-pip python3-tk python3-setuptools &> "$LOGFILE" && \ + sudo pip3 install --upgrade pip &> "$LOGFILE" fi if which python3 &> /dev/null; then @@ -87,22 +88,68 @@ elif which python &> /dev/null; then fi echo "Using python: $PYTHON with pip: $PIP" | tee -a "$LOGFILE" +if [[ "$USE_CONDA" -eq 0 ]]; then + echo "Downloading python packages for HNN with pip..." | tee -a "$LOGFILE" + $PIP download pyqt5 nlopt psutil hnn-core &> "$LOGFILE" & + PIP_PID=$! +fi + echo "Installing OS compilation toolchain..." | tee -a "$LOGFILE" # get prerequisites from pip. requires gcc to build psutil sudo -E apt-get install --no-install-recommends -y \ - make gcc g++ python3-dev &>> "$LOGFILE" + make gcc g++ python3-dev &> "$LOGFILE" + +if [[ "$USE_CONDA" -eq 0 ]]; then + echo "Waiting for python packages for HNN downloads to finish..." + NAME="downloading python packages for HNN " + wait_for_pid "${PIP_PID}" "$NAME" + + # install hnn-core and prerequisites (e.g. NEURON) + echo "Installing python prequisites for HNN with pip..." | tee -a "$LOGFILE" + $PIP install --no-cache-dir pyqt5 nlopt psutil hnn-core &> "$LOGFILE" + + # WSL may not have nrnivmodl in PATH + if ! which nrnivmodl &> /dev/null; then + export PATH="$PATH:$HOME/.local/bin" + echo 'export PATH="$PATH:$HOME/.local/bin"' >> ~/.bashrc + fi +else + URL="https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh" + FILENAME="$HOME/miniconda.sh" + start_download "$FILENAME" "$URL" + + echo "Installing miniconda..." + chmod +x "$HOME/miniconda.sh" + "$HOME/miniconda.sh" -b -p "${HOME}/Miniconda3" + export PATH=${HOME}/Miniconda3/bin:$PATH -$PIP install --no-cache-dir NEURON + # create conda environment + conda env create -f environment.yml -# WSL may not have nrnivmodl in PATH -if ! which nrnivmodl &> /dev/null; then - export PATH="$PATH:$HOME/.local/bin" - echo 'export PATH="$PATH:$HOME/.local/bin"' >> ~/.bashrc + # conda is faster to install nlopt + conda install -y -n hnn -c conda-forge nlopt + + source activate hnn && echo "activated conda HNN environment" + + echo "Installing MPI compilation toolchain..." | tee -a "$LOGFILE" + # get prerequisites to build mpi4py + sudo -E apt-get install --no-install-recommends -y \ + libopenmpi-dev &> "$LOGFILE" + + # install hnn-core and prerequisites (e.g. NEURON) + pip install mpi4py pyqt5 hnn-core fi -echo "Installing python packages for HNN with pip..." | tee -a "$LOGFILE" -$PIP install --no-cache-dir --user matplotlib PyOpenGL \ - pyqt5 pyqtgraph scipy numpy nlopt psutil &>> "$LOGFILE" +echo "Downloading runtime prerequisite packages..." | tee -a "$LOGFILE" +apt-get download \ + openmpi-bin lsof libfontconfig1 libxext6 libx11-xcb1 libxcb-glx0 \ + libxkbcommon-x11-0 libgl1-mesa-glx \ + libc6-dev libtinfo-dev libncurses5-dev \ + libx11-dev libreadline-dev \ + libxcb-icccm4 libxcb-util1 libxcb-image0 libxcb-keysyms1 \ + libxcb-render0 libxcb-shape0 libxcb-randr0 libxcb-render-util0 \ + libxcb-xinerama0 &> "$LOGFILE" & +APT_DOWNLOAD=$! # save dir installing hnn to startdir=$(pwd) @@ -115,11 +162,11 @@ if [[ $TRAVIS_TESTING -ne 1 ]]; then cd hnn_source_code if [ -d "$source_code_dir/.git" ]; then - git pull origin master &>> "$LOGFILE" + git pull origin master &> "$LOGFILE" fi else echo "Downloading and extracting HNN..." | tee -a "$LOGFILE" - wget --no-hsts --no-check-certificate -O hnn.tar.gz https://github.com/jonescompneurolab/hnn/releases/latest/download/hnn.tar.gz | tee -a "$LOGFILE" + wget -O hnn.tar.gz https://github.com/jonescompneurolab/hnn/releases/latest/download/hnn.tar.gz | tee -a "$LOGFILE" mkdir hnn_source_code tar -x --strip-components 1 -f hnn.tar.gz -C hnn_source_code &>> "$LOGFILE" && \ cd hnn_source_code &>> "$LOGFILE" @@ -132,10 +179,6 @@ else fi fi -echo "Building HNN..." | tee -a "$LOGFILE" -make -j4 &>> "$LOGFILE" -MAKE_PID=$! - # create the global session variables echo '# these lines define global session variables for HNN' >> ~/.bashrc echo "export PATH=\$PATH:\"$source_code_dir\"" >> ~/.bashrc @@ -147,7 +190,7 @@ if [[ -d "$HOME/Desktop" ]]; then cp -f hnn.desktop "$HOME/Desktop" && \ sed -i "s~/home/hnn_user\(.*\)$~\"$startdir\"\1~g" "$HOME/Desktop/hnn.desktop" && \ chmod +x "$HOME/Desktop/hnn.desktop" - } &>> "$LOGFILE" + } &> "$LOGFILE" fi echo "Installing prerequisites..." | tee -a "$LOGFILE" @@ -160,17 +203,11 @@ sudo -E apt-get install --no-install-recommends -y \ libx11-dev libreadline-dev \ libxcb-icccm4 libxcb-util1 libxcb-image0 libxcb-keysyms1 \ libxcb-render0 libxcb-shape0 libxcb-randr0 libxcb-render-util0 \ - libxcb-xinerama0 libxcb-xfixes0 &>> "$LOGFILE" + libxcb-xinerama0 libxcb-xfixes0 &> "$LOGFILE" # Clean up a little echo "Cleaning up..." | tee -a "$LOGFILE" -sudo -E apt-get clean &>> "$LOGFILE" - -if [[ $TRAVIS_TESTING -ne 1 ]]; then - echo "Waiting for HNN module build to finish..." - NAME="building HNN modules" - wait_for_pid "${MAKE_PID}" "$NAME" -fi +sudo -E apt-get clean &> "$LOGFILE" echo "HNN installation successful" | tee -a "$LOGFILE" echo "Source code is at $source_code_dir" | tee -a "$LOGFILE" diff --git a/installer/windows/hnn-windows.ps1 b/installer/windows/hnn-windows.ps1 index ccb092abf..c8759ffe8 100644 --- a/installer/windows/hnn-windows.ps1 +++ b/installer/windows/hnn-windows.ps1 @@ -6,7 +6,7 @@ install Miniconda: - Miniconda3-latest-Windows-x86_64.exe Additionally the following will be installed if they are not found: - - nrn-7.7.w64-mingwsetup.exe + - nrn-7.8.w64-mingwsetup.exe Other requirements: - Only 64-bit installs are supported due to NEURON compatibility @@ -310,8 +310,8 @@ if ($script:installMiniconda) { $program = "NEURON" if (!(Test-Installed($program))) { - $file = "nrn-7.7.w64-mingwsetup.exe" - $url = "https://neuron.yale.edu/ftp/neuron/versions/v7.7/$file" + $file = "nrn-7.8.w64-mingwsetup.exe" + $url = "https://neuron.yale.edu/ftp/neuron/versions/v7.8/$file" Download-Program $program $file $url $dirpath = $script:NEURON_PATH Write-Host "Installing $program to $dirpath..." @@ -379,6 +379,13 @@ if ($proc1) { Write-Host "Miniconda is finished" } +if ($proc2) { + Write-Host "Waiting for NEURON install to finish..." + $proc2.WaitForExit() 2>$null + Update-User-Paths("$script:NEURON_PATH\bin") + Write-Host "NEURON is finished" +} + # setup python with virtualenv or 'conda if ($null -ne $script:VIRTUALENV) { Write-Host "Creating Python virtualenv at $HOME\venv\hnn..." @@ -390,7 +397,9 @@ if ($null -ne $script:VIRTUALENV) { $script:PYTHON = "$HOME\venv\hnn\Scripts\python.exe" if (Test-Python-3($script:PYTHON)) { # use pip3 for good measure - Start-Process "$HOME\venv\hnn\Scripts\pip3" "install PyOpenGL pyqtgraph matplotlib scipy PyQt5 psutil nlopt" -Wait + Start-Process "$HOME\venv\hnn\Scripts\pip3" "install matplotlib scipy PyQt5 psutil nlopt" -Wait + # get hnn-core, but skip NEURON dependency + Start-Process "$HOME\venv\hnn\Scripts\pip3" "install --no-deps mpi4py hnn-core" -Wait } else { Write-Warning "Virtualenv failed to create a valid python3 environment" @@ -415,14 +424,21 @@ elseif ($null -ne $script:CONDA_PATH) { if (!$script:env_exists) { Write-Host "Setting up anaconda hnn environment..." - conda create -y -n hnn python=3.7 PyOpenGL pyqtgraph matplotlib scipy conda psutil + conda env create -f environment.yml conda install -y -n hnn -c conda-forge nlopt + + # need to call the right pip to install in miniconda environment + # get hnn-core, but skip NEURON dependency + Set-Location $HOME + Miniconda3\envs\hnn\Scripts\pip install --no-deps mpi4py hnn-core + Set-Location $CONDA_ENV mkdir .\etc\conda\activate.d 2>&1>$null mkdir .\etc\conda\deactivate.d 2>&1>$null - #"set NRN_PYLIB=$script:PYTHON_DLL" | Set-Content "$CONDA_ENV\etc\conda\activate.d\env_vars.bat" - "set PYTHONHOME=$CONDA_ENV" | Add-Content "$CONDA_ENV\etc\conda\activate.d\env_vars.bat" + # "set NRN_PYLIB=$script:PYTHON_DLL" | Set-Content "$CONDA_ENV\etc\conda\activate.d\env_vars.bat" + "set PYTHONPATH=$script:NEURON_PATH\lib\python" | Add-Content "$CONDA_ENV\etc\conda\activate.d\env_vars.bat" + "export PYTHONPATH=/c/nrn/lib/python" | Add-Content "$CONDA_ENV\etc\conda\activate.d\env_vars.sh" } else { Write-Host "Miniconda hnn environment already exists" @@ -433,27 +449,6 @@ else { } - -if ($proc2) { - Write-Host "Waiting for NEURON install to finish..." - $proc2.WaitForExit() 2>$null - Update-User-Paths("$script:NEURON_PATH\bin") - Write-Host "NEURON is finished" -} - -if (!(Test-Path "$HNN_PATH\nrnmech.dll" -PathType Leaf)) { - Write-Host "Creating nrnmech.dll" - Set-Location $HNN_PATH\mod - Start-Process "$script:NEURON_PATH\mingw\usr\bin\sh.exe" "$script:NEURON_ESC_PATH/lib/mknrndll.sh C:\nrn\" - $obj = New-Object -com Wscript.Shell - sleep -s 10 - $obj.SendKeys("{ENTER}") - Copy-Item $HNN_PATH\mod\nrnmech.dll -Destination $HNN_PATH -} -else { - Write-Host "nrnmech.dll already exists $HNN_PATH\nrnmech.dll" -} - Write-Host "" Write-Host "Finished installing HNN and prerequisites." Write-Host "Activate the environment from cmd.exe (not Powershell):"