From d085b25cff0f589cddce846f4e43bef0004281d7 Mon Sep 17 00:00:00 2001 From: Farah Salhany Date: Thu, 3 Oct 2024 15:34:56 -0400 Subject: [PATCH 1/2] Updated README --- src/jupyterlabs/README.md | 55 +++++++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 20 deletions(-) diff --git a/src/jupyterlabs/README.md b/src/jupyterlabs/README.md index 6b9d08b..70dec96 100644 --- a/src/jupyterlabs/README.md +++ b/src/jupyterlabs/README.md @@ -1,31 +1,46 @@ -# README +# Jupyter Lab on HPC Cluster -This directory has example scripts to run Jupyter Labs+conda+python without containers +This folder contains scripts and instructions for setting up and running Jupyter Lab on the HPC cluster using Conda without container. These scripts are designed to simplify the process of allocating resources, setting up environments, and starting Jupyter Lab. - -## Preparation -After opening an interactive session (salloc) + +## Prerequisites +Before starting, ensure you have access to the HPC cluster. -Run the script `firsttime.sh`, ONLY 1 time, it will -- Create a /speed-scratch/$USER/Jupyter directory: change Jupyter to any name of your choice -- Set environment variables for tmp directories and for CONDA_PKGS_DIRS -- Create a conda environmen named: jupyter-env (change it to any name of your choice) -- Install JupyterLabs + pytorch -- exit the interactive session + +## Instructions +### For first time use, we need to -## Execution -After opening an interactive session (salloc) +* start an interactive session -Run the script `run.sh`, everytime you want to execute jupyterlabs + salloc --mem=20G --gpus=1 -## Browsing -After Execution, create an ssh tunnel (read https://nag-devops.github.io/speed-hpc/#jupyter-notebooks-in-singularity) +* load and initilize the environment -Open a browser: http://localhost:XXXX/lab?token=XXXXXXXXXX + module load anaconda/2023.03/default + conda init tcsh + source ~/.tcshrc -## REMINDER -Don't forget to setup the 2 scripts with the parameters of your choice -Jupyter: directory, jupiter-env: environment name +* run `setup-conda.sh` + + ./setup-conda.sh + + The script will: + - create a `/speed-scratch/$USER/Jupyter` directory (change Jupyter to any name of your choice) + - set environment variables for **TMP** directories and for **CONDA_PKGS_DIRS** + - create a conda environment named **jupyter-env** + - install JupyterLabs and pytorch + - exit the interactive session + +## Launching Jupeter +* Run the `start_jupyterlab.sh` script each time you need to launch Jupyterlab + + ./start_jupyterlab.sh + + The script will: + - allocate resources for your job + - start jupyter server by running `run_jupyterlab.sh` + - print the ssh command that you can use to connect to the compute node runnung the jupyter notebook (this is done in a new terminal) + - print the token/link to the jupyter server to paste in a web browser (starting with `http://127.0.0.1/...`) \ No newline at end of file From c5abfe0de2e91ebbeefd0263460823732000ec88 Mon Sep 17 00:00:00 2001 From: Farah Salhany Date: Thu, 3 Oct 2024 15:36:18 -0400 Subject: [PATCH 2/2] Deleted torchaudio and torchvision for faster preparation --- src/jupyterlabs/setup_conda.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/jupyterlabs/setup_conda.sh b/src/jupyterlabs/setup_conda.sh index f85dca5..8058f05 100755 --- a/src/jupyterlabs/setup_conda.sh +++ b/src/jupyterlabs/setup_conda.sh @@ -8,5 +8,5 @@ setenv CONDA_PKGS_DIRS /speed-scratch/$USER/Jupyter/pkgs conda create -p /speed-scratch/$USER/Jupyter/jupyter-env -y conda activate /speed-scratch/$USER/Jupyter/jupyter-env conda install -c conda-forge jupyterlab -y -pip3 install --quiet torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118 +pip3 install --quiet torch --index-url https://download.pytorch.org/whl/cu118 exit