From b5cdf8a2a4b86ee2717ea6c0b157c2dc1d8dc157 Mon Sep 17 00:00:00 2001 From: Rafael Sarmiento Date: Mon, 18 Nov 2024 15:54:24 +0100 Subject: [PATCH 01/24] add JupyterHub demo --- examples/jupyterhub/Dockerfile | 36 +++ examples/jupyterhub/README.md | 95 +++++++ examples/jupyterhub/docker-compose.yml | 317 +++++++++++++++++++++++ examples/jupyterhub/env.sh | 4 + examples/jupyterhub/jhub-client.json | 73 ++++++ examples/jupyterhub/jupyterhub-config.py | 68 +++++ 6 files changed, 593 insertions(+) create mode 100644 examples/jupyterhub/Dockerfile create mode 100644 examples/jupyterhub/README.md create mode 100644 examples/jupyterhub/docker-compose.yml create mode 100644 examples/jupyterhub/env.sh create mode 100644 examples/jupyterhub/jhub-client.json create mode 100644 examples/jupyterhub/jupyterhub-config.py diff --git a/examples/jupyterhub/Dockerfile b/examples/jupyterhub/Dockerfile new file mode 100644 index 00000000..a1c132cd --- /dev/null +++ b/examples/jupyterhub/Dockerfile @@ -0,0 +1,36 @@ +FROM f7t-cluster + +RUN yum update -y && \ + yum install -y sqlite-devel openssl-devel bzip2-devel libffi-devel wget && \ + yum groupinstall -y "Development Tools" + +RUN wget https://github.com/openssl/openssl/releases/download/OpenSSL_1_1_1/openssl-1.1.1.tar.gz && \ + tar -xzvf openssl-1.1.1.tar.gz && \ + cd openssl-1.1.1 && \ + ./config --prefix=/usr/openssl1.1 --openssldir=/etc/ssl --libdir=lib no-shared zlib-dynamic && \ + make && \ + make install && \ + rm -r ../openssl-1.1.1.tar.gz ../openssl-1.1.1 + +RUN wget https://www.python.org/ftp/python/3.10.2/Python-3.10.2.tgz && \ + tar -xzf Python-3.10.2.tgz && \ + cd Python-3.10.2 && \ + export TCLTK_LIBS='-ltk8.5 -ltcl8.5' && \ + ./configure --enable-shared --with-openssl=/usr/openssl1.1 --with-openssl-rpath=auto --enable-optimizations && \ + make altinstall + +ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH + +RUN python3.10 -m venv /opt/jhub-env + +RUN . /opt/jhub-env/bin/activate && \ + python3.10 -m pip install --no-cache jupyterlab jupyterhub==4.1.6 pyfirecrest==2.1.0 SQLAlchemy==1.4.52 oauthenticator==16.3.1 notebook==7.2.1 + +RUN . /opt/jhub-env/bin/activate && \ + git clone https://github.com/eth-cscs/firecrestspawner.git && \ + cd firecrestspawner && \ + pip install --no-cache . && \ + cd .. && \ + rm -r firecrestspawner + +RUN localedef -i en_US -f UTF-8 en_US.UTF-8 diff --git a/examples/jupyterhub/README.md b/examples/jupyterhub/README.md new file mode 100644 index 00000000..a944ec31 --- /dev/null +++ b/examples/jupyterhub/README.md @@ -0,0 +1,95 @@ +# FirecRESTSpawner on the Docker demo + + +This is a tutorial on how to run JupyterHub with [FirecRESTSpawner](https://github.com/eth-cscs/firecrestspawner) on the [Docker demo of FirecREST](https://github.com/eth-cscs/firecrest/tree/master/deploy/demo). + +We are going to start by deploying FirecREST together with a slurm cluster using [Docker Compose](https://docs.docker.com/compose). Then we will install JupyterHub on a virtual environment and configure it to launch notebooks on the slurm cluster via FirecREST. + + +## Requirements + +For this tutorial it's necessary + * a recent installation of Docker, which includes the `docker compose` command. Alternatively the old `docker-compose` command line could be used. + * a python installation (`>=3.9`) + + +## Setup + + +### Building images from FirecREST's Docker Compose demo + +The `docker-compose.yaml` that we use in this the demo is a copy of the the one used in the Docker demo of FirecREST with only a few small changes. So, to get started, let's clone the FirecREST repository + +```bash +git clone https://github.com/eth-cscs/firecrest.git +cd firecrest/deploy/demo/ +``` + +and build the images used in it's [Docker Compose demo](https://github.com/eth-cscs/firecrest/tree/master/deploy/demo): + +```bash +docker compose build +``` + +This step is going to last a few minutes. In the meanwhile we can install JupyterHub on a virtual environment on our machine. + + +### Install JupyterHub and FirecRESTSpawner + +We can create a virtual environment + +```bash +python -m venv jhub-env +``` + +and install JupyterHub and FirecRESTSpawner + +```bash +. jhub-env/bin/activate + +pip install --no-cache jupyterhub==4.1.6 pyfirecrest==2.6.0 SQLAlchemy==1.4.52 oauthenticator==16.3.1 python-hostlist==1.23.0 + +git clone https://github.com/eth-cscs/firecrestspawner.git +cd firecrestspawner +. jhub-env/bin/activate +pip install --no-cache . +``` + +## Deployment of the FirecREST and Slurm cluster + +Once all the images have been built we can move to the JupyterHub demo directory and start deploy the [`docker-compose.yaml`](docker-compose.yaml). + +```bash +cd firecrest/examples/jupyterhub +docker compose up -d --build +``` + +The deployment can be shutdown with + +``` +cd firecrest/examples/jupyterhub +docker compose down +``` + +### Setting up the authorization + +To login on JupyterHub and authenticate with FirecREST in the spawner, we need to create an Authorization Code Flow client in the `kcrealm` in Keycloak. In [this page](http://localhost:8080/auth/admin/master/console/#/realms/kcrealm/clients) (username: admin, password: admin2), click on "Create" and then on "Select file" choose the [`jhub-client.json`](jhub-client.json) file from the file system. + +Once that's done, the client `jhub-client` can be seen listed on the "Clients" tab. + + +### Launching JupyterHub + +The configuration file provided [here](jupyterhub-config.py) has all the setting to use JupyterHub with the our deployment, but the secret for the client we just created must be added on `c.Authenticator.client_secret`. The secret can be found in the client's ["Credentials" tab](http://localhost:8080/auth/admin/master/console/#/realms/kcrealm/clients/f969b69d-4aec-4646-bdbe-09a268f52111/credentials). + +JupyterHub can be run with + +```bash +. jhub-env/bin/activate +. env.sh +jupyterhub --config jupyterhub-config.py --port 8003 --ip 0.0.0.0 --debug +``` +Here we are sourcing the file [`env.sh`](env.sh) which defines environment variables needed by the spawner. +We use the port `8003` for the hub since `8000` is already used for FirecREST itself in the demo deployment. The ip `0.0.0.0` is necessary to allow JupyterLab to connect back to the hub. + +The hub should be accessible in the browser at [http://localhost:8003](http://localhost:8003/) (username: test1 and password: test11) and you should be able to submit notebook jobs. diff --git a/examples/jupyterhub/docker-compose.yml b/examples/jupyterhub/docker-compose.yml new file mode 100644 index 00000000..8b9bd75f --- /dev/null +++ b/examples/jupyterhub/docker-compose.yml @@ -0,0 +1,317 @@ +## +## Copyright (c) 2019-2024, ETH Zurich. All rights reserved. +## +## Please, refer to the LICENSE file in the root directory. +## SPDX-License-Identifier: BSD-3-Clause +## +version: '3.4' + +networks: + firecrest-internal: + driver: bridge + ipam: + driver: default + config: + - subnet: 192.168.220.0/24 + + +services: + f7t-base: + image: f7t-base + container_name: f7t-base + build: + context: ../../ + network: host + # FirecREST microservices + certificator: + image: f7t-certificator + container_name: certificator + depends_on: + - f7t-base + build: + context: ../../ + network: host + env_file: + - ./common/common.env + environment: + - F7T_CA_KEY_PATH=/etc/firecrest/ca-key + - F7T_PUB_USER_KEY_PATH=/etc/firecrest/user-key.pub + networks: + firecrest-internal: + ipv4_address: 192.168.220.11 + ports: + - 5010:5010 + volumes: + - ./logs/firecrest:/var/log:delegated + - ../test-build/environment/keys/user-key.pub:/etc/firecrest/user-key.pub:ro + - ../test-build/environment/keys/ca-key:/etc/firecrest/ca-key:ro + - ./ssl:/ssl + - ./logs/profs:/var/log/profs + + compute: + image: f7t-compute + container_name: compute + depends_on: + - f7t-base + build: + context: ../../ + network: host + environment: + - F7T_PRIV_USER_KEY_PATH=/etc/firecrest/ssh/user-key + env_file: + - ./common/common.env + networks: + firecrest-internal: + ipv4_address: 192.168.220.9 + ports: + - 5006:5006 + volumes: + - ./logs/firecrest:/var/log:delegated + - ../test-build/environment/keys/user-key:/etc/firecrest/ssh/user-key:ro + - ./ssl:/ssl + - ./logs/profs:/var/log/profs + + status: + image: f7t-status + container_name: status + depends_on: + - f7t-base + build: + context: ../../ + network: host + args: + GENERAL_INFORMATION: '{"FIRECREST_VERSION": "demo"}' + env_file: + - ./common/common.env + networks: + firecrest-internal: + ipv4_address: 192.168.220.4 + ports: + - 5001:5001 + volumes: + - ./logs/firecrest:/var/log:delegated + - ./ssl:/ssl + + storage: + image: f7t-storage + container_name: storage + depends_on: + - f7t-base + build: + context: ../../ + network: host + env_file: + - ./common/common.env + environment: + F7T_S3_PRIVATE_URL: http://192.168.220.19:9000 + F7T_S3_PUBLIC_URL: http://192.168.220.19:9000 + F7T_S3_ACCESS_KEY: storage_access_key + F7T_S3_SECRET_KEY: storage_secret_key + F7T_S3_REGION: us-east-1 + F7T_S3_TENANT: null + F7T_STORAGE_POLLING_INTERVAL: 60 + F7T_CERT_CIPHER_KEY: 'Df6UZuoPoJ2u5yRwxNfFQ46Nwy8eW1OGTcuhlqn4ONo=' + networks: + firecrest-internal: + ipv4_address: 192.168.220.5 + volumes: + - ./logs/firecrest:/var/log:delegated + - ../test-build/environment/keys/user-key:/user-key:ro + - ./ssl:/ssl + - ./logs/profs:/var/log/profs + + tasks: + image: f7t-tasks + container_name: tasks + depends_on: + - f7t-base + - "taskpersistence" + build: + context: ../../ + network: host + env_file: + - ./common/common.env + environment: + - F7T_PERSIST_PORT=6379 + - F7T_PERSIST_PWD=rediS2200 + - F7T_COMPUTE_TASK_EXP_TIME=86400 + - F7T_STORAGE_TASK_EXP_TIME=2678400 + networks: + firecrest-internal: + ipv4_address: 192.168.220.6 + ports: + - 5003:5003 + volumes: + - ./logs/firecrest:/var/log:delegated + - ./ssl:/ssl + - ./logs/profs:/var/log/profs + + utilities: + image: f7t-utilities + container_name: utilities + depends_on: + - f7t-base + build: + context: ../../ + network: host + env_file: + - ./common/common.env + command: + [ + "sh", + "-c", + "pip install debugpy && python3 -Xfrozen_modules=off -m debugpy --listen 0.0.0.0:5678 -m gunicorn --bind :$${F7T_UTILITIES_PORT} $${F7T_GUNICORN_LOG} $${F7T_GUNICORN_SSL} $${F7T_GUNICORN_WORKER} utilities:app", + ] + networks: + firecrest-internal: + ipv4_address: 192.168.220.7 + ports: + - 5004:5004 + - 5678:5678 + volumes: + - ./logs/firecrest:/var/log:delegated + - ../test-build/environment/keys/user-key:/user-key:ro + - ./ssl:/ssl + - ./logs/profs:/var/log/profs + + reservations: + image: f7t-reservations + container_name: reservations + depends_on: + - f7t-base + build: + context: ../../ + network: host + env_file: + - ./common/common.env + networks: + firecrest-internal: + ipv4_address: 192.168.220.8 + ports: + - 5005:5005 + volumes: + - ./logs/firecrest:/var/log:delegated + - ../test-build/environment/keys/user-key:/user-key:ro + - ./ssl:/ssl + - ./logs/profs:/var/log/profs + + # dummy cluster + cluster: + image: f7t-cluster:jhub-amd64 + container_name: cluster + build: + context: ../test-build + dockerfile: Dockerfile + network: host + networks: + firecrest-internal: + ipv4_address: 192.168.220.12 + hostname: cluster + ports: + - 56123:56123 + volumes: + - ./logs/cluster/:/var/log/slurm/:delegated + + # complementary 3rd party services + keycloak: + image: quay.io/keycloak/keycloak:18.0.2 + container_name: fckeycloak + env_file: keycloak/keycloak.env + command: start-dev --http-relative-path /auth --import-realm + + networks: + firecrest-internal: + ipv4_address: 192.168.220.20 + ports: + - "${KEYCLOAK_PORT:-8080}:8080" + volumes: + - ./keycloak/config.json:/opt/keycloak/data/import/config.json:ro + - ./logs/keycloak:/opt/jboss/keycloak/standalone/log/:delegated + + kong: + image: kong:3.2 + container_name: kong + environment: + - KONG_DATABASE=off + - KONG_DECLARATIVE_CONFIG=/kong.yml + - KONG_LOG_LEVEL=error + volumes: + - ./kong/kong.yml:/kong.yml:ro + networks: + firecrest-internal: + ipv4_address: 192.168.220.21 + ports: + - 8000:8000 + + minio: + image: minio/minio:RELEASE.2022-10-24T18-35-07Z + container_name: minio + command: minio server /data --console-address ":9001" + environment: + MINIO_ROOT_USER: storage_access_key + MINIO_ROOT_PASSWORD: storage_secret_key + networks: + firecrest-internal: + ipv4_address: 192.168.220.19 + ports: + - "9000:9000" + - "9001:9001" + volumes: + - ./minio:/data:delegated + + taskpersistence: + container_name: taskpersistence + image: redis:5 + command: redis-server /redis.conf + networks: + firecrest-internal: + ipv4_address: 192.168.220.13 + ports: + - 6379:6379 + volumes: + - ./taskpersistence/redis.conf:/redis.conf:ro + - ./taskpersistence-data:/data:delegated + - ./logs:/var/log:delegated + + opa: + image: openpolicyagent/opa:0.22.0 + container_name: opa + command: run --server --log-level=debug --log-format=json-pretty --skip-version-check --tls-cert-file=/ssl/f7t_internal.crt --tls-private-key-file=/ssl/f7t_internal.key /opa-files/data.json /opa-files/policy.rego + networks: + firecrest-internal: + ipv4_address: 192.168.220.40 + ports: + - "8181:8181" + volumes: + - ./opa:/opa-files + - ./ssl:/ssl + + openapi: + image: swaggerapi/swagger-ui:v4.19.1 + container_name: openapi + networks: + firecrest-internal: + ipv4_address: 192.168.220.41 + environment: + SWAGGER_JSON: /tmp/firecrest-api.yaml + BASE_URL: /docs + volumes: + - ../../doc/openapi/:/tmp/ + + jaeger: + image: jaegertracing/all-in-one:1.24 + container_name: jaeger + networks: + firecrest-internal: + ipv4_address: 192.168.220.50 + environment: + COLLECTOR_ZIPKIN_HOST_PORT: "9411" + ports: + #- 5775:5775/udp + - 6831:6831/udp + - 6832:6832/udp + - 5778:5778 + - 16686:16686 + - 14268:14268 + - 14250:14250 + - 9411:9411 diff --git a/examples/jupyterhub/env.sh b/examples/jupyterhub/env.sh new file mode 100644 index 00000000..cc4d0373 --- /dev/null +++ b/examples/jupyterhub/env.sh @@ -0,0 +1,4 @@ +export FIRECREST_URL=http://localhost:8000 +export SA_CLIENT_ID=firecrest-sample +export SA_CLIENT_SECRET=b391e177-fa50-4987-beaf-e6d33ca93571 +export SA_AUTH_TOKEN_URL=http://localhost:8080/auth/realms/kcrealm/protocol/openid-connect/token diff --git a/examples/jupyterhub/jhub-client.json b/examples/jupyterhub/jhub-client.json new file mode 100644 index 00000000..990ff298 --- /dev/null +++ b/examples/jupyterhub/jhub-client.json @@ -0,0 +1,73 @@ +{ + "clientId": "jhub-client", + "name": "jhub-client", + "surrogateAuthRequired": false, + "enabled": true, + "alwaysDisplayInConsole": false, + "clientAuthenticatorType": "client-secret", + "redirectUris": [ + "http://localhost:8003/hub/oauth_callback" + ], + "webOrigins": [], + "notBefore": 0, + "bearerOnly": false, + "consentRequired": false, + "standardFlowEnabled": true, + "implicitFlowEnabled": false, + "directAccessGrantsEnabled": false, + "serviceAccountsEnabled": false, + "publicClient": false, + "frontchannelLogout": false, + "protocol": "openid-connect", + "attributes": { + "saml.force.post.binding": "false", + "saml.multivalued.roles": "false", + "frontchannel.logout.session.required": "false", + "oauth2.device.authorization.grant.enabled": "false", + "backchannel.logout.revoke.offline.tokens": "false", + "saml.server.signature.keyinfo.ext": "false", + "use.refresh.tokens": "true", + "oidc.ciba.grant.enabled": "false", + "backchannel.logout.session.required": "true", + "client_credentials.use_refresh_token": "false", + "require.pushed.authorization.requests": "false", + "saml.client.signature": "false", + "saml.allow.ecp.flow": "false", + "id.token.as.detached.signature": "false", + "saml.assertion.signature": "false", + "client.secret.creation.time": "1731487602", + "saml.encrypt": "false", + "saml.server.signature": "false", + "exclude.session.state.from.auth.response": "false", + "saml.artifact.binding": "false", + "saml_force_name_id_format": "false", + "acr.loa.map": "{}", + "tls.client.certificate.bound.access.tokens": "false", + "saml.authnstatement": "false", + "display.on.consent.screen": "false", + "token.response.type.bearer.lower-case": "false", + "saml.onetimeuse.condition": "false" + }, + "authenticationFlowBindingOverrides": {}, + "fullScopeAllowed": true, + "nodeReRegistrationTimeout": -1, + "defaultClientScopes": [ + "web-origins", + "acr", + "firecrest", + "roles", + "profile", + "email" + ], + "optionalClientScopes": [ + "address", + "phone", + "offline_access", + "microprofile-jwt" + ], + "access": { + "view": true, + "configure": true, + "manage": true + } +} \ No newline at end of file diff --git a/examples/jupyterhub/jupyterhub-config.py b/examples/jupyterhub/jupyterhub-config.py new file mode 100644 index 00000000..0039ce38 --- /dev/null +++ b/examples/jupyterhub/jupyterhub-config.py @@ -0,0 +1,68 @@ +import secrets +from oauthenticator.generic import GenericOAuthenticator + + +def gen_hex_string(num_bytes=32, num_hex_strings=4): + """Generate keys to encode the auth state""" + hex_strings = [secrets.token_hex(num_bytes) + for i in range(num_hex_strings)] + return hex_strings + + +c = get_config() + +c.JupyterHub.authenticator_class = GenericOAuthenticator + +# Keycloak setup +c.Authenticator.client_id = "jhub-client" +c.Authenticator.client_secret = "" +c.Authenticator.oauth_callback_url = "http://localhost:8003/hub/oauth_callback" +c.Authenticator.authorize_url = "http://localhost:8080/auth/realms/kcrealm/protocol/openid-connect/auth" +c.Authenticator.token_url = "http://localhost:8080/auth/realms/kcrealm/protocol/openid-connect/token" +c.Authenticator.userdata_url = "http://localhost:8080/auth/realms/kcrealm/protocol/openid-connect/userinfo" +c.Authenticator.login_service = "http://localhost:8080" +c.Authenticator.username_key = "preferred_username" +c.Authenticator.userdata_params = {"state": "state"} +c.Authenticator.scope = ["openid", "profile", "firecrest"] + +# Hub access +c.Authenticator.admin_users = {"test1" } +c.Authenticator.allow_all = True + +# Auth state enabled +c.Authenticator.enable_auth_state = True +c.CryptKeeper.keys = gen_hex_string() + +c.JupyterHub.default_url = "/hub/home" + +# Spawner setup +c.JupyterHub.spawner_class = "firecrestspawner.spawner.SlurmSpawner" +c.Spawner.req_host = "cluster" +c.Spawner.cmd = "firecrestspawner-singleuser jupyterhub-singleuser" +c.Spawner.enable_aux_fc_client = True +c.Spawner.node_name_template = "localhost" +c.Spawner.port = 56123 +c.Spawner.batch_script = """#!/bin/bash +#SBATCH --job-name=jhub + +export JUPYTERHUB_API_URL="http://host.docker.internal:8003/hub/api" +export JUPYTERHUB_ACTIVITY_URL="http://host.docker.internal:8003//hub/api/users/${USER}/activity" + +export JUPYTERHUB_OAUTH_ACCESS_SCOPES=$(echo $JUPYTERHUB_OAUTH_ACCESS_SCOPES | base64 --decode) +export JUPYTERHUB_OAUTH_SCOPES=$(echo $JUPYTERHUB_OAUTH_SCOPES | base64 --decode) + +export JUPYTERHUB_CRYPT_KEY=$(/usr/openssl1.1/bin/openssl rand -hex 32) + +export PATH=/usr/openssl1.1/bin:$PATH +export LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH + +set -euo pipefail + +. /opt/jhub-env/bin/activate + +trap 'echo SIGTERM received' TERM +# {{prologue}} +{% if srun %}{{srun}}{% endif %} {{cmd}} +echo "jupyterhub-singleuser ended gracefully" +# {{epilogue}} +""" From 82e516b5cdec583d1ec74f532bb9f680ef3ed44e Mon Sep 17 00:00:00 2001 From: Rafael Sarmiento Date: Mon, 18 Nov 2024 16:21:39 +0100 Subject: [PATCH 02/24] add more info --- examples/jupyterhub/README.md | 47 +++++++++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/examples/jupyterhub/README.md b/examples/jupyterhub/README.md index a944ec31..01073f99 100644 --- a/examples/jupyterhub/README.md +++ b/examples/jupyterhub/README.md @@ -18,20 +18,20 @@ For this tutorial it's necessary ### Building images from FirecREST's Docker Compose demo -The `docker-compose.yaml` that we use in this the demo is a copy of the the one used in the Docker demo of FirecREST with only a few small changes. So, to get started, let's clone the FirecREST repository +The [`docker-compose.yaml`](docker-compose.yaml) that we use in this the demo is a copy of the the one from the Docker demo of FirecREST with only a few small changes. So, to get started, let's clone the FirecREST repository ```bash git clone https://github.com/eth-cscs/firecrest.git -cd firecrest/deploy/demo/ ``` and build the images used in it's [Docker Compose demo](https://github.com/eth-cscs/firecrest/tree/master/deploy/demo): ```bash +cd firecrest/deploy/demo/ docker compose build ``` -This step is going to last a few minutes. In the meanwhile we can install JupyterHub on a virtual environment on our machine. +This step is going to take a few minutes. In the meanwhile we can install JupyterHub on a virtual environment on our machine. ### Install JupyterHub and FirecRESTSpawner @@ -55,16 +55,41 @@ cd firecrestspawner pip install --no-cache . ``` -## Deployment of the FirecREST and Slurm cluster +## Deployment of FirecREST and Slurm cluster -Once all the images have been built we can move to the JupyterHub demo directory and start deploy the [`docker-compose.yaml`](docker-compose.yaml). +Once all the images have been built we can move to the JupyterHub demo directory and deploy the [`docker-compose.yaml`](docker-compose.yaml). ```bash cd firecrest/examples/jupyterhub docker compose up -d --build ``` -The deployment can be shutdown with +This step will create a new image that extends the `f7t-cluster` image from the Docker Compose demo with JupyterHub and FirecRESTSpawner. +It will take some time as it needs first to build a few dependencies needed for JupyterHub. + +Once the building is finished you can check that all containers are running + +```bash +docker compose -p demo ps --format 'table {{.ID}}\t{{.Name}}\t{{.State}}' +# CONTAINER ID NAME STATE +# fa355219633c certificator running +# 8dada9a2f57a cluster running +# bd5f33b3b34e compute running +# 8b8029c9bec2 fckeycloak running +# e66970df55a8 jaeger running +# 1be08e3707f4 kong running +# 9dd5a68a84b0 minio running +# 33ce4e9df9c5 opa running +# b0cfba2eb816 openapi running +# 974356ee229a reservations running +# 143375c02912 status running +# c424bca5efef storage running +# 5004cc49e1b8 taskpersistence running +# 163d91b0bd8d tasks running +# 5239294e62bb utilities running +``` + +When we are done with the tutorial, the deployment can be shutdown with ``` cd firecrest/examples/jupyterhub @@ -73,16 +98,16 @@ docker compose down ### Setting up the authorization -To login on JupyterHub and authenticate with FirecREST in the spawner, we need to create an Authorization Code Flow client in the `kcrealm` in Keycloak. In [this page](http://localhost:8080/auth/admin/master/console/#/realms/kcrealm/clients) (username: admin, password: admin2), click on "Create" and then on "Select file" choose the [`jhub-client.json`](jhub-client.json) file from the file system. +For both logging in on JupyterHub and letting the spawner authenticate with FirecREST, we need to create an Authorization Code Flow client in the `kcrealm` in Keycloak. In [this page](http://localhost:8080/auth/admin/master/console/#/realms/kcrealm/clients) (username: admin, password: admin2), click on "Create" and then on "Select file". A file system explorer window will open. Navigate to the demo's directory and choose the [`jhub-client.json`](jhub-client.json) file from the file system. -Once that's done, the client `jhub-client` can be seen listed on the "Clients" tab. +Once that's done, the client `jhub-client` can be seen listed on the "Clients" tab of the side panel. ### Launching JupyterHub -The configuration file provided [here](jupyterhub-config.py) has all the setting to use JupyterHub with the our deployment, but the secret for the client we just created must be added on `c.Authenticator.client_secret`. The secret can be found in the client's ["Credentials" tab](http://localhost:8080/auth/admin/master/console/#/realms/kcrealm/clients/f969b69d-4aec-4646-bdbe-09a268f52111/credentials). +The configuration file provided [here](jupyterhub-config.py) has all the setting for using JupyterHub with the our deployment, but the secret for the client we just created must be added on `c.Authenticator.client_secret`. The secret can be found in the client's ["Credentials" tab](http://localhost:8080/auth/admin/master/console/#/realms/kcrealm/clients/f969b69d-4aec-4646-bdbe-09a268f52111/credentials). -JupyterHub can be run with +Once that's done, JupyterHub can be run with ```bash . jhub-env/bin/activate @@ -92,4 +117,4 @@ jupyterhub --config jupyterhub-config.py --port 8003 --ip 0.0.0.0 --debug Here we are sourcing the file [`env.sh`](env.sh) which defines environment variables needed by the spawner. We use the port `8003` for the hub since `8000` is already used for FirecREST itself in the demo deployment. The ip `0.0.0.0` is necessary to allow JupyterLab to connect back to the hub. -The hub should be accessible in the browser at [http://localhost:8003](http://localhost:8003/) (username: test1 and password: test11) and you should be able to submit notebook jobs. +The hub should be accessible in the browser at [http://localhost:8003](http://localhost:8003/) (username: test1 and password: test11) and you should be able to launch notebooks on the slurm cluster. From c87e225c6482f5f5eb75b811878d157bc0f46858 Mon Sep 17 00:00:00 2001 From: Rafael Sarmiento Date: Mon, 18 Nov 2024 16:26:36 +0100 Subject: [PATCH 03/24] specify platform on docker compose --- examples/jupyterhub/docker-compose.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/jupyterhub/docker-compose.yml b/examples/jupyterhub/docker-compose.yml index 8b9bd75f..589975c2 100644 --- a/examples/jupyterhub/docker-compose.yml +++ b/examples/jupyterhub/docker-compose.yml @@ -4,8 +4,6 @@ ## Please, refer to the LICENSE file in the root directory. ## SPDX-License-Identifier: BSD-3-Clause ## -version: '3.4' - networks: firecrest-internal: driver: bridge @@ -202,6 +200,7 @@ services: build: context: ../test-build dockerfile: Dockerfile + platform: linux/amd64 network: host networks: firecrest-internal: From a6cec6190e6eaf96b5e4d1376ddb45644ee5903a Mon Sep 17 00:00:00 2001 From: Rafael Sarmiento Date: Mon, 18 Nov 2024 16:34:53 +0100 Subject: [PATCH 04/24] change format --- examples/jupyterhub/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/jupyterhub/README.md b/examples/jupyterhub/README.md index 01073f99..c166659c 100644 --- a/examples/jupyterhub/README.md +++ b/examples/jupyterhub/README.md @@ -18,7 +18,7 @@ For this tutorial it's necessary ### Building images from FirecREST's Docker Compose demo -The [`docker-compose.yaml`](docker-compose.yaml) that we use in this the demo is a copy of the the one from the Docker demo of FirecREST with only a few small changes. So, to get started, let's clone the FirecREST repository +The [docker-compose.yaml](docker-compose.yaml) that we use in this the demo is a copy of the the one from the Docker demo of FirecREST with only a few small changes. So, to get started, let's clone the FirecREST repository ```bash git clone https://github.com/eth-cscs/firecrest.git @@ -57,7 +57,7 @@ pip install --no-cache . ## Deployment of FirecREST and Slurm cluster -Once all the images have been built we can move to the JupyterHub demo directory and deploy the [`docker-compose.yaml`](docker-compose.yaml). +Once all the images have been built we can move to the JupyterHub demo directory and deploy the [docker-compose.yaml](docker-compose.yaml). ```bash cd firecrest/examples/jupyterhub @@ -98,7 +98,7 @@ docker compose down ### Setting up the authorization -For both logging in on JupyterHub and letting the spawner authenticate with FirecREST, we need to create an Authorization Code Flow client in the `kcrealm` in Keycloak. In [this page](http://localhost:8080/auth/admin/master/console/#/realms/kcrealm/clients) (username: admin, password: admin2), click on "Create" and then on "Select file". A file system explorer window will open. Navigate to the demo's directory and choose the [`jhub-client.json`](jhub-client.json) file from the file system. +For both logging in on JupyterHub and letting the spawner authenticate with FirecREST, we need to create an Authorization Code Flow client in the `kcrealm` in Keycloak. In [this page](http://localhost:8080/auth/admin/master/console/#/realms/kcrealm/clients) (username: admin, password: admin2), click on "Create" and then on "Select file". A file system explorer window will open. Navigate to the demo's directory and choose the [jhub-client.json](jhub-client.json) file from the file system. Once that's done, the client `jhub-client` can be seen listed on the "Clients" tab of the side panel. From 034613b96400239461d764657c811f4046cb8584 Mon Sep 17 00:00:00 2001 From: Rafael Sarmiento Date: Tue, 19 Nov 2024 09:58:27 +0100 Subject: [PATCH 05/24] change format --- examples/jupyterhub/README.md | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/examples/jupyterhub/README.md b/examples/jupyterhub/README.md index c166659c..dfe4c730 100644 --- a/examples/jupyterhub/README.md +++ b/examples/jupyterhub/README.md @@ -3,13 +3,14 @@ This is a tutorial on how to run JupyterHub with [FirecRESTSpawner](https://github.com/eth-cscs/firecrestspawner) on the [Docker demo of FirecREST](https://github.com/eth-cscs/firecrest/tree/master/deploy/demo). -We are going to start by deploying FirecREST together with a slurm cluster using [Docker Compose](https://docs.docker.com/compose). Then we will install JupyterHub on a virtual environment and configure it to launch notebooks on the slurm cluster via FirecREST. +We are going to start by deploying FirecREST together with a slurm cluster using [Docker Compose](https://docs.docker.com/compose). +Then we will install JupyterHub on a virtual environment and configure it to launch notebooks on the slurm cluster via FirecREST. ## Requirements For this tutorial it's necessary - * a recent installation of Docker, which includes the `docker compose` command. Alternatively the old `docker-compose` command line could be used. + * a recent installation of Docker, which includes the `docker compose` command or the older `docker-compose` command line * a python installation (`>=3.9`) @@ -18,7 +19,8 @@ For this tutorial it's necessary ### Building images from FirecREST's Docker Compose demo -The [docker-compose.yaml](docker-compose.yaml) that we use in this the demo is a copy of the the one from the Docker demo of FirecREST with only a few small changes. So, to get started, let's clone the FirecREST repository +The [docker-compose.yaml](docker-compose.yaml) that we use in this the demo is a copy of the the one from the Docker demo of FirecREST with only a few small changes. +So, to get started, let's clone the FirecREST repository ```bash git clone https://github.com/eth-cscs/firecrest.git @@ -64,8 +66,8 @@ cd firecrest/examples/jupyterhub docker compose up -d --build ``` -This step will create a new image that extends the `f7t-cluster` image from the Docker Compose demo with JupyterHub and FirecRESTSpawner. -It will take some time as it needs first to build a few dependencies needed for JupyterHub. +This step will create a new image that extends the `f7t-cluster` image from the FirecREST demo to include JupyterLab and other requirements. +It will take some time since it needs to first build from source a few dependencies of JupyterLab. Once the building is finished you can check that all containers are running @@ -98,14 +100,22 @@ docker compose down ### Setting up the authorization -For both logging in on JupyterHub and letting the spawner authenticate with FirecREST, we need to create an Authorization Code Flow client in the `kcrealm` in Keycloak. In [this page](http://localhost:8080/auth/admin/master/console/#/realms/kcrealm/clients) (username: admin, password: admin2), click on "Create" and then on "Select file". A file system explorer window will open. Navigate to the demo's directory and choose the [jhub-client.json](jhub-client.json) file from the file system. +A requirement for running JupyterHub with FirecRESTSpawner is to use an authenticator that prompts users for login and password in exchange for an access token. +That token will then be passed to the spawner, allowing users to authenticate with FirecREST when submitting, stopping or polling for jobs. +For this purpose, we will use an Authorization Code Flow client, which we need to create on the Keycloak web interface. + +Let's go to [this page](http://localhost:8080/auth/admin/master/console/#/realms/kcrealm/clients) (username: admin, password: admin2) and make sure that the top left side indicates that we are within the `Kcrealm` realm. +We click on "Create" and then on "Select file". +A file system explorer will open. +Navigate to the demo's directory and choose the [jhub-client.json](jhub-client.json) file. Once that's done, the client `jhub-client` can be seen listed on the "Clients" tab of the side panel. ### Launching JupyterHub -The configuration file provided [here](jupyterhub-config.py) has all the setting for using JupyterHub with the our deployment, but the secret for the client we just created must be added on `c.Authenticator.client_secret`. The secret can be found in the client's ["Credentials" tab](http://localhost:8080/auth/admin/master/console/#/realms/kcrealm/clients/f969b69d-4aec-4646-bdbe-09a268f52111/credentials). +The configuration file provided [here](jupyterhub-config.py) has all the setting for using JupyterHub with the our deployment, but the secret for the client we just created must be added on `c.Authenticator.client_secret`. +The secret can be found in the client's ["Credentials" tab](http://localhost:8080/auth/admin/master/console/#/realms/kcrealm/clients/f969b69d-4aec-4646-bdbe-09a268f52111/credentials). Once that's done, JupyterHub can be run with @@ -114,7 +124,8 @@ Once that's done, JupyterHub can be run with . env.sh jupyterhub --config jupyterhub-config.py --port 8003 --ip 0.0.0.0 --debug ``` -Here we are sourcing the file [`env.sh`](env.sh) which defines environment variables needed by the spawner. -We use the port `8003` for the hub since `8000` is already used for FirecREST itself in the demo deployment. The ip `0.0.0.0` is necessary to allow JupyterLab to connect back to the hub. +Here we are sourcing the file [env.sh](env.sh) which defines environment variables needed by the spawner(More information can be found [here](https://firecrestspawner.readthedocs.io/en/latest/authentication.html)). +We use the port `8003` for the hub since `8000` is already used for FirecREST itself in the demo deployment. +The ip `0.0.0.0` is necessary to allow JupyterLab to connect back to the hub. -The hub should be accessible in the browser at [http://localhost:8003](http://localhost:8003/) (username: test1 and password: test11) and you should be able to launch notebooks on the slurm cluster. +The hub should be accessible in the browser at [http://localhost:8003](http://localhost:8003/) (username: test1 and password: test11) and it should be possible to launch notebooks on the slurm cluster. From 0d5d1b82a78286461c3ad19f657d75e1f7c7c131 Mon Sep 17 00:00:00 2001 From: Rafael Sarmiento Date: Tue, 19 Nov 2024 10:08:01 +0100 Subject: [PATCH 06/24] update --- examples/jupyterhub/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/jupyterhub/README.md b/examples/jupyterhub/README.md index dfe4c730..74ceafc3 100644 --- a/examples/jupyterhub/README.md +++ b/examples/jupyterhub/README.md @@ -114,7 +114,8 @@ Once that's done, the client `jhub-client` can be seen listed on the "Clients" t ### Launching JupyterHub -The configuration file provided [here](jupyterhub-config.py) has all the setting for using JupyterHub with the our deployment, but the secret for the client we just created must be added on `c.Authenticator.client_secret`. +The [configuration file](jupyterhub-config.py) provided in the demo has all the settings needed for using JupyterHub with our deployment. +We only need to add in `c.Authenticator.client_secret` the secret for the client we just created. The secret can be found in the client's ["Credentials" tab](http://localhost:8080/auth/admin/master/console/#/realms/kcrealm/clients/f969b69d-4aec-4646-bdbe-09a268f52111/credentials). Once that's done, JupyterHub can be run with From 30a06f994b14c8df860a801556fed83fd09c1c20 Mon Sep 17 00:00:00 2001 From: Rafael Sarmiento Date: Tue, 19 Nov 2024 10:10:27 +0100 Subject: [PATCH 07/24] update --- examples/jupyterhub/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/jupyterhub/README.md b/examples/jupyterhub/README.md index 74ceafc3..d70af576 100644 --- a/examples/jupyterhub/README.md +++ b/examples/jupyterhub/README.md @@ -125,7 +125,7 @@ Once that's done, JupyterHub can be run with . env.sh jupyterhub --config jupyterhub-config.py --port 8003 --ip 0.0.0.0 --debug ``` -Here we are sourcing the file [env.sh](env.sh) which defines environment variables needed by the spawner(More information can be found [here](https://firecrestspawner.readthedocs.io/en/latest/authentication.html)). +Here we are sourcing the file [env.sh](env.sh) which defines environment variables needed by the spawner (more information can be found [here](https://firecrestspawner.readthedocs.io/en/latest/authentication.html)). We use the port `8003` for the hub since `8000` is already used for FirecREST itself in the demo deployment. The ip `0.0.0.0` is necessary to allow JupyterLab to connect back to the hub. From a8f91aac47755042a15d8a6f8b484399ccd8d02c Mon Sep 17 00:00:00 2001 From: Rafael Sarmiento Date: Tue, 19 Nov 2024 10:12:15 +0100 Subject: [PATCH 08/24] update --- examples/jupyterhub/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/jupyterhub/README.md b/examples/jupyterhub/README.md index d70af576..0edfa961 100644 --- a/examples/jupyterhub/README.md +++ b/examples/jupyterhub/README.md @@ -126,7 +126,7 @@ Once that's done, JupyterHub can be run with jupyterhub --config jupyterhub-config.py --port 8003 --ip 0.0.0.0 --debug ``` Here we are sourcing the file [env.sh](env.sh) which defines environment variables needed by the spawner (more information can be found [here](https://firecrestspawner.readthedocs.io/en/latest/authentication.html)). -We use the port `8003` for the hub since `8000` is already used for FirecREST itself in the demo deployment. +We use the port `8003` for the hub since the default one `8000` is already used for FirecREST itself in the demo deployment. The ip `0.0.0.0` is necessary to allow JupyterLab to connect back to the hub. The hub should be accessible in the browser at [http://localhost:8003](http://localhost:8003/) (username: test1 and password: test11) and it should be possible to launch notebooks on the slurm cluster. From bb534f41fe156e0bb76275955447b2ce7ab5cace Mon Sep 17 00:00:00 2001 From: Rafael Sarmiento Date: Wed, 20 Nov 2024 10:17:39 +0100 Subject: [PATCH 09/24] use override docker compose file --- examples/jupyterhub/Dockerfile | 2 +- examples/jupyterhub/README.md | 6 +- examples/jupyterhub/docker-compose.yml | 315 +------------------------ 3 files changed, 8 insertions(+), 315 deletions(-) diff --git a/examples/jupyterhub/Dockerfile b/examples/jupyterhub/Dockerfile index a1c132cd..b4f86f13 100644 --- a/examples/jupyterhub/Dockerfile +++ b/examples/jupyterhub/Dockerfile @@ -1,4 +1,4 @@ -FROM f7t-cluster +FROM --platform=linux/amd64 f7t-cluster RUN yum update -y && \ yum install -y sqlite-devel openssl-devel bzip2-devel libffi-devel wget && \ diff --git a/examples/jupyterhub/README.md b/examples/jupyterhub/README.md index 0edfa961..075842bf 100644 --- a/examples/jupyterhub/README.md +++ b/examples/jupyterhub/README.md @@ -63,7 +63,7 @@ Once all the images have been built we can move to the JupyterHub demo directory ```bash cd firecrest/examples/jupyterhub -docker compose up -d --build +docker compose -f ../../deploy/demo/docker-compose.yml -f docker-compose.yml up --build ``` This step will create a new image that extends the `f7t-cluster` image from the FirecREST demo to include JupyterLab and other requirements. @@ -91,11 +91,11 @@ docker compose -p demo ps --format 'table {{.ID}}\t{{.Name}}\t{{.State}}' # 5239294e62bb utilities running ``` -When we are done with the tutorial, the deployment can be shutdown with +When we are done with the tutorial, the deployment can be shutdown by pressing `ctrl+c` and then ``` cd firecrest/examples/jupyterhub -docker compose down +docker compose -f ../../deploy/demo/docker-compose.yml -f docker-compose.yml down ``` ### Setting up the authorization diff --git a/examples/jupyterhub/docker-compose.yml b/examples/jupyterhub/docker-compose.yml index 589975c2..406b6d31 100644 --- a/examples/jupyterhub/docker-compose.yml +++ b/examples/jupyterhub/docker-compose.yml @@ -1,316 +1,9 @@ -## -## Copyright (c) 2019-2024, ETH Zurich. All rights reserved. -## -## Please, refer to the LICENSE file in the root directory. -## SPDX-License-Identifier: BSD-3-Clause -## -networks: - firecrest-internal: - driver: bridge - ipam: - driver: default - config: - - subnet: 192.168.220.0/24 - - services: - f7t-base: - image: f7t-base - container_name: f7t-base - build: - context: ../../ - network: host - # FirecREST microservices - certificator: - image: f7t-certificator - container_name: certificator - depends_on: - - f7t-base - build: - context: ../../ - network: host - env_file: - - ./common/common.env - environment: - - F7T_CA_KEY_PATH=/etc/firecrest/ca-key - - F7T_PUB_USER_KEY_PATH=/etc/firecrest/user-key.pub - networks: - firecrest-internal: - ipv4_address: 192.168.220.11 - ports: - - 5010:5010 - volumes: - - ./logs/firecrest:/var/log:delegated - - ../test-build/environment/keys/user-key.pub:/etc/firecrest/user-key.pub:ro - - ../test-build/environment/keys/ca-key:/etc/firecrest/ca-key:ro - - ./ssl:/ssl - - ./logs/profs:/var/log/profs - - compute: - image: f7t-compute - container_name: compute - depends_on: - - f7t-base - build: - context: ../../ - network: host - environment: - - F7T_PRIV_USER_KEY_PATH=/etc/firecrest/ssh/user-key - env_file: - - ./common/common.env - networks: - firecrest-internal: - ipv4_address: 192.168.220.9 - ports: - - 5006:5006 - volumes: - - ./logs/firecrest:/var/log:delegated - - ../test-build/environment/keys/user-key:/etc/firecrest/ssh/user-key:ro - - ./ssl:/ssl - - ./logs/profs:/var/log/profs - - status: - image: f7t-status - container_name: status - depends_on: - - f7t-base - build: - context: ../../ - network: host - args: - GENERAL_INFORMATION: '{"FIRECREST_VERSION": "demo"}' - env_file: - - ./common/common.env - networks: - firecrest-internal: - ipv4_address: 192.168.220.4 - ports: - - 5001:5001 - volumes: - - ./logs/firecrest:/var/log:delegated - - ./ssl:/ssl - - storage: - image: f7t-storage - container_name: storage - depends_on: - - f7t-base - build: - context: ../../ - network: host - env_file: - - ./common/common.env - environment: - F7T_S3_PRIVATE_URL: http://192.168.220.19:9000 - F7T_S3_PUBLIC_URL: http://192.168.220.19:9000 - F7T_S3_ACCESS_KEY: storage_access_key - F7T_S3_SECRET_KEY: storage_secret_key - F7T_S3_REGION: us-east-1 - F7T_S3_TENANT: null - F7T_STORAGE_POLLING_INTERVAL: 60 - F7T_CERT_CIPHER_KEY: 'Df6UZuoPoJ2u5yRwxNfFQ46Nwy8eW1OGTcuhlqn4ONo=' - networks: - firecrest-internal: - ipv4_address: 192.168.220.5 - volumes: - - ./logs/firecrest:/var/log:delegated - - ../test-build/environment/keys/user-key:/user-key:ro - - ./ssl:/ssl - - ./logs/profs:/var/log/profs - - tasks: - image: f7t-tasks - container_name: tasks - depends_on: - - f7t-base - - "taskpersistence" - build: - context: ../../ - network: host - env_file: - - ./common/common.env - environment: - - F7T_PERSIST_PORT=6379 - - F7T_PERSIST_PWD=rediS2200 - - F7T_COMPUTE_TASK_EXP_TIME=86400 - - F7T_STORAGE_TASK_EXP_TIME=2678400 - networks: - firecrest-internal: - ipv4_address: 192.168.220.6 - ports: - - 5003:5003 - volumes: - - ./logs/firecrest:/var/log:delegated - - ./ssl:/ssl - - ./logs/profs:/var/log/profs - - utilities: - image: f7t-utilities - container_name: utilities - depends_on: - - f7t-base - build: - context: ../../ - network: host - env_file: - - ./common/common.env - command: - [ - "sh", - "-c", - "pip install debugpy && python3 -Xfrozen_modules=off -m debugpy --listen 0.0.0.0:5678 -m gunicorn --bind :$${F7T_UTILITIES_PORT} $${F7T_GUNICORN_LOG} $${F7T_GUNICORN_SSL} $${F7T_GUNICORN_WORKER} utilities:app", - ] - networks: - firecrest-internal: - ipv4_address: 192.168.220.7 - ports: - - 5004:5004 - - 5678:5678 - volumes: - - ./logs/firecrest:/var/log:delegated - - ../test-build/environment/keys/user-key:/user-key:ro - - ./ssl:/ssl - - ./logs/profs:/var/log/profs - - reservations: - image: f7t-reservations - container_name: reservations - depends_on: - - f7t-base - build: - context: ../../ - network: host - env_file: - - ./common/common.env - networks: - firecrest-internal: - ipv4_address: 192.168.220.8 - ports: - - 5005:5005 - volumes: - - ./logs/firecrest:/var/log:delegated - - ../test-build/environment/keys/user-key:/user-key:ro - - ./ssl:/ssl - - ./logs/profs:/var/log/profs - - # dummy cluster cluster: - image: f7t-cluster:jhub-amd64 - container_name: cluster + image: "f7t-cluster:jhub" build: - context: ../test-build - dockerfile: Dockerfile - platform: linux/amd64 + context: . + dockerfile: /Users/sarafael/git_/firecrest_gitlab/examples/jupyterhub/Dockerfile network: host - networks: - firecrest-internal: - ipv4_address: 192.168.220.12 - hostname: cluster - ports: - - 56123:56123 - volumes: - - ./logs/cluster/:/var/log/slurm/:delegated - - # complementary 3rd party services - keycloak: - image: quay.io/keycloak/keycloak:18.0.2 - container_name: fckeycloak - env_file: keycloak/keycloak.env - command: start-dev --http-relative-path /auth --import-realm - - networks: - firecrest-internal: - ipv4_address: 192.168.220.20 - ports: - - "${KEYCLOAK_PORT:-8080}:8080" - volumes: - - ./keycloak/config.json:/opt/keycloak/data/import/config.json:ro - - ./logs/keycloak:/opt/jboss/keycloak/standalone/log/:delegated - - kong: - image: kong:3.2 - container_name: kong - environment: - - KONG_DATABASE=off - - KONG_DECLARATIVE_CONFIG=/kong.yml - - KONG_LOG_LEVEL=error - volumes: - - ./kong/kong.yml:/kong.yml:ro - networks: - firecrest-internal: - ipv4_address: 192.168.220.21 - ports: - - 8000:8000 - - minio: - image: minio/minio:RELEASE.2022-10-24T18-35-07Z - container_name: minio - command: minio server /data --console-address ":9001" - environment: - MINIO_ROOT_USER: storage_access_key - MINIO_ROOT_PASSWORD: storage_secret_key - networks: - firecrest-internal: - ipv4_address: 192.168.220.19 - ports: - - "9000:9000" - - "9001:9001" - volumes: - - ./minio:/data:delegated - - taskpersistence: - container_name: taskpersistence - image: redis:5 - command: redis-server /redis.conf - networks: - firecrest-internal: - ipv4_address: 192.168.220.13 - ports: - - 6379:6379 - volumes: - - ./taskpersistence/redis.conf:/redis.conf:ro - - ./taskpersistence-data:/data:delegated - - ./logs:/var/log:delegated - - opa: - image: openpolicyagent/opa:0.22.0 - container_name: opa - command: run --server --log-level=debug --log-format=json-pretty --skip-version-check --tls-cert-file=/ssl/f7t_internal.crt --tls-private-key-file=/ssl/f7t_internal.key /opa-files/data.json /opa-files/policy.rego - networks: - firecrest-internal: - ipv4_address: 192.168.220.40 - ports: - - "8181:8181" - volumes: - - ./opa:/opa-files - - ./ssl:/ssl - - openapi: - image: swaggerapi/swagger-ui:v4.19.1 - container_name: openapi - networks: - firecrest-internal: - ipv4_address: 192.168.220.41 - environment: - SWAGGER_JSON: /tmp/firecrest-api.yaml - BASE_URL: /docs - volumes: - - ../../doc/openapi/:/tmp/ - - jaeger: - image: jaegertracing/all-in-one:1.24 - container_name: jaeger - networks: - firecrest-internal: - ipv4_address: 192.168.220.50 - environment: - COLLECTOR_ZIPKIN_HOST_PORT: "9411" ports: - #- 5775:5775/udp - - 6831:6831/udp - - 6832:6832/udp - - 5778:5778 - - 16686:16686 - - 14268:14268 - - 14250:14250 - - 9411:9411 + - "56123:56123" From 62fbff362982583935b95efdf25c1b2be61a5b41 Mon Sep 17 00:00:00 2001 From: Rafael Sarmiento Date: Wed, 20 Nov 2024 10:45:52 +0100 Subject: [PATCH 10/24] add secret to client json --- examples/jupyterhub/README.md | 6 ++---- examples/jupyterhub/jhub-client.json | 3 ++- examples/jupyterhub/jupyterhub-config.py | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/examples/jupyterhub/README.md b/examples/jupyterhub/README.md index 075842bf..70ec732d 100644 --- a/examples/jupyterhub/README.md +++ b/examples/jupyterhub/README.md @@ -107,7 +107,7 @@ For this purpose, we will use an Authorization Code Flow client, which we need t Let's go to [this page](http://localhost:8080/auth/admin/master/console/#/realms/kcrealm/clients) (username: admin, password: admin2) and make sure that the top left side indicates that we are within the `Kcrealm` realm. We click on "Create" and then on "Select file". A file system explorer will open. -Navigate to the demo's directory and choose the [jhub-client.json](jhub-client.json) file. +Navigate to the demo's directory and choose the [jhub-client.json](jhub-client.json) file and click on "Save". Once that's done, the client `jhub-client` can be seen listed on the "Clients" tab of the side panel. @@ -115,10 +115,8 @@ Once that's done, the client `jhub-client` can be seen listed on the "Clients" t ### Launching JupyterHub The [configuration file](jupyterhub-config.py) provided in the demo has all the settings needed for using JupyterHub with our deployment. -We only need to add in `c.Authenticator.client_secret` the secret for the client we just created. -The secret can be found in the client's ["Credentials" tab](http://localhost:8080/auth/admin/master/console/#/realms/kcrealm/clients/f969b69d-4aec-4646-bdbe-09a268f52111/credentials). -Once that's done, JupyterHub can be run with +Now we can run JupyterHub with ```bash . jhub-env/bin/activate diff --git a/examples/jupyterhub/jhub-client.json b/examples/jupyterhub/jhub-client.json index 990ff298..ef4dfb8f 100644 --- a/examples/jupyterhub/jhub-client.json +++ b/examples/jupyterhub/jhub-client.json @@ -5,6 +5,7 @@ "enabled": true, "alwaysDisplayInConsole": false, "clientAuthenticatorType": "client-secret", + "secret": "Ap45Agq2KpnbTXUGQxaUF1WiVOPm8Wf0", "redirectUris": [ "http://localhost:8003/hub/oauth_callback" ], @@ -70,4 +71,4 @@ "configure": true, "manage": true } -} \ No newline at end of file +} diff --git a/examples/jupyterhub/jupyterhub-config.py b/examples/jupyterhub/jupyterhub-config.py index 0039ce38..a5ec1044 100644 --- a/examples/jupyterhub/jupyterhub-config.py +++ b/examples/jupyterhub/jupyterhub-config.py @@ -15,7 +15,7 @@ def gen_hex_string(num_bytes=32, num_hex_strings=4): # Keycloak setup c.Authenticator.client_id = "jhub-client" -c.Authenticator.client_secret = "" +c.Authenticator.client_secret = "Ap45Agq2KpnbTXUGQxaUF1WiVOPm8Wf0" c.Authenticator.oauth_callback_url = "http://localhost:8003/hub/oauth_callback" c.Authenticator.authorize_url = "http://localhost:8080/auth/realms/kcrealm/protocol/openid-connect/auth" c.Authenticator.token_url = "http://localhost:8080/auth/realms/kcrealm/protocol/openid-connect/token" From e142930c71063d0276f873bdbfcb10d031f981bd Mon Sep 17 00:00:00 2001 From: Rafael Sarmiento Date: Wed, 20 Nov 2024 10:59:48 +0100 Subject: [PATCH 11/24] remove --debug from jhub --- examples/jupyterhub/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/jupyterhub/README.md b/examples/jupyterhub/README.md index 70ec732d..d6a3c002 100644 --- a/examples/jupyterhub/README.md +++ b/examples/jupyterhub/README.md @@ -121,7 +121,7 @@ Now we can run JupyterHub with ```bash . jhub-env/bin/activate . env.sh -jupyterhub --config jupyterhub-config.py --port 8003 --ip 0.0.0.0 --debug +jupyterhub --config jupyterhub-config.py --port 8003 --ip 0.0.0.0 ``` Here we are sourcing the file [env.sh](env.sh) which defines environment variables needed by the spawner (more information can be found [here](https://firecrestspawner.readthedocs.io/en/latest/authentication.html)). We use the port `8003` for the hub since the default one `8000` is already used for FirecREST itself in the demo deployment. From 7c7c2c3e371886a0b43ed99cd28577217b679f57 Mon Sep 17 00:00:00 2001 From: Rafael Sarmiento Date: Wed, 20 Nov 2024 12:27:26 +0100 Subject: [PATCH 12/24] remove hardcoded path --- examples/jupyterhub/docker-compose.yml | 2 +- examples/jupyterhub/env.sh | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/jupyterhub/docker-compose.yml b/examples/jupyterhub/docker-compose.yml index 406b6d31..3d4659e9 100644 --- a/examples/jupyterhub/docker-compose.yml +++ b/examples/jupyterhub/docker-compose.yml @@ -3,7 +3,7 @@ services: image: "f7t-cluster:jhub" build: context: . - dockerfile: /Users/sarafael/git_/firecrest_gitlab/examples/jupyterhub/Dockerfile + dockerfile: $JHUB_DOCKERFILE_DIR/Dockerfile network: host ports: - "56123:56123" diff --git a/examples/jupyterhub/env.sh b/examples/jupyterhub/env.sh index cc4d0373..009e4d6f 100644 --- a/examples/jupyterhub/env.sh +++ b/examples/jupyterhub/env.sh @@ -2,3 +2,5 @@ export FIRECREST_URL=http://localhost:8000 export SA_CLIENT_ID=firecrest-sample export SA_CLIENT_SECRET=b391e177-fa50-4987-beaf-e6d33ca93571 export SA_AUTH_TOKEN_URL=http://localhost:8080/auth/realms/kcrealm/protocol/openid-connect/token + +export JHUB_DOCKERFILE_DIR=$PWD From 52128000dfa2cb5218762aaa02e8b67b33d2aeb6 Mon Sep 17 00:00:00 2001 From: Rafael Sarmiento Date: Wed, 20 Nov 2024 12:29:49 +0100 Subject: [PATCH 13/24] update --- examples/jupyterhub/README.md | 37 ++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 16 deletions(-) diff --git a/examples/jupyterhub/README.md b/examples/jupyterhub/README.md index d6a3c002..5cdeb078 100644 --- a/examples/jupyterhub/README.md +++ b/examples/jupyterhub/README.md @@ -73,22 +73,27 @@ Once the building is finished you can check that all containers are running ```bash docker compose -p demo ps --format 'table {{.ID}}\t{{.Name}}\t{{.State}}' -# CONTAINER ID NAME STATE -# fa355219633c certificator running -# 8dada9a2f57a cluster running -# bd5f33b3b34e compute running -# 8b8029c9bec2 fckeycloak running -# e66970df55a8 jaeger running -# 1be08e3707f4 kong running -# 9dd5a68a84b0 minio running -# 33ce4e9df9c5 opa running -# b0cfba2eb816 openapi running -# 974356ee229a reservations running -# 143375c02912 status running -# c424bca5efef storage running -# 5004cc49e1b8 taskpersistence running -# 163d91b0bd8d tasks running -# 5239294e62bb utilities running +``` + +That should show something like this + +```bash +CONTAINER ID NAME STATE +fa355219633c certificator running +8dada9a2f57a cluster running +bd5f33b3b34e compute running +8b8029c9bec2 fckeycloak running +e66970df55a8 jaeger running +1be08e3707f4 kong running +9dd5a68a84b0 minio running +33ce4e9df9c5 opa running +b0cfba2eb816 openapi running +974356ee229a reservations running +143375c02912 status running +c424bca5efef storage running +5004cc49e1b8 taskpersistence running +163d91b0bd8d tasks running +5239294e62bb utilities running ``` When we are done with the tutorial, the deployment can be shutdown by pressing `ctrl+c` and then From 096070483d51d192bcda8ab0c4c1ae4b2089f475 Mon Sep 17 00:00:00 2001 From: Rafael Sarmiento Date: Wed, 20 Nov 2024 13:38:17 +0100 Subject: [PATCH 14/24] update --- examples/jupyterhub/README.md | 1 + examples/jupyterhub/env.sh | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/jupyterhub/README.md b/examples/jupyterhub/README.md index 5cdeb078..fcb479db 100644 --- a/examples/jupyterhub/README.md +++ b/examples/jupyterhub/README.md @@ -63,6 +63,7 @@ Once all the images have been built we can move to the JupyterHub demo directory ```bash cd firecrest/examples/jupyterhub +export JHUB_DOCKERFILE_DIR=$PWD docker compose -f ../../deploy/demo/docker-compose.yml -f docker-compose.yml up --build ``` diff --git a/examples/jupyterhub/env.sh b/examples/jupyterhub/env.sh index 009e4d6f..cc4d0373 100644 --- a/examples/jupyterhub/env.sh +++ b/examples/jupyterhub/env.sh @@ -2,5 +2,3 @@ export FIRECREST_URL=http://localhost:8000 export SA_CLIENT_ID=firecrest-sample export SA_CLIENT_SECRET=b391e177-fa50-4987-beaf-e6d33ca93571 export SA_AUTH_TOKEN_URL=http://localhost:8080/auth/realms/kcrealm/protocol/openid-connect/token - -export JHUB_DOCKERFILE_DIR=$PWD From 27b208ebf2441799731959139d660089af1ff1b5 Mon Sep 17 00:00:00 2001 From: Rafael Sarmiento Date: Thu, 21 Nov 2024 09:11:10 +0100 Subject: [PATCH 15/24] install jhub with miniconda --- examples/jupyterhub/README.md | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/examples/jupyterhub/README.md b/examples/jupyterhub/README.md index fcb479db..06c1d6b7 100644 --- a/examples/jupyterhub/README.md +++ b/examples/jupyterhub/README.md @@ -38,22 +38,25 @@ This step is going to take a few minutes. In the meanwhile we can install Jupyte ### Install JupyterHub and FirecRESTSpawner -We can create a virtual environment +An easy way to install JupyterHub is via [Miniconda](https://docs.anaconda.com/miniconda/install/). +We need to download the Miniconda installer for our platforms and install it with ```bash -python -m venv jhub-env +bash Miniconda3-latest-.sh -p /path/to/mc-jhub -b ``` -and install JupyterHub and FirecRESTSpawner +Here we use `-p` to pass the absolute path to the install directory and `-b` to accept the [terms of service](https://legal.anaconda.com/policies/en/). -```bash -. jhub-env/bin/activate +Then we can activate our conda environment and install configurable-http-proxy, JupyterHub and FirecRESTSpawner +```bash +. /path/to/mc-jhub/bin/activate +conda install -y configurable-http-proxy pip install --no-cache jupyterhub==4.1.6 pyfirecrest==2.6.0 SQLAlchemy==1.4.52 oauthenticator==16.3.1 python-hostlist==1.23.0 git clone https://github.com/eth-cscs/firecrestspawner.git cd firecrestspawner -. jhub-env/bin/activate +. /path/to/mc-jhub/bin/activate pip install --no-cache . ``` @@ -125,7 +128,7 @@ The [configuration file](jupyterhub-config.py) provided in the demo has all the Now we can run JupyterHub with ```bash -. jhub-env/bin/activate +. /path/to/mc-jhub/bin/activate . env.sh jupyterhub --config jupyterhub-config.py --port 8003 --ip 0.0.0.0 ``` From f6a070496fefc840358da22a57e447de1c00a990 Mon Sep 17 00:00:00 2001 From: Rafael Sarmiento Date: Thu, 21 Nov 2024 09:40:26 +0100 Subject: [PATCH 16/24] add note on bridge network ip --- examples/jupyterhub/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/jupyterhub/README.md b/examples/jupyterhub/README.md index 06c1d6b7..ae410c67 100644 --- a/examples/jupyterhub/README.md +++ b/examples/jupyterhub/README.md @@ -113,10 +113,10 @@ A requirement for running JupyterHub with FirecRESTSpawner is to use an authenti That token will then be passed to the spawner, allowing users to authenticate with FirecREST when submitting, stopping or polling for jobs. For this purpose, we will use an Authorization Code Flow client, which we need to create on the Keycloak web interface. -Let's go to [this page](http://localhost:8080/auth/admin/master/console/#/realms/kcrealm/clients) (username: admin, password: admin2) and make sure that the top left side indicates that we are within the `Kcrealm` realm. +Let's go to the [Clients page](http://localhost:8080/auth/admin/master/console/#/realms/kcrealm/clients) in Keycloak (username: admin, password: admin2) within the `kcrealm` realm. We click on "Create" and then on "Select file". A file system explorer will open. -Navigate to the demo's directory and choose the [jhub-client.json](jhub-client.json) file and click on "Save". +Navigate to the tutorial's directory, choose the [jhub-client.json](jhub-client.json) file and click on "Save". Once that's done, the client `jhub-client` can be seen listed on the "Clients" tab of the side panel. @@ -124,6 +124,7 @@ Once that's done, the client `jhub-client` can be seen listed on the "Clients" t ### Launching JupyterHub The [configuration file](jupyterhub-config.py) provided in the demo has all the settings needed for using JupyterHub with our deployment. +Depending on the platform and Docker setup, you may need to adjust a few lines in the configuration to set the correct host IP address for the Docker bridge network. On most Linux systems, you can find this address with `ip addr show docker0`. It is typically `172.17.0.1`, which you can use to replace `host.docker.internal` in the configuration if the latter doesn't work. Now we can run JupyterHub with From 0d434d72a37cc2a1b06477b3b143a8940b523cd5 Mon Sep 17 00:00:00 2001 From: Rafael Sarmiento Date: Thu, 21 Nov 2024 09:50:53 +0100 Subject: [PATCH 17/24] update --- examples/jupyterhub/README.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/examples/jupyterhub/README.md b/examples/jupyterhub/README.md index ae410c67..bd8857e3 100644 --- a/examples/jupyterhub/README.md +++ b/examples/jupyterhub/README.md @@ -19,7 +19,7 @@ For this tutorial it's necessary ### Building images from FirecREST's Docker Compose demo -The [docker-compose.yaml](docker-compose.yaml) that we use in this the demo is a copy of the the one from the Docker demo of FirecREST with only a few small changes. +The [docker-compose.yaml](docker-compose.yaml) that we use in this tutorial is a copy of the the one from the Docker demo of FirecREST with only a few small changes. So, to get started, let's clone the FirecREST repository ```bash @@ -33,7 +33,7 @@ cd firecrest/deploy/demo/ docker compose build ``` -This step is going to take a few minutes. In the meanwhile we can install JupyterHub on a virtual environment on our machine. +This step takes a few minutes. In the meanwhile we can install JupyterHub on a virtual environment on our machine. ### Install JupyterHub and FirecRESTSpawner @@ -62,7 +62,7 @@ pip install --no-cache . ## Deployment of FirecREST and Slurm cluster -Once all the images have been built we can move to the JupyterHub demo directory and deploy the [docker-compose.yaml](docker-compose.yaml). +Once all the images have been built we can move to the tutorial directory and deploy the [docker-compose.yaml](docker-compose.yaml). ```bash cd firecrest/examples/jupyterhub @@ -70,8 +70,8 @@ export JHUB_DOCKERFILE_DIR=$PWD docker compose -f ../../deploy/demo/docker-compose.yml -f docker-compose.yml up --build ``` -This step will create a new image that extends the `f7t-cluster` image from the FirecREST demo to include JupyterLab and other requirements. -It will take some time since it needs to first build from source a few dependencies of JupyterLab. +This step will create a new image that extends the `f7t-cluster` image from the Docker demo of FirecREST to include JupyterLab and other requirements. +It takes a few minutes since a few dependencies of JupyterLab must be built from source. Once the building is finished you can check that all containers are running @@ -110,7 +110,7 @@ docker compose -f ../../deploy/demo/docker-compose.yml -f docker-compose.yml dow ### Setting up the authorization A requirement for running JupyterHub with FirecRESTSpawner is to use an authenticator that prompts users for login and password in exchange for an access token. -That token will then be passed to the spawner, allowing users to authenticate with FirecREST when submitting, stopping or polling for jobs. +That token is then be passed to the spawner, allowing users to authenticate with FirecREST when submitting, stopping or polling for jobs. For this purpose, we will use an Authorization Code Flow client, which we need to create on the Keycloak web interface. Let's go to the [Clients page](http://localhost:8080/auth/admin/master/console/#/realms/kcrealm/clients) in Keycloak (username: admin, password: admin2) within the `kcrealm` realm. @@ -123,7 +123,7 @@ Once that's done, the client `jhub-client` can be seen listed on the "Clients" t ### Launching JupyterHub -The [configuration file](jupyterhub-config.py) provided in the demo has all the settings needed for using JupyterHub with our deployment. +The [configuration file](jupyterhub-config.py) provided in this tutorial has all the settings needed for using JupyterHub with our deployment. Depending on the platform and Docker setup, you may need to adjust a few lines in the configuration to set the correct host IP address for the Docker bridge network. On most Linux systems, you can find this address with `ip addr show docker0`. It is typically `172.17.0.1`, which you can use to replace `host.docker.internal` in the configuration if the latter doesn't work. Now we can run JupyterHub with @@ -134,7 +134,7 @@ Now we can run JupyterHub with jupyterhub --config jupyterhub-config.py --port 8003 --ip 0.0.0.0 ``` Here we are sourcing the file [env.sh](env.sh) which defines environment variables needed by the spawner (more information can be found [here](https://firecrestspawner.readthedocs.io/en/latest/authentication.html)). -We use the port `8003` for the hub since the default one `8000` is already used for FirecREST itself in the demo deployment. +We use the port `8003` for the hub since the default one `8000` is already used for FirecREST itself in the deployment. The ip `0.0.0.0` is necessary to allow JupyterLab to connect back to the hub. The hub should be accessible in the browser at [http://localhost:8003](http://localhost:8003/) (username: test1 and password: test11) and it should be possible to launch notebooks on the slurm cluster. From 2731ef1c23af19e8788108441ba05387e8ea581f Mon Sep 17 00:00:00 2001 From: Rafael Sarmiento Date: Thu, 21 Nov 2024 10:37:37 +0100 Subject: [PATCH 18/24] update --- examples/jupyterhub/README.md | 43 ++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/examples/jupyterhub/README.md b/examples/jupyterhub/README.md index bd8857e3..83f349c7 100644 --- a/examples/jupyterhub/README.md +++ b/examples/jupyterhub/README.md @@ -1,17 +1,25 @@ # FirecRESTSpawner on the Docker demo -This is a tutorial on how to run JupyterHub with [FirecRESTSpawner](https://github.com/eth-cscs/firecrestspawner) on the [Docker demo of FirecREST](https://github.com/eth-cscs/firecrest/tree/master/deploy/demo). +This tutorial explains how to run JupyterHub with [FirecRESTSpawner](https://github.com/eth-cscs/firecrestspawner) using the [Docker demo of FirecREST](https://github.com/eth-cscs/firecrest/tree/master/deploy/demo). -We are going to start by deploying FirecREST together with a slurm cluster using [Docker Compose](https://docs.docker.com/compose). -Then we will install JupyterHub on a virtual environment and configure it to launch notebooks on the slurm cluster via FirecREST. +FirecRESTSpawner is a tool for launching Jupyter Notebook servers from JupyterHub on HPC clusters through [FirecREST](https://firecrest.readthedocs.io/en/stable/). +It can be deployed on Kubernetes as part of JupyterHub and configured to target different systems. + +In this tutorial, we will set up a simplified environment on a local machine, including: + +- a [Docker Compose](https://docs.docker.com/compose) deployment of FirecREST and a single-node Slurm cluster +- a local installation of JupyterHub, configured to launch notebooks on the Slurm cluster + +This deployment not only demonstrates the use case but also serves as a platform for testing and developing FirecRESTSpawner. ## Requirements -For this tutorial it's necessary - * a recent installation of Docker, which includes the `docker compose` command or the older `docker-compose` command line - * a python installation (`>=3.9`) +For this tutorial you will need + + * a recent installation of Docker, which includes the `docker compose` command (or the older `docker-compose` command line tool) + * a Python installation (version 3.9 or higher) ## Setup @@ -19,8 +27,11 @@ For this tutorial it's necessary ### Building images from FirecREST's Docker Compose demo -The [docker-compose.yaml](docker-compose.yaml) that we use in this tutorial is a copy of the the one from the Docker demo of FirecREST with only a few small changes. -So, to get started, let's clone the FirecREST repository +This tutorial builds on the Docker demo of FirecREST. +We will use a small [docker-compose.yaml](docker-compose.yaml) file to override some settings in the FirecREST demo. +This can be done by passing both files to the `docker compose` command. + +To get started, let's clone the FirecREST repository ```bash git clone https://github.com/eth-cscs/firecrest.git @@ -33,13 +44,13 @@ cd firecrest/deploy/demo/ docker compose build ``` -This step takes a few minutes. In the meanwhile we can install JupyterHub on a virtual environment on our machine. +This step takes a few minutes. In the meanwhile we can install JupyterHub on a local virtual environment. ### Install JupyterHub and FirecRESTSpawner An easy way to install JupyterHub is via [Miniconda](https://docs.anaconda.com/miniconda/install/). -We need to download the Miniconda installer for our platforms and install it with +We need to download the Miniconda installer for our platforms and install it using the following command ```bash bash Miniconda3-latest-.sh -p /path/to/mc-jhub -b @@ -47,7 +58,7 @@ bash Miniconda3-latest-.sh -p /path/to/mc-jhub -b Here we use `-p` to pass the absolute path to the install directory and `-b` to accept the [terms of service](https://legal.anaconda.com/policies/en/). -Then we can activate our conda environment and install configurable-http-proxy, JupyterHub and FirecRESTSpawner +We can activate our conda base environment and install configurable-http-proxy, JupyterHub and FirecRESTSpawner ```bash . /path/to/mc-jhub/bin/activate @@ -71,9 +82,9 @@ docker compose -f ../../deploy/demo/docker-compose.yml -f docker-compose.yml up ``` This step will create a new image that extends the `f7t-cluster` image from the Docker demo of FirecREST to include JupyterLab and other requirements. -It takes a few minutes since a few dependencies of JupyterLab must be built from source. +The process may take a few minutes, as some dependencies for JupyterLab need to be built from source. -Once the building is finished you can check that all containers are running +Once that's finished, you can check that all containers are running ```bash docker compose -p demo ps --format 'table {{.ID}}\t{{.Name}}\t{{.State}}' @@ -134,7 +145,7 @@ Now we can run JupyterHub with jupyterhub --config jupyterhub-config.py --port 8003 --ip 0.0.0.0 ``` Here we are sourcing the file [env.sh](env.sh) which defines environment variables needed by the spawner (more information can be found [here](https://firecrestspawner.readthedocs.io/en/latest/authentication.html)). -We use the port `8003` for the hub since the default one `8000` is already used for FirecREST itself in the deployment. -The ip `0.0.0.0` is necessary to allow JupyterLab to connect back to the hub. +We use the port `8003` for the JupyterHub since the default one `8000` is already used for FirecREST in the deployment. +The ip `0.0.0.0` is necessary to allow JupyterLab to connect back to the JupyterHub. -The hub should be accessible in the browser at [http://localhost:8003](http://localhost:8003/) (username: test1 and password: test11) and it should be possible to launch notebooks on the slurm cluster. +JupyterHub should be accessible in the browser at [http://localhost:8003](http://localhost:8003/) (username: test1 and password: test11) and it should be possible to launch notebooks on the slurm cluster. From c69a1a77a7a96a8bda44a61c9b34b3c6bbba2da9 Mon Sep 17 00:00:00 2001 From: Rafael Sarmiento Date: Thu, 21 Nov 2024 15:12:02 +0100 Subject: [PATCH 19/24] import the spawner on the config --- examples/jupyterhub/jupyterhub-config.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/jupyterhub/jupyterhub-config.py b/examples/jupyterhub/jupyterhub-config.py index a5ec1044..81d3bac2 100644 --- a/examples/jupyterhub/jupyterhub-config.py +++ b/examples/jupyterhub/jupyterhub-config.py @@ -1,4 +1,5 @@ import secrets +from firecrestspawner.spawner import SlurmSpawner from oauthenticator.generic import GenericOAuthenticator @@ -21,7 +22,7 @@ def gen_hex_string(num_bytes=32, num_hex_strings=4): c.Authenticator.token_url = "http://localhost:8080/auth/realms/kcrealm/protocol/openid-connect/token" c.Authenticator.userdata_url = "http://localhost:8080/auth/realms/kcrealm/protocol/openid-connect/userinfo" c.Authenticator.login_service = "http://localhost:8080" -c.Authenticator.username_key = "preferred_username" +c.Authenticator.username_claim = "preferred_username" c.Authenticator.userdata_params = {"state": "state"} c.Authenticator.scope = ["openid", "profile", "firecrest"] @@ -36,7 +37,7 @@ def gen_hex_string(num_bytes=32, num_hex_strings=4): c.JupyterHub.default_url = "/hub/home" # Spawner setup -c.JupyterHub.spawner_class = "firecrestspawner.spawner.SlurmSpawner" +c.JupyterHub.spawner_class = SlurmSpawner c.Spawner.req_host = "cluster" c.Spawner.cmd = "firecrestspawner-singleuser jupyterhub-singleuser" c.Spawner.enable_aux_fc_client = True From 24f6af89fd60a6079532c31fc9a340cc66beb501 Mon Sep 17 00:00:00 2001 From: Rafael Sarmiento Date: Mon, 25 Nov 2024 14:25:34 +0100 Subject: [PATCH 20/24] fix typo --- examples/jupyterhub/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/jupyterhub/README.md b/examples/jupyterhub/README.md index 83f349c7..46cd3788 100644 --- a/examples/jupyterhub/README.md +++ b/examples/jupyterhub/README.md @@ -67,7 +67,6 @@ pip install --no-cache jupyterhub==4.1.6 pyfirecrest==2.6.0 SQLAlchemy==1.4.52 o git clone https://github.com/eth-cscs/firecrestspawner.git cd firecrestspawner -. /path/to/mc-jhub/bin/activate pip install --no-cache . ``` From e194b20e97d97a2024862f0f4f709e74a8b278ab Mon Sep 17 00:00:00 2001 From: Rafael Sarmiento Date: Mon, 25 Nov 2024 14:46:54 +0100 Subject: [PATCH 21/24] fix comments --- examples/jupyterhub/README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/jupyterhub/README.md b/examples/jupyterhub/README.md index 46cd3788..df253913 100644 --- a/examples/jupyterhub/README.md +++ b/examples/jupyterhub/README.md @@ -1,14 +1,14 @@ # FirecRESTSpawner on the Docker demo -This tutorial explains how to run JupyterHub with [FirecRESTSpawner](https://github.com/eth-cscs/firecrestspawner) using the [Docker demo of FirecREST](https://github.com/eth-cscs/firecrest/tree/master/deploy/demo). +This tutorial explains how to run [JupyterHub](https://jupyterhub.readthedocs.io/en/stable/) with [FirecRESTSpawner](https://github.com/eth-cscs/firecrestspawner) using the [Docker demo of FirecREST](https://github.com/eth-cscs/firecrest/tree/master/deploy/demo). FirecRESTSpawner is a tool for launching Jupyter Notebook servers from JupyterHub on HPC clusters through [FirecREST](https://firecrest.readthedocs.io/en/stable/). It can be deployed on Kubernetes as part of JupyterHub and configured to target different systems. In this tutorial, we will set up a simplified environment on a local machine, including: -- a [Docker Compose](https://docs.docker.com/compose) deployment of FirecREST and a single-node Slurm cluster +- a [Docker Compose](https://docs.docker.com/compose) deployment of FirecREST, a single-node Slurm cluster and a [Keycloak](https://www.keycloak.org) server which will be used as identity provider for the authentication - a local installation of JupyterHub, configured to launch notebooks on the Slurm cluster This deployment not only demonstrates the use case but also serves as a platform for testing and developing FirecRESTSpawner. @@ -50,7 +50,7 @@ This step takes a few minutes. In the meanwhile we can install JupyterHub on a l ### Install JupyterHub and FirecRESTSpawner An easy way to install JupyterHub is via [Miniconda](https://docs.anaconda.com/miniconda/install/). -We need to download the Miniconda installer for our platforms and install it using the following command +We need to [download the Miniconda installer](https://docs.anaconda.com/miniconda/install/) for our platforms and install it using the following command ```bash bash Miniconda3-latest-.sh -p /path/to/mc-jhub -b From 63315c64bf3545bff76b89e5752f1690e13cc098 Mon Sep 17 00:00:00 2001 From: Rafael Sarmiento Date: Mon, 25 Nov 2024 16:53:44 +0100 Subject: [PATCH 22/24] update docs --- doc/source/usecases.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doc/source/usecases.rst b/doc/source/usecases.rst index 8e6e340b..f3a0e9bd 100644 --- a/doc/source/usecases.rst +++ b/doc/source/usecases.rst @@ -20,3 +20,12 @@ FirecREST Operators for Airflow ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ In this `example `__ we define an Airflow graph combining small tasks which run localy in a laptop with compute-intensive tasks that must run on an HPC system. The idea is to add in Airflow the support for executing the compute-intensive tasks in a supercomputer via FirecREST. For that we are going to write custom Airflow operators that will use FirecREST to access the HPC system. + +JupyterHub with FirecRESTSpawner +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +This is a `tutorial `__ on how to run `JupyterHub `__ with `FirecRESTSpawner `__ using the `Docker demo of FirecREST `__. + +FirecRESTSpawner is a tool for launching Jupyter Notebook servers from JupyterHub on HPC clusters through FirecREST. +It can be deployed on Kubernetes as part of JupyterHub and configured to target different systems. +In this tutorial, we will set up a simplified environment on a local machine, including a `Docker Compose `__ deployment of FirecREST, a single-node Slurm cluster and a `Keycloak `__ server which will be used as identity provider for the authentication. Then we will install JupyterHub locally and configure it to launch notebooks on the Slurm cluster. From 73160fdc153fe7779ac7c6db0d0bb9a694db8836 Mon Sep 17 00:00:00 2001 From: Rafael Sarmiento Date: Mon, 25 Nov 2024 17:57:52 +0100 Subject: [PATCH 23/24] add chmod command for keys --- examples/jupyterhub/README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/jupyterhub/README.md b/examples/jupyterhub/README.md index df253913..d3e656b6 100644 --- a/examples/jupyterhub/README.md +++ b/examples/jupyterhub/README.md @@ -76,10 +76,13 @@ Once all the images have been built we can move to the tutorial directory and de ```bash cd firecrest/examples/jupyterhub +chmod 400 ../../deploy/test-build/environment/keys/ca-key ../../deploy/test-build/environment/keys/user-key export JHUB_DOCKERFILE_DIR=$PWD docker compose -f ../../deploy/demo/docker-compose.yml -f docker-compose.yml up --build ``` +The `chmod` command we run before `docker compose` is to make the SSH private keys to access the Slurm cluster (`ca-key` and `user-key`) readable by its owner on the host machine. + This step will create a new image that extends the `f7t-cluster` image from the Docker demo of FirecREST to include JupyterLab and other requirements. The process may take a few minutes, as some dependencies for JupyterLab need to be built from source. From 3dffa8a954d0a047b9d724949faff15b4e95130a Mon Sep 17 00:00:00 2001 From: Rafael Sarmiento Date: Tue, 26 Nov 2024 11:55:25 +0100 Subject: [PATCH 24/24] update docs --- examples/jupyterhub/README.md | 9 +++++++-- examples/jupyterhub/jupyterhub-config.py | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/examples/jupyterhub/README.md b/examples/jupyterhub/README.md index d3e656b6..bfad2d84 100644 --- a/examples/jupyterhub/README.md +++ b/examples/jupyterhub/README.md @@ -81,7 +81,8 @@ export JHUB_DOCKERFILE_DIR=$PWD docker compose -f ../../deploy/demo/docker-compose.yml -f docker-compose.yml up --build ``` -The `chmod` command we run before `docker compose` is to make the SSH private keys to access the Slurm cluster (`ca-key` and `user-key`) readable by its owner on the host machine. +The `chmod` command we run before `docker compose` comes from the Docker demo of FirecREST. +It's needed to make the SSH private keys for accessing the Slurm cluster readable by their owner on the host machine. This step will create a new image that extends the `f7t-cluster` image from the Docker demo of FirecREST to include JupyterLab and other requirements. The process may take a few minutes, as some dependencies for JupyterLab need to be built from source. @@ -137,7 +138,11 @@ Once that's done, the client `jhub-client` can be seen listed on the "Clients" t ### Launching JupyterHub The [configuration file](jupyterhub-config.py) provided in this tutorial has all the settings needed for using JupyterHub with our deployment. -Depending on the platform and Docker setup, you may need to adjust a few lines in the configuration to set the correct host IP address for the Docker bridge network. On most Linux systems, you can find this address with `ip addr show docker0`. It is typically `172.17.0.1`, which you can use to replace `host.docker.internal` in the configuration if the latter doesn't work. + +> Depending on the platform and Docker setup, you may need to adjust a few lines in the configuration to set the correct host IP address for the Docker bridge network. +> On most Linux systems, you can find this address with `ip addr show docker0`. +> It's typically `172.17.0.1`. +> If JupyterHub gets a timeout when launching a notebook, you can try replacing the two instances of `host.docker.internal` in the configuration by that ip. Now we can run JupyterHub with diff --git a/examples/jupyterhub/jupyterhub-config.py b/examples/jupyterhub/jupyterhub-config.py index 81d3bac2..dc288b01 100644 --- a/examples/jupyterhub/jupyterhub-config.py +++ b/examples/jupyterhub/jupyterhub-config.py @@ -47,7 +47,7 @@ def gen_hex_string(num_bytes=32, num_hex_strings=4): #SBATCH --job-name=jhub export JUPYTERHUB_API_URL="http://host.docker.internal:8003/hub/api" -export JUPYTERHUB_ACTIVITY_URL="http://host.docker.internal:8003//hub/api/users/${USER}/activity" +export JUPYTERHUB_ACTIVITY_URL="http://host.docker.internal:8003/hub/api/users/${USER}/activity" export JUPYTERHUB_OAUTH_ACCESS_SCOPES=$(echo $JUPYTERHUB_OAUTH_ACCESS_SCOPES | base64 --decode) export JUPYTERHUB_OAUTH_SCOPES=$(echo $JUPYTERHUB_OAUTH_SCOPES | base64 --decode)