Skip to content

Commit

Permalink
first attempt at daedalus constellation
Browse files Browse the repository at this point in the history
EmmaLRussell committed Aug 20, 2024
1 parent 9638768 commit 3372225
Showing 8 changed files with 254 additions and 11 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/pytest.yml
Original file line number Diff line number Diff line change
@@ -17,7 +17,7 @@ jobs:
- name: Set up Python
uses: actions/setup-python@v5
with:
python-version: 3.12.3
python-version: 3.12.3 # TODO: what version is on the server?

- name: Install dependencies
# https://docs.github.com/en/free-pro-team@latest/actions/guides/building-and-testing-python#installing-dependencies
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
@@ -30,6 +30,6 @@ The configuration usage information is stored in `config/.last_deploy.`

## Deployment configurations
There are three configurations
- noproxy: for local testing
- fakeproxy: for local testing
- staging: for deployment onto our staging server at `daedalus.dev.dide.ic.ac.uk`
- prod: tbd!
13 changes: 12 additions & 1 deletion config/daedalus.yml
Original file line number Diff line number Diff line change
@@ -1,18 +1,29 @@
docker:
network: daedalus
prefix: daedalus
proxy:
image:
repo: ghcr.io/jameel-institute
name: daedalus-proxy
# TODO: update to main when merged
tag: jidea-60
host: localhost
port_http: 80
port_https: 443
api:
image:
repo: mrcide
name: daedalus-api
tag: latest
port: 8001
db:
web_app_db:
image:
repo: ghcr.io/jameel-institute
name: daedalus-web-app-db
# TODO: update to main when merged
tag: jidea-59-dockerise-web-app
port: 5432
data_location: /pgdata
web_app:
image:
repo: ghcr/jameel-institute
1 change: 1 addition & 0 deletions config/fakeproxy.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
# uses fake proxy by not specifying ssl vault keys
1 change: 0 additions & 1 deletion config/noproxy.yml

This file was deleted.

12 changes: 5 additions & 7 deletions config/staging.yml
Original file line number Diff line number Diff line change
@@ -1,10 +1,8 @@
proxy:
image:
repo: ghcr.io/jameel-institute
name: daedalus-proxy
# TODO: update to main when merged
tag: jidea-50
host: beebop-dev.dide.ic.ac.uk
ssl:
certificate: VAULT:secret/daedalus/ssl/staging:cert
key: VAULT:secret/beebop/daedalus/ssl/staging:key
key: VAULT:secret/daedalus/daedalus/ssl/staging:key
vault:
addr: https://vault.dide.ic.ac.uk:8200
auth:
method: github
111 changes: 111 additions & 0 deletions src/daedalus_cli.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
"""
Usage:
./daedalus start [--pull] [<configname>]
./daedalus stop [--volumes] [--network] [--kill] [--force]
./daedalus destroy
./daedalus status
./daedalus upgrade
Options:
--pull Pull images before starting
--volumes Remove volumes (WARNING: irreversible data loss)
--network Remove network
--kill Kill the containers (faster, but possible db corruption)
"""

import docopt
import os
import os.path
import pickle
import time
import timeago

from src.daedalus_deploy import \
DaedalusConfig, \
daedalus_constellation, \
daedalus_start


def parse(argv=None):
path = "config"
config_name = None
dat = docopt.docopt(__doc__, argv)
if dat["start"]:
action = "start"
config_name = dat["<configname>"]
args = {"pull_images": dat["--pull"]}
options = {}
elif dat["stop"]:
action = "stop"
args = {"kill": dat["--kill"],
"remove_network": dat["--network"],
"remove_volumes": dat["--volumes"]}
options = {}
elif dat["destroy"]:
action = "stop"
args = {"kill": True,
"remove_network": True,
"remove_volumes": True}
options = {}
elif dat["status"]:
action = "status"
args = {}
options = {}
elif dat["upgrade"]:
args = {}
options = {}
action = "upgrade"
return path, config_name, action, args, options


def path_last_deploy(path):
return path + "/.last_deploy"


def save_config(path, config_name, cfg):
dat = {"config_name": config_name,
"time": time.time(),
"data": cfg}
with open(path_last_deploy(path), "wb") as f:
pickle.dump(dat, f)


def read_config(path):
with open(path_last_deploy(path), "rb") as f:
dat = pickle.load(f)
return dat


def load_config(path, config_name=None, options=None):
if os.path.exists(path_last_deploy(path)):
dat = read_config(path)
when = timeago.format(dat["time"])
cfg = DaedalusConfig(path, dat["config_name"], options=options)
config_name = dat["config_name"]
print("[Loaded configuration '{}' ({})]".format(
config_name or "<base>", when))
else:
cfg = DaedalusConfig(path, config_name, options=options)
return config_name, cfg


def remove_config(path):
p = path_last_deploy(path)
if os.path.exists(p):
print("Removing configuration")
os.unlink(p)


def main(argv=None):
path, config_name, action, args, options = parse(argv)
config_name, cfg = load_config(path, config_name, options)
obj = daedalus_constellation(cfg)
if action == "upgrade":
obj.restart(pull_images=True)
elif action == "start":
save_config(path, config_name, cfg)
daedalus_start(obj, args)
else:
obj.__getattribute__(action)(**args)
if action == "stop" and args["remove_volumes"]:
remove_config(path)
123 changes: 123 additions & 0 deletions src/daedalus_deploy.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
import time
import os

import docker
import json
import constellation
import constellation.config as config
import constellation.docker_util as docker_util

def get_image_reference(config_section, dat):
repo = config.config_string(
dat, [config_section, "image", "repo"])
name = config.config_string(
dat, [config_section, "image", "name"])
tag = config.config_string(
dat, [config_section, "image", "tag"])
return self.proxy_ref = constellation.ImageReference(repo, name, tag)

class DaedalusConfig:
def __init__(self, path, config_name=None, options=None):
dat = config.read_yaml("{}/daedalus.yml".format(path))
dat = config.config_build(path, dat, config_name, options=options)
self.path = path
self.data = dat
self.vault = config.config_vault(dat, ["vault"])
self.network = config.config_string(dat, ["docker", "network"])
self.container_prefix = config.config_string(dat, ["docker", "prefix"])

# TODO: do we need this??
self.containers = {
"api": "api",
"web_app_db": "web_app_db",
"web_app": "web_app",
"proxy": "proxy"
}

self.volumes = {
"daedalus_data": "daedalus_data"
}

# api
self.api_ref = get_image_reference("api")
self.api_port = config.config_string(dat, ["proxy", "port"])

# web_app_db
self.web_app_db_ref = get_image_reference("web_app_db")
self.web_app_db_port = config.config_string(dat, ["web_app_db", "port"])
self.web_app_db_data_location = config.config_string(dat, ["web_app_db", "data_location"])

# web_app
self.web_app_ref = get_image_reference("web_app")
self.web_app_port = config.config_string(dat, ["web_app", "port"])

# proxy
self.proxy_ref = get_image_reference("proxy")
self.proxy_host = config.config_string(dat, ["proxy", "host"])
self.proxy_port_http = config.config_integer(dat,
["proxy", "port_http"])
self.proxy_port_https = config.config_integer(dat,
["proxy", "port_https"])
if "ssl" in dat["proxy"]:
self.proxy_ssl_certificate = config.config_string(dat,
["proxy",
"ssl",
"certificate"])
self.proxy_ssl_key = config.config_string(dat,
["proxy",
"ssl",
"key"])
self.ssl = True
else:
self.ssl = False


def daedalus_constellation(cfg):
# 1. api
api = constellation.ConstellationContainer("api", cfg.api_ref)

# 2. web_app_db
# TODO: non-default db credentials
web_app_db_mounts = [constellation.ConstellationMount("daedalus_data", cfg.web_app_db_data_location)]
web_app_db_env = {"PGDATA", cfg.web_app_db_data_location}
web_app_db = constellation.ConstellationContainer(
"web_app_db", cfg.web_app_db_ref, environment=web_app_db_env, mounts=web_app_db_mounts)

# 3. web_app
web_app_env = {
"DATABASE_URL": "postgresql://daedalus-web-app-user:changeme@daedalus-web-app-db:{}/daedalus-web-app".format(cfg.web_app_db_port),
"NUXT_R_API_BASE": "http://daedalus-api:{}/".format(cfg.api_port)
}
web_app = constellation.ConstellationContainer("web_app", cfg.web_app_ref, environment=web_app_env)

# 4. proxy
proxy_ports = [cfg.proxy_port_http, cfg.proxy_port_https]
proxy = constellation.ConstellationContainer(
"proxy", cfg.proxy_ref, ports=proxy_ports, configure=proxy_configure,
args=[cfg.proxy_host, web_app.name])

containers = [api, web_app_db, web_app, proxy]

obj = constellation.Constellation("daedalus", cfg.container_prefix,
containers,
cfg.network, cfg.volumes,
data=cfg, vault_config=cfg.vault)
return obj


def daedalus_start(obj, args):
obj.start(**args)

def proxy_configure(container, cfg):
print("[proxy] Configuring proxy")
if cfg.ssl:
print("Copying ssl certificate and key into proxy")
docker_util.string_into_container(cfg.proxy_ssl_certificate, container,
"/run/proxy/certificate.pem")
docker_util.string_into_container(cfg.proxy_ssl_key, container,
"/run/proxy/key.pem")
else:
print("Generating self-signed certificates for proxy")
args = ["/usr/local/bin/build-self-signed-certificate", "/run/proxy",
"GB", "London", "IC", "bacpop", cfg.proxy_host]
docker_util.exec_safely(container, args)

0 comments on commit 3372225

Please sign in to comment.