diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 24fe1ec..05bfec4 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -17,7 +17,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: 3.12.3 + python-version: 3.12.3 # TODO: what version is on the server? - name: Install dependencies # https://docs.github.com/en/free-pro-team@latest/actions/guides/building-and-testing-python#installing-dependencies diff --git a/README.md b/README.md index 0a1588b..ef2d40b 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,6 @@ The configuration usage information is stored in `config/.last_deploy.` ## Deployment configurations There are three configurations -- noproxy: for local testing +- fakeproxy: for local testing - staging: for deployment onto our staging server at `daedalus.dev.dide.ic.ac.uk` - prod: tbd! \ No newline at end of file diff --git a/config/daedalus.yml b/config/daedalus.yml index a0e90aa..adf1a56 100644 --- a/config/daedalus.yml +++ b/config/daedalus.yml @@ -1,18 +1,29 @@ docker: network: daedalus + prefix: daedalus +proxy: + image: + repo: ghcr.io/jameel-institute + name: daedalus-proxy + # TODO: update to main when merged + tag: jidea-60 + host: localhost + port_http: 80 + port_https: 443 api: image: repo: mrcide name: daedalus-api tag: latest port: 8001 -db: +web_app_db: image: repo: ghcr.io/jameel-institute name: daedalus-web-app-db # TODO: update to main when merged tag: jidea-59-dockerise-web-app port: 5432 + data_location: /pgdata web_app: image: repo: ghcr/jameel-institute diff --git a/config/fakeproxy.yml b/config/fakeproxy.yml new file mode 100644 index 0000000..97be2ab --- /dev/null +++ b/config/fakeproxy.yml @@ -0,0 +1 @@ +# uses fake proxy by not specifying ssl vault keys diff --git a/config/noproxy.yml b/config/noproxy.yml deleted file mode 100644 index 1e144df..0000000 --- a/config/noproxy.yml +++ /dev/null @@ -1 +0,0 @@ -# Default config only for now diff --git a/config/staging.yml b/config/staging.yml index 790abbc..34e764f 100644 --- a/config/staging.yml +++ b/config/staging.yml @@ -1,10 +1,8 @@ proxy: - image: - repo: ghcr.io/jameel-institute - name: daedalus-proxy - # TODO: update to main when merged - tag: jidea-50 - host: beebop-dev.dide.ic.ac.uk ssl: certificate: VAULT:secret/daedalus/ssl/staging:cert - key: VAULT:secret/beebop/daedalus/ssl/staging:key \ No newline at end of file + key: VAULT:secret/daedalus/daedalus/ssl/staging:key +vault: + addr: https://vault.dide.ic.ac.uk:8200 + auth: + method: github \ No newline at end of file diff --git a/src/daedalus_cli.py b/src/daedalus_cli.py new file mode 100644 index 0000000..cf6431d --- /dev/null +++ b/src/daedalus_cli.py @@ -0,0 +1,111 @@ +""" +Usage: + ./daedalus start [--pull] [] + ./daedalus stop [--volumes] [--network] [--kill] [--force] + ./daedalus destroy + ./daedalus status + ./daedalus upgrade + +Options: + --pull Pull images before starting + --volumes Remove volumes (WARNING: irreversible data loss) + --network Remove network + --kill Kill the containers (faster, but possible db corruption) +""" + +import docopt +import os +import os.path +import pickle +import time +import timeago + +from src.daedalus_deploy import \ + DaedalusConfig, \ + daedalus_constellation, \ + daedalus_start + + +def parse(argv=None): + path = "config" + config_name = None + dat = docopt.docopt(__doc__, argv) + if dat["start"]: + action = "start" + config_name = dat[""] + args = {"pull_images": dat["--pull"]} + options = {} + elif dat["stop"]: + action = "stop" + args = {"kill": dat["--kill"], + "remove_network": dat["--network"], + "remove_volumes": dat["--volumes"]} + options = {} + elif dat["destroy"]: + action = "stop" + args = {"kill": True, + "remove_network": True, + "remove_volumes": True} + options = {} + elif dat["status"]: + action = "status" + args = {} + options = {} + elif dat["upgrade"]: + args = {} + options = {} + action = "upgrade" + return path, config_name, action, args, options + + +def path_last_deploy(path): + return path + "/.last_deploy" + + +def save_config(path, config_name, cfg): + dat = {"config_name": config_name, + "time": time.time(), + "data": cfg} + with open(path_last_deploy(path), "wb") as f: + pickle.dump(dat, f) + + +def read_config(path): + with open(path_last_deploy(path), "rb") as f: + dat = pickle.load(f) + return dat + + +def load_config(path, config_name=None, options=None): + if os.path.exists(path_last_deploy(path)): + dat = read_config(path) + when = timeago.format(dat["time"]) + cfg = DaedalusConfig(path, dat["config_name"], options=options) + config_name = dat["config_name"] + print("[Loaded configuration '{}' ({})]".format( + config_name or "", when)) + else: + cfg = DaedalusConfig(path, config_name, options=options) + return config_name, cfg + + +def remove_config(path): + p = path_last_deploy(path) + if os.path.exists(p): + print("Removing configuration") + os.unlink(p) + + +def main(argv=None): + path, config_name, action, args, options = parse(argv) + config_name, cfg = load_config(path, config_name, options) + obj = daedalus_constellation(cfg) + if action == "upgrade": + obj.restart(pull_images=True) + elif action == "start": + save_config(path, config_name, cfg) + daedalus_start(obj, args) + else: + obj.__getattribute__(action)(**args) + if action == "stop" and args["remove_volumes"]: + remove_config(path) diff --git a/src/daedalus_deploy.py b/src/daedalus_deploy.py new file mode 100644 index 0000000..25329d2 --- /dev/null +++ b/src/daedalus_deploy.py @@ -0,0 +1,123 @@ +import time +import os + +import docker +import json +import constellation +import constellation.config as config +import constellation.docker_util as docker_util + +def get_image_reference(config_section, dat): + repo = config.config_string( + dat, [config_section, "image", "repo"]) + name = config.config_string( + dat, [config_section, "image", "name"]) + tag = config.config_string( + dat, [config_section, "image", "tag"]) + return self.proxy_ref = constellation.ImageReference(repo, name, tag) + +class DaedalusConfig: + def __init__(self, path, config_name=None, options=None): + dat = config.read_yaml("{}/daedalus.yml".format(path)) + dat = config.config_build(path, dat, config_name, options=options) + self.path = path + self.data = dat + self.vault = config.config_vault(dat, ["vault"]) + self.network = config.config_string(dat, ["docker", "network"]) + self.container_prefix = config.config_string(dat, ["docker", "prefix"]) + + # TODO: do we need this?? + self.containers = { + "api": "api", + "web_app_db": "web_app_db", + "web_app": "web_app", + "proxy": "proxy" + } + + self.volumes = { + "daedalus_data": "daedalus_data" + } + + # api + self.api_ref = get_image_reference("api") + self.api_port = config.config_string(dat, ["proxy", "port"]) + + # web_app_db + self.web_app_db_ref = get_image_reference("web_app_db") + self.web_app_db_port = config.config_string(dat, ["web_app_db", "port"]) + self.web_app_db_data_location = config.config_string(dat, ["web_app_db", "data_location"]) + + # web_app + self.web_app_ref = get_image_reference("web_app") + self.web_app_port = config.config_string(dat, ["web_app", "port"]) + + # proxy + self.proxy_ref = get_image_reference("proxy") + self.proxy_host = config.config_string(dat, ["proxy", "host"]) + self.proxy_port_http = config.config_integer(dat, + ["proxy", "port_http"]) + self.proxy_port_https = config.config_integer(dat, + ["proxy", "port_https"]) + if "ssl" in dat["proxy"]: + self.proxy_ssl_certificate = config.config_string(dat, + ["proxy", + "ssl", + "certificate"]) + self.proxy_ssl_key = config.config_string(dat, + ["proxy", + "ssl", + "key"]) + self.ssl = True + else: + self.ssl = False + + +def daedalus_constellation(cfg): + # 1. api + api = constellation.ConstellationContainer("api", cfg.api_ref) + + # 2. web_app_db + # TODO: non-default db credentials + web_app_db_mounts = [constellation.ConstellationMount("daedalus_data", cfg.web_app_db_data_location)] + web_app_db_env = {"PGDATA", cfg.web_app_db_data_location} + web_app_db = constellation.ConstellationContainer( + "web_app_db", cfg.web_app_db_ref, environment=web_app_db_env, mounts=web_app_db_mounts) + + # 3. web_app + web_app_env = { + "DATABASE_URL": "postgresql://daedalus-web-app-user:changeme@daedalus-web-app-db:{}/daedalus-web-app".format(cfg.web_app_db_port), + "NUXT_R_API_BASE": "http://daedalus-api:{}/".format(cfg.api_port) + } + web_app = constellation.ConstellationContainer("web_app", cfg.web_app_ref, environment=web_app_env) + + # 4. proxy + proxy_ports = [cfg.proxy_port_http, cfg.proxy_port_https] + proxy = constellation.ConstellationContainer( + "proxy", cfg.proxy_ref, ports=proxy_ports, configure=proxy_configure, + args=[cfg.proxy_host, web_app.name]) + + containers = [api, web_app_db, web_app, proxy] + + obj = constellation.Constellation("daedalus", cfg.container_prefix, + containers, + cfg.network, cfg.volumes, + data=cfg, vault_config=cfg.vault) + return obj + + +def daedalus_start(obj, args): + obj.start(**args) + +def proxy_configure(container, cfg): + print("[proxy] Configuring proxy") + if cfg.ssl: + print("Copying ssl certificate and key into proxy") + docker_util.string_into_container(cfg.proxy_ssl_certificate, container, + "/run/proxy/certificate.pem") + docker_util.string_into_container(cfg.proxy_ssl_key, container, + "/run/proxy/key.pem") + else: + print("Generating self-signed certificates for proxy") + args = ["/usr/local/bin/build-self-signed-certificate", "/run/proxy", + "GB", "London", "IC", "bacpop", cfg.proxy_host] + docker_util.exec_safely(container, args)