From 67733da192e323e2f403095717f08b9acdeacbae Mon Sep 17 00:00:00 2001
From: Stef Piatek <s.piatek@ucl.ac.uk>
Date: Tue, 10 Dec 2024 16:09:47 +0000
Subject: [PATCH] Improve docker build speed and caching (#118)

---
 .dockerignore            |  4 ++--
 .env.sample              |  1 +
 app/Dockerfile           | 33 +++++++++++++++++++++++++++------
 docker-compose.yml       |  2 ++
 preprocessing/Dockerfile | 32 +++++++++++++++++++++++++++-----
 public.env.sample        |  1 +
 6 files changed, 60 insertions(+), 13 deletions(-)

diff --git a/.dockerignore b/.dockerignore
index 8ef521d..e26a3a1 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -1,3 +1,3 @@
-**/renv
-**/.Rprofile
+**/renv/library
+**/renv/staging
 **/.Renviron
diff --git a/.env.sample b/.env.sample
index c3c496b..9d878a9 100644
--- a/.env.sample
+++ b/.env.sample
@@ -11,6 +11,7 @@ PREPROCESS_DB_USERNAME=             # username for the source database
 PREPROCESS_DB_PASSWORD=             # password for the source database
 PREPROCESS_DB_CDM_SCHEMA=           # Schema name in the database to connect the OMOP CDM to
 PREPROCESS_SUMMARISE_LEVEL=monthly  # Level to summarise record counts at (monthly or quarterly)
+BUILD_CORES_MAKE=4                  # Number of cores to use for make in docker build
 
 # Low-frequency replacement
 LOW_FREQUENCY_THRESHOLD=10
diff --git a/app/Dockerfile b/app/Dockerfile
index b87de04..a283542 100644
--- a/app/Dockerfile
+++ b/app/Dockerfile
@@ -1,19 +1,40 @@
-FROM rocker/shiny-verse:4.4.1
+# Stage 1: installing renv environment
+FROM rocker/shiny-verse:4.4.1 AS base
 
 WORKDIR /app
-ADD app .
-COPY app/renv.lock ./renv.lock
+
+# Speed up building by setting make with multiple cores from env
+ARG CORES
+ENV MAKE="make -j${CORES}"
+
+RUN mkdir -p renv
+COPY app/renv.lock renv.lock
+COPY app/.Rprofile .Rprofile
+COPY app/renv/activate.R renv/activate.R
+COPY app/renv/settings.json renv/settings.json
+
+# change default location of cache to project folder
+RUN mkdir renv/.cache
+ENV RENV_PATHS_CACHE renv/.cache
 
 # Install renv and restore environment
 # omopbundles is installed separately as renv is giving problems
 # with GitHub packages
-RUN install2.r --error --skipinstalled renv remotes && \
+RUN install2.r --error --skipinstalled renv devtools remotes && \
     R -e 'renv::restore(exclude = "omopbundles")' && \
     rm -rf /tmp/downloaded_packages
 RUN R -e 'remotes::install_github("SAFEHR-data/omop-bundles")'
 
-# Install the app
-RUN R -e 'remotes::install_local(".", dependencies = TRUE)'
+# Stage 2: Installing omopcat
+FROM rocker/shiny-verse:4.4.1
+
+WORKDIR /app
+COPY --from=base /app .
+
+# Install omopcat package
+# Put package in a subdirectory to avoid overwriting renv files from previous stage
+COPY app ./pkg
+RUN R -e 'devtools::install("pkg", dependencies = FALSE)'
 
 EXPOSE 3838
 CMD ["R", "-e", "options('shiny.port'=3838, shiny.host='0.0.0.0'); omopcat::run_app()" ]
diff --git a/docker-compose.yml b/docker-compose.yml
index 2bc2a52..9f7a37c 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -7,6 +7,7 @@ services:
         # Required for running on GAE
         HTTP_PROXY: ${HTTP_PROXY}
         HTTPS_PROXY: ${HTTPS_PROXY}
+        CORES: ${BUILD_CORES_MAKE}
     image: omopcat_preprocessing:latest
     platform: linux/amd64
     environment:
@@ -36,6 +37,7 @@ services:
         # Required for running on GAE
         HTTP_PROXY: ${HTTP_PROXY}
         HTTPS_PROXY: ${HTTPS_PROXY}
+        CORES: ${BUILD_CORES_MAKE}
     image: omopcat:latest
     platform: linux/amd64
     restart: unless-stopped
diff --git a/preprocessing/Dockerfile b/preprocessing/Dockerfile
index e277e02..65c507f 100644
--- a/preprocessing/Dockerfile
+++ b/preprocessing/Dockerfile
@@ -1,13 +1,35 @@
-FROM rocker/tidyverse:4.4.1
+# Stage 1: installing renv environment
+FROM rocker/tidyverse:4.4.1 AS base
 
 WORKDIR /pkg
-ADD preprocessing .
-COPY preprocessing/renv.lock ./renv.lock
 
-RUN install2.r --error --skipinstalled renv remotes && \
+# Speed up building by setting make with multiple cores from env
+ARG CORES
+ENV MAKE="make -j${CORES}"
+
+RUN mkdir -p renv
+COPY preprocessing/renv.lock renv.lock
+COPY preprocessing/.Rprofile .Rprofile
+COPY preprocessing/renv/activate.R renv/activate.R
+COPY preprocessing/renv/settings.json renv/settings.json
+
+# change default location of cache to project folder
+RUN mkdir renv/.cache
+ENV RENV_PATHS_CACHE renv/.cache
+
+RUN install2.r --error --skipinstalled renv && \
     R -e 'renv::restore()' && \
     rm -rf /tmp/downloaded_packages
 
-RUN R -e 'remotes::install_local(path = ".", dependencies = TRUE)'
+# Stage 2: Installing omopcat.preprocessing
+FROM rocker/tidyverse:4.4.1
+
+WORKDIR /pkg
+COPY --from=base /pkg .
+
+# Install omopcat.preprocessing package
+# Put package in a subdirectory to avoid overwriting renv files from previous stage
+COPY preprocessing ./preprocessing
+RUN R -e 'devtools::install("preprocessing", dependencies = FALSE)'
 
 CMD ["R", "-e", "omopcat.preprocessing::preprocess()"]
diff --git a/public.env.sample b/public.env.sample
index e9b0c46..9b394e0 100644
--- a/public.env.sample
+++ b/public.env.sample
@@ -11,6 +11,7 @@ PREPROCESS_DB_USERNAME=             # username for the source database
 PREPROCESS_DB_PASSWORD=             # password for the source database
 PREPROCESS_DB_CDM_SCHEMA=           # Schema name in the database to connect the OMOP CDM to
 PREPROCESS_SUMMARISE_LEVEL=quarterly  # Level to summarise record counts at (monthly or quarterly)
+BUILD_CORES_MAKE=4                  # Number of cores to use for make in docker build
 
 # Low-frequency replacement
 LOW_FREQUENCY_THRESHOLD=10