Improve docker build speed and caching (#118)

.dockerignore

@@ -1,3 +1,3 @@
-**/renv
-**/.Rprofile
+**/renv/library
+**/renv/staging
 **/.Renviron

.env.sample

@@ -11,6 +11,7 @@ PREPROCESS_DB_USERNAME=             # username for the source database
 PREPROCESS_DB_PASSWORD=             # password for the source database
 PREPROCESS_DB_CDM_SCHEMA=           # Schema name in the database to connect the OMOP CDM to
 PREPROCESS_SUMMARISE_LEVEL=monthly  # Level to summarise record counts at (monthly or quarterly)
+BUILD_CORES_MAKE=4                  # Number of cores to use for make in docker build
 
 # Low-frequency replacement
 LOW_FREQUENCY_THRESHOLD=10

app/Dockerfile

@@ -1,19 +1,40 @@
-FROM rocker/shiny-verse:4.4.1
+# Stage 1: installing renv environment
+FROM rocker/shiny-verse:4.4.1 AS base
 
 WORKDIR /app
-ADD app .
-COPY app/renv.lock ./renv.lock
+
+# Speed up building by setting make with multiple cores from env
+ARG CORES
+ENV MAKE="make -j${CORES}"
+
+RUN mkdir -p renv
+COPY app/renv.lock renv.lock
+COPY app/.Rprofile .Rprofile
+COPY app/renv/activate.R renv/activate.R
+COPY app/renv/settings.json renv/settings.json
+
+# change default location of cache to project folder
+RUN mkdir renv/.cache
+ENV RENV_PATHS_CACHE renv/.cache
 
 # Install renv and restore environment
 # omopbundles is installed separately as renv is giving problems
 # with GitHub packages
-RUN install2.r --error --skipinstalled renv remotes && \
+RUN install2.r --error --skipinstalled renv devtools remotes && \
     R -e 'renv::restore(exclude = "omopbundles")' && \
     rm -rf /tmp/downloaded_packages
 RUN R -e 'remotes::install_github("SAFEHR-data/omop-bundles")'
 
-# Install the app
-RUN R -e 'remotes::install_local(".", dependencies = TRUE)'
+# Stage 2: Installing omopcat
+FROM rocker/shiny-verse:4.4.1
+
+WORKDIR /app
+COPY --from=base /app .
+
+# Install omopcat package
+# Put package in a subdirectory to avoid overwriting renv files from previous stage
+COPY app ./pkg
+RUN R -e 'devtools::install("pkg", dependencies = FALSE)'
 
 EXPOSE 3838
 CMD ["R", "-e", "options('shiny.port'=3838, shiny.host='0.0.0.0'); omopcat::run_app()" ]

docker-compose.yml

@@ -7,6 +7,7 @@ services:
         # Required for running on GAE
         HTTP_PROXY: ${HTTP_PROXY}
         HTTPS_PROXY: ${HTTPS_PROXY}
+        CORES: ${BUILD_CORES_MAKE}
     image: omopcat_preprocessing:latest
     platform: linux/amd64
     environment:
@@ -36,6 +37,7 @@ services:
         # Required for running on GAE
         HTTP_PROXY: ${HTTP_PROXY}
         HTTPS_PROXY: ${HTTPS_PROXY}
+        CORES: ${BUILD_CORES_MAKE}
     image: omopcat:latest
     platform: linux/amd64
     restart: unless-stopped

preprocessing/Dockerfile

@@ -1,13 +1,35 @@
-FROM rocker/tidyverse:4.4.1
+# Stage 1: installing renv environment
+FROM rocker/tidyverse:4.4.1 AS base
 
 WORKDIR /pkg
-ADD preprocessing .
-COPY preprocessing/renv.lock ./renv.lock
 
-RUN install2.r --error --skipinstalled renv remotes && \
+# Speed up building by setting make with multiple cores from env
+ARG CORES
+ENV MAKE="make -j${CORES}"
+
+RUN mkdir -p renv
+COPY preprocessing/renv.lock renv.lock
+COPY preprocessing/.Rprofile .Rprofile
+COPY preprocessing/renv/activate.R renv/activate.R
+COPY preprocessing/renv/settings.json renv/settings.json
+
+# change default location of cache to project folder
+RUN mkdir renv/.cache
+ENV RENV_PATHS_CACHE renv/.cache
+
+RUN install2.r --error --skipinstalled renv && \
     R -e 'renv::restore()' && \
     rm -rf /tmp/downloaded_packages
 
-RUN R -e 'remotes::install_local(path = ".", dependencies = TRUE)'
+# Stage 2: Installing omopcat.preprocessing
+FROM rocker/tidyverse:4.4.1
+
+WORKDIR /pkg
+COPY --from=base /pkg .
+
+# Install omopcat.preprocessing package
+# Put package in a subdirectory to avoid overwriting renv files from previous stage
+COPY preprocessing ./preprocessing
+RUN R -e 'devtools::install("preprocessing", dependencies = FALSE)'
 
 CMD ["R", "-e", "omopcat.preprocessing::preprocess()"]

public.env.sample

@@ -11,6 +11,7 @@ PREPROCESS_DB_USERNAME=             # username for the source database
 PREPROCESS_DB_PASSWORD=             # password for the source database
 PREPROCESS_DB_CDM_SCHEMA=           # Schema name in the database to connect the OMOP CDM to
 PREPROCESS_SUMMARISE_LEVEL=quarterly  # Level to summarise record counts at (monthly or quarterly)
+BUILD_CORES_MAKE=4                  # Number of cores to use for make in docker build
 
 # Low-frequency replacement
 LOW_FREQUENCY_THRESHOLD=10