From 809355482986dacf696f9541b9d8177d6842e5e6 Mon Sep 17 00:00:00 2001 From: Dima Gerasimov Date: Mon, 20 May 2024 02:42:35 +0100 Subject: [PATCH] end2end tests: update docker container version and cleanup --- .ci/end2end/.dockerignore | 1 + .ci/end2end/Dockerfile | 20 +++++++ .ci/end2end/build_and_run.sh | 20 +++++++ .ci/end2end/scripts/build_and_run_tests.sh | 15 +++++ .ci/end2end/scripts/setup_chrome.sh | 40 +++++++++++++ .ci/end2end/scripts/setup_firefox.sh | 26 +++++++++ .ci/end2end/scripts/setup_node.sh | 7 +++ .ci/end2end_tests.Dockerfile | 65 ---------------------- .github/workflows/main.yml | 9 +-- 9 files changed, 131 insertions(+), 72 deletions(-) create mode 100644 .ci/end2end/.dockerignore create mode 100644 .ci/end2end/Dockerfile create mode 100755 .ci/end2end/build_and_run.sh create mode 100755 .ci/end2end/scripts/build_and_run_tests.sh create mode 100755 .ci/end2end/scripts/setup_chrome.sh create mode 100755 .ci/end2end/scripts/setup_firefox.sh create mode 100755 .ci/end2end/scripts/setup_node.sh delete mode 100644 .ci/end2end_tests.Dockerfile diff --git a/.ci/end2end/.dockerignore b/.ci/end2end/.dockerignore new file mode 100644 index 00000000..27368b34 --- /dev/null +++ b/.ci/end2end/.dockerignore @@ -0,0 +1 @@ +build_and_run.sh diff --git a/.ci/end2end/Dockerfile b/.ci/end2end/Dockerfile new file mode 100644 index 00000000..eb10211e --- /dev/null +++ b/.ci/end2end/Dockerfile @@ -0,0 +1,20 @@ +FROM ubuntu:latest + +ENV DEBIAN_FRONTEND=noninteractive + +COPY scripts /scripts + +RUN /scripts/setup_firefox.sh \ + && /scripts/setup_chrome.sh \ + && /scripts/setup_node.sh \ + && apt install --yes tox git \ +# using python docs as a source of some html test data +# need to prevent dpkg from excluding doc files... + && sed -i '/usr.share.doc/d' /etc/dpkg/dpkg.cfg.d/excludes \ + && apt install --yes python3-doc \ + && apt clean \ + && mkdir /promnesia + +WORKDIR /promnesia + +ENTRYPOINT ["/scripts/build_and_run_tests.sh"] diff --git a/.ci/end2end/build_and_run.sh b/.ci/end2end/build_and_run.sh new file mode 100755 index 00000000..5a834a4e --- /dev/null +++ b/.ci/end2end/build_and_run.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -eux + +# TODO assert we're running under github ci? +# since this setup is kinda elaborate and can be somewhat unsafe to run blindly + +# supposed to be called from promnesia repository root +[ -e src/promnesia ] +[ -e extension/src ] + +PROMNESIA_SRC="$(pwd)" + +cd .ci/end2end + +IMAGE='promnesia_end2end_tests' + +docker build -t "$IMAGE" . + +# NOTE: dev/shm mount to prevent crashes during headless chrome +docker run -v /dev/shm:/dev/shm --mount "type=bind,src=$PROMNESIA_SRC,dst=/promnesia_source,readonly=true" -e CI "$IMAGE" "$@" diff --git a/.ci/end2end/scripts/build_and_run_tests.sh b/.ci/end2end/scripts/build_and_run_tests.sh new file mode 100755 index 00000000..7dcb591a --- /dev/null +++ b/.ci/end2end/scripts/build_and_run_tests.sh @@ -0,0 +1,15 @@ +#!/bin/bash +set -eux + +# Seems wrong to keep the whole repository in docker build context. +# So instead, we mount the repository inside the container (into /promnesia_source) +# (as read only to avoid messing up host files and crapping with caches etc.) +# However to actually run tests we do need a writable directory.. +# So we copy the repo to the actual working dir here + +# ugh, kinda annoying -- not sure how to update source files when we change them on the host system... +cp -R -T /promnesia_source /promnesia +extension/.ci/build + +git init # todo ??? otherwise setuptools-scm fails to detect the version... +python3 -m tox -e end2end -- "$@" diff --git a/.ci/end2end/scripts/setup_chrome.sh b/.ci/end2end/scripts/setup_chrome.sh new file mode 100755 index 00000000..b168da81 --- /dev/null +++ b/.ci/end2end/scripts/setup_chrome.sh @@ -0,0 +1,40 @@ +#!/bin/bash +set -eux -o pipefail + +apt update --yes + +apt install --yes wget + +install -d -m 0755 /etc/apt/keyrings +wget -q https://dl.google.com/linux/linux_signing_key.pub -O- | tee /etc/apt/keyrings/linux_signing_key.pub.asc > /dev/null +echo "deb [signed-by=/etc/apt/keyrings/linux_signing_key.pub.asc] https://dl.google.com/linux/chrome/deb/ stable main" | tee -a /etc/apt/sources.list.d/google-chrome.list > /dev/null + +apt update + +apt install --yes google-chrome-stable + +# sadly latest version of chrome/chromedriver isn't working due to some bugs with iframes (see install_custom_chrome) + +# remove the actual chrome to get it out of the way (we do want dependencies though) +apt remove --yes google-chrome-stable +! which google-chrome # check there is no binary (in case of virtual packages or whatever) + +function install_custom_chrome() { + ## this installs last revision that was actually working (1110897) or 113.0.5623.0 + ## see https://bugs.chromium.org/p/chromedriver/issues/detail?id=4440 + apt install --yes unzip + + mkdir /tmp/chrome + + wget -q 'https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2F1110897%2Fchrome-linux.zip?generation=1677589092014487&alt=media' \ + -O /tmp/chrome/chrome-linux.zip + unzip /tmp/chrome/chrome-linux.zip -d /tmp/chrome + ln -sf /tmp/chrome/chrome-linux/chrome /usr/bin/google-chrome + + wget -q 'https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2F1110897%2Fchromedriver_linux64.zip?generation=1677589097630198&alt=media' \ + -O /tmp/chrome/chromedriver_linux64.zip + unzip /tmp/chrome/chromedriver_linux64.zip -d /tmp/chrome + ln -sf /tmp/chrome/chromedriver_linux64/chromedriver /usr/bin/chromedriver +} + +install_custom_chrome diff --git a/.ci/end2end/scripts/setup_firefox.sh b/.ci/end2end/scripts/setup_firefox.sh new file mode 100755 index 00000000..d36159f8 --- /dev/null +++ b/.ci/end2end/scripts/setup_firefox.sh @@ -0,0 +1,26 @@ +#!/bin/bash +set -eux -o pipefail + +apt update --yes + +apt install --yes wget + +# NOTE: these days ubuntu provisions firefox via snap, and it's a nightmare to make it work with webdriver +# so we force it to use a regular package (following these instructions https://askubuntu.com/a/1510872/427470) +install -d -m 0755 /etc/apt/keyrings +wget -q https://packages.mozilla.org/apt/repo-signing-key.gpg -O- | tee /etc/apt/keyrings/packages.mozilla.org.asc > /dev/null +echo "deb [signed-by=/etc/apt/keyrings/packages.mozilla.org.asc] https://packages.mozilla.org/apt mozilla main" | tee -a /etc/apt/sources.list.d/mozilla.list > /dev/null + +# prevent snap version from overriding: +echo ' +Package: * +Pin: origin packages.mozilla.org +Pin-Priority: 1000 +' | tee /etc/apt/preferences.d/mozilla +# to check: -- should not show anything mentioning snap +# apt install --verbose-versions --dry-run firefox + +apt update + +apt install --yes firefox +# NOTE: selenium should download the corresponding geckodriver itself via selenium_manager diff --git a/.ci/end2end/scripts/setup_node.sh b/.ci/end2end/scripts/setup_node.sh new file mode 100755 index 00000000..a9ad2a11 --- /dev/null +++ b/.ci/end2end/scripts/setup_node.sh @@ -0,0 +1,7 @@ +#!/bin/bash +set -eux -o pipefail + +apt update --yes +apt install --yes curl +curl -fsSL https://deb.nodesource.com/setup_20.x | bash - +apt install --yes nodejs diff --git a/.ci/end2end_tests.Dockerfile b/.ci/end2end_tests.Dockerfile deleted file mode 100644 index fb8ea2c8..00000000 --- a/.ci/end2end_tests.Dockerfile +++ /dev/null @@ -1,65 +0,0 @@ -FROM ubuntu:jammy - -ENV DEBIAN_FRONTEND=noninteractive - -RUN apt-get update \ - ## install chrome - && apt-get install -y wget \ - && wget -q https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb \ - && apt-get install --yes ./google-chrome-stable_current_amd64.deb \ - ## install chromium (sadly chrome has a bug with iframes atm) - ## this installs last revision that was actually working (1110897), see https://bugs.chromium.org/p/chromedriver/issues/detail?id=4440 - && apt-get install -y unzip \ - && mkdir /tmp/chrome \ - && wget 'https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2F1110897%2Fchrome-linux.zip?generation=1677589092014487&alt=media' -O /tmp/chrome/chrome-linux.zip \ - && wget 'https://www.googleapis.com/download/storage/v1/b/chromium-browser-snapshots/o/Linux_x64%2F1110897%2Fchromedriver_linux64.zip?generation=1677589097630198&alt=media' -O /tmp/chrome/chromedriver_linux64.zip \ - && unzip /tmp/chrome/chrome-linux.zip -d /tmp/chrome \ - && unzip /tmp/chrome/chromedriver_linux64.zip -d /tmp/chrome \ - && ln -sf /tmp/chrome/chrome-linux/chrome /usr/bin/google-chrome \ - && ln -sf /tmp/chrome/chromedriver_linux64/chromedriver /usr/bin/chromedriver \ - ## TODO don't install chrome above? just install necessary deps - ## - ## install firefox - # install add-apt-repository command - && apt-get install --yes software-properties-common \ - && add-apt-repository ppa:mozillateam/ppa \ - # force to use firefox from ppa instead of snap - && printf 'Package: *\nPin: release o=LP-PPA-mozillateam\nPin-Priority: 1001\n' > /etc/apt/preferences.d/mozilla-firefox \ - && apt-get install --yes firefox \ - ## - # install extra dependencies - && apt-get install --yes \ - # gcc needed for psutil? - python3 python3-dev gcc python3-pip tox \ - curl git \ - # using python docs as a source of some html test data - # also prevent dpkg from excluding doc files... - && sed -i '/usr.share.doc/d' /etc/dpkg/dpkg.cfg.d/excludes \ - && apt-get install --yes python3-doc \ - # https://github.com/nodesource/distributions/blob/master/README.md#installation-instructions - && (curl -sL https://deb.nodesource.com/setup_18.x | bash - ) \ - && apt-get install --yes nodejs \ - && apt-get clean \ - # geckodriver isn't available in ubuntu repos anymore because of snap - && curl -L https://github.com/mozilla/geckodriver/releases/download/v0.33.0/geckodriver-v0.33.0-linux64.tar.gz | tar xz -C /usr/local/bin - ## TODO don't need geckodriver anymore since selenium manages to download it itself? - - -# ugh. so -# - chromium (as well as the chromedriver???) is packaged as snap in ubuntu 20.04 and basically it doesn't work under docker -# - debian image lacks many convenient binaries.. -# - it's apparently easier to usea actual Google fucking Chrome instead -# https://stackoverflow.com/questions/58997430/how-to-install-chromium-in-docker-based-on-ubuntu-19-10-and-20-04/60908332#60908332 -# (+ driver from here https://chromedriver.chromium.org/downloads) -# but either way exentesions don't work under headless chrome??? (see end2end tests source code) - -# TODO would be nice to only copy git tracked files?... -COPY . /repo -WORKDIR /repo - - -# FIXME fuck. otherwise setuptools-scm fails to detect the version... -RUN git init - -# builds both firefox and chrome targets -ENTRYPOINT ["/bin/bash", "-c", "extension/.ci/build && tox -e end2end -- \"$@\""] diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 0c13a401..fb5c1b7c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -80,10 +80,7 @@ jobs: submodules: recursive - uses: mxschmitt/action-tmate@v3 if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} - # NOTE: dev/shm mount to prevent crashes during headless chrome - - run: | - docker build -f .ci/end2end_tests.Dockerfile . -t promnesia_end2end_tests - docker run -v /dev/shm:/dev/shm -e CI promnesia_end2end_tests -- -k chrome + - run: .ci/end2end/build_and_run.sh -k chrome end2end_tests_firefox: runs-on: ubuntu-latest @@ -94,9 +91,7 @@ jobs: - uses: mxschmitt/action-tmate@v3 if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }} # NOTE: dev/shm mount to prevent crashes during headless chrome - - run: | - docker build -f .ci/end2end_tests.Dockerfile . -t promnesia_end2end_tests - docker run -v /dev/shm:/dev/shm -e CI promnesia_end2end_tests -- -k firefox + - run: .ci/end2end/build_and_run.sh -k firefox install_and_run_test: # todo use setup-python thing?