squash! refactor GitHub workflows

.github/workflows/build.yml

@@ -185,6 +185,7 @@ jobs:
 
           sudo apt-get install --no-install-recommends -y \
             libacl1-dev \
+            liblxc-dev \
             libcap-dev \
             libcowsql-dev \
             libudev-dev
@@ -222,3 +223,162 @@ jobs:
       - name: Unit tests (all)
         run: |
           sudo --preserve-env=CGO_CFLAGS,CGO_LDFLAGS,CGO_LDFLAGS_ALLOW,LD_LIBRARY_PATH LD_LIBRARY_PATH=${LD_LIBRARY_PATH} env "PATH=${PATH}" go test ./...
+
+  system-tests:
+    env:
+      CGO_LDFLAGS_ALLOW: "(-Wl,-wrap,pthread_create)|(-Wl,-z,now)"
+      INCUS_CEPH_CLUSTER: "ceph"
+      INCUS_CEPH_CEPHFS: "cephfs"
+      INCUS_CEPH_CEPHOBJECT_RADOSGW: "http://127.0.0.1"
+      INCUS_CONCURRENT: "1"
+      INCUS_VERBOSE: "1"
+      INCUS_OFFLINE: "1"
+      INCUS_TMPFS: "1"
+      INCUS_REQUIRED_TESTS: "test_storage_buckets"
+    name: System Tests
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false
+      matrix:
+        go:
+          - stable
+        suite:
+          - cluster
+          - standalone
+        backend:
+          - dir
+          - btrfs
+          - lvm
+          - zfs
+          - ceph
+          - random
+        include:
+          - go: oldstable
+            suite: cluster
+            backend: dir
+          - go: oldstable
+            suite: standalone
+            backend: dir
+          - go: tip
+            suite: cluster
+            backend: dir
+          - go: tip
+            suite: standalone
+            backend: dir
+
+    steps:
+      - name: Performance tuning
+        run: |
+          set -eux
+          # optimize ext4 FSes for performance, not reliability
+          for fs in $(findmnt --noheading --type ext4 --list --uniq | awk '{print $1}'); do
+            # nombcache and data=writeback cannot be changed on remount
+            sudo mount -o remount,noatime,barrier=0,commit=6000 "${fs}" || true
+          done
+
+          # disable dpkg from calling sync()
+          echo "force-unsafe-io" | sudo tee /etc/dpkg/dpkg.cfg.d/force-unsafe-io
+
+      - name: Reclaim some space
+        run: |
+          set -eux
+
+          sudo snap remove lxd --purge
+          # Purge older snap revisions that are disabled/superseded by newer revisions of the same snap
+          snap list --all | while read -r name _ rev _ _ notes _; do
+            [ "${notes}" = "disabled" ] && snap remove "${name}" --revision "${rev}" --purge
+          done || true
+
+          # This was inspired from https://github.com/easimon/maximize-build-space
+          df -h /
+          # dotnet
+          sudo rm -rf /usr/share/dotnet
+          # android
+          sudo rm -rf /usr/local/lib/android
+          # haskell
+          sudo rm -rf /opt/ghc
+          df -h /
+
+      - name: Remove docker
+        run: |
+          set -eux
+          sudo apt-get autopurge -y moby-containerd docker uidmap
+          sudo ip link delete docker0
+          sudo nft flush ruleset
+
+      - name: Get source
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ matrix.go }}-incus-src
+
+      - name: Get binaries
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ matrix.go }}-incus-bin
+          path: /go/bin
+
+      - name: Setup MicroCeph
+        if: ${{ matrix.backend == 'ceph' }}
+        run: |
+          set -x
+
+          # If the rootfs and the ephemeral part are on the same physical disk, giving the whole
+          # disk to microceph would wipe our rootfs. Since it is pretty rare for GitHub Action
+          # runners to have a single disk, we immediately bail rather than trying to gracefully
+          # handle it. Once snapd releases with https://github.com/snapcore/snapd/pull/13150,
+          # we will be able to stop worrying about that special case.
+          if [ "$(stat -c '%d' /)" = "$(stat -c '%d' /mnt)" ]; then
+            echo "FAIL: rootfs and ephemeral part on the same disk, aborting"
+            exit 1
+          fi
+
+          sudo snap install microceph --channel=quincy/stable
+          sudo apt-get install --no-install-recommends -y ceph-common
+          sudo microceph cluster bootstrap
+          sudo microceph.ceph config set global osd_pool_default_size 1
+          sudo microceph.ceph config set global mon_allow_pool_delete true
+          sudo microceph.ceph config set global osd_memory_target 939524096
+          sudo microceph.ceph osd crush rule rm replicated_rule
+          sudo microceph.ceph osd crush rule create-replicated replicated default osd
+          for flag in nosnaptrim noscrub nobackfill norebalance norecover noscrub nodeep-scrub; do
+              sudo microceph.ceph osd set $flag
+          done
+          # Repurpose the ephemeral disk for ceph OSD.
+          sudo swapoff /mnt/swapfile
+          ephemeral_disk="$(findmnt --noheadings --output SOURCE --target /mnt | sed 's/[0-9]\+$//')"
+          sudo umount /mnt
+          sudo microceph disk add --wipe "${ephemeral_disk}"
+          sudo rm -rf /etc/ceph
+          sudo ln -s /var/snap/microceph/current/conf/ /etc/ceph
+          sudo microceph enable rgw
+          sudo microceph.ceph osd pool create cephfs_meta 32
+          sudo microceph.ceph osd pool create cephfs_data 32
+          sudo microceph.ceph fs new cephfs cephfs_meta cephfs_data
+          sudo microceph.ceph fs ls
+          sleep 30
+          sudo microceph.ceph status
+          # Wait until there are no more "unkowns" pgs
+          for _ in $(seq 60); do
+            if sudo microceph.ceph pg stat | grep -wF unknown; then
+              sleep 1
+            else
+              break
+            fi
+          done
+          sudo microceph.ceph status
+          sudo rm -f /snap/bin/rbd
+
+      - name: "Ensure offline mode (block image server)"
+        run: |
+          sudo nft add table inet filter
+          sudo nft add chain 'inet filter output { type filter hook output priority 10 ; }'
+          sudo nft add rule inet filter output ip daddr 45.45.148.8 reject
+          sudo nft add rule inet filter output ip6 daddr 2602:fc62:a:1::8 reject
+
+      - name: "Run system tests (${{ matrix.go }}, ${{ matrix.suite }}, ${{ matrix.backend }})"
+        run: |
+          chmod +x ~
+          echo "root:1000000:1000000000" | sudo tee /etc/subuid /etc/subgid
+          cd test
+          export PATH=/go/bin:$PATH
+          sudo --preserve-env=PATH,GOPATH,GITHUB_ACTIONS,INCUS_VERBOSE,INCUS_BACKEND,INCUS_CEPH_CLUSTER,INCUS_CEPH_CEPHFS,INCUS_CEPH_CEPHOBJECT_RADOSGW,INCUS_OFFLINE,INCUS_SKIP_TESTS,INCUS_REQUIRED_TESTS, INCUS_BACKEND=${{ matrix.backend }} ./main.sh ${{ matrix.suite }}