Update how dependencies are installed in workflow

.github/workflows/ci.yaml

@@ -66,6 +66,8 @@ jobs:
         with:
           submodules: recursive
           persist-credentials: false
+      - name: Install required dependencies
+        run: /bin/bash .scripts/ci-install-deps.sh
       - name: Build
         run: /bin/bash .scripts/ci-build.sh "${{ matrix.nginx }}"
       - name: Unit Tests
@@ -79,6 +81,8 @@ jobs:
         with:
           submodules: recursive
           persist-credentials: false
+      - name: Install required dependencies
+        run: /bin/bash .scripts/ci-install-deps.sh
       - name: Build
         run: /bin/bash .scripts/ci-build.sh "1.25.1"
       - name: Run WAF efficacy tests (core only)

.github/workflows/distros.yaml

@@ -93,6 +93,8 @@ jobs:
             "debian-bookworm",
             "ubuntu-focal",
             "ubuntu-jammy",
+            "ubuntu-lunar",
+            "ubuntu-mantic",
             "alpine-3.17.4",
             "alpine-3.18.2",
           ]
@@ -113,7 +115,7 @@ jobs:
           - name: debian-bookworm
             container: debian:bookworm-slim
             package: deb
-            pcredep: libpcre2-dev
+            pcredep: libpcre3-dev
             nginxdep: nginx
           - name: ubuntu-focal
             container: ubuntu:focal
@@ -125,6 +127,16 @@ jobs:
             package: deb
             pcredep: libpcre3-dev
             nginxdep: nginx-common
+          - name: ubuntu-lunar
+            container: ubuntu:lunar
+            package: deb
+            pcredep: libpcre3-dev
+            nginxdep: nginx-common
+          - name: ubuntu-mantic
+            container: ubuntu:mantic
+            package: deb
+            pcredep: libpcre2-dev
+            nginxdep: nginx
           - name: alpine-3.17.4
             container: alpine:3.17.4
             package: apk
@@ -136,10 +148,10 @@ jobs:
     - name: Checkout repository
       uses: actions/checkout@v4
 
-    # APK package
     - name: Install required dependencies
-      if: matrix.package == 'apk'
-      run: apk add alpine-sdk doas
+      run: /bin/sh .scripts/ci-install-deps.sh
+
+    # APK package
     - name: Checkout needed repos
       if: matrix.package == 'apk'
       run: |
@@ -163,38 +175,6 @@ jobs:
         path: build/*.apk
 
     # DEB package
-    - name: Update OS & Install deps
-      if: matrix.package == 'deb'
-      run: |
-        if [ -f "/etc/apt/sources.list" ]; then
-          cat /etc/apt/sources.list > /tmp/sources.list.orig
-          cat /tmp/sources.list.orig | grep -v "#" | sed 's/^deb /deb-src /g' >> /etc/apt/sources.list
-        fi
-        apt-get -qqy update
-        DEBIAN_FRONTEND=noninteractive apt-get -qqy --no-install-recommends install \
-            build-essential \
-            ca-certificates \
-            dpkg-dev \
-            gzip \
-            git \
-            libgd-dev \
-            libgeoip-dev \
-            ${{ matrix.pcredep }} \
-            libperl-dev \
-            libssl-dev \
-            libxslt1-dev \
-            nginx \
-            tar \
-            wget \
-            zlib1g-dev
-        if [ -f "/etc/apt/sources.list.d/debian.sources" ]; then
-          # bookworm only.
-          echo "deb https://ftp.debian.org/debian/ bookworm contrib main non-free non-free-firmware" >> /etc/apt/sources.list.d/nginx.list
-          echo "deb-src https://ftp.debian.org/debian/ bookworm contrib main non-free non-free-firmware" >> /etc/apt/sources.list.d/nginx.list
-          apt-get -qqy update
-          DEBIAN_FRONTEND=noninteractive apt-get -qqy --no-install-recommends install \
-            libperl-dev
-        fi
     - name: Checkout needed repos
       if: matrix.package == 'deb'
       run: |
@@ -262,7 +242,6 @@ jobs:
     - name: ${{ matrix.name }} package
       if: matrix.package == 'arch'
       run: |
-        pacman -Syy --needed --noconfirm sudo base-devel git perl-rename
         chmod 777 distros/arch
         useradd build-user -m
         passwd -d build-user

.scripts/ci-install-deps.sh

@@ -0,0 +1,129 @@
+#!/bin/sh
+# SPDX-FileCopyrightText: 2025 deroad <[email protected]>
+# SPDX-License-Identifier: LGPL-3.0-only
+
+patch_apt_source() {
+    # enable deb-src
+    sed -i 's/^# deb-src/deb-src/g' "$1"
+    if [ ! -z "$(cat "$1" | grep '^Types: deb$')" ]; then
+        echo "patching deb822 style $1"
+        sed -i 's/^Types: deb$/Types: deb deb-src/g' "$1"
+    elif [ -z "$(cat "$1" | grep '^deb-src')" ]; then
+        echo "patching classic style $1"
+        cat "$1" > /tmp/sources.list.orig
+        cat /tmp/sources.list.orig | grep "^deb " | sed 's/^deb /deb-src /g' >> "$1"
+    fi
+}
+
+install_deps_apt_get() {
+    if [ -f "/etc/apt/sources.list" ]; then
+        patch_apt_source "/etc/apt/sources.list"
+    fi
+
+    if [ -d "/etc/apt/sources.list.d/" ]; then
+        for source in $(find /etc/apt/sources.list.d/ -type f); do
+            patch_apt_source "$source"
+        done
+    fi
+
+    # install the dependency
+    DEBIAN_FRONTEND=noninteractive apt-get -y update
+    DEBIAN_FRONTEND=noninteractive apt-get -y --no-install-recommends install \
+        build-essential \
+        ca-certificates \
+        dpkg-dev \
+        gzip \
+        git \
+        libgd-dev \
+        libgeoip-dev \
+        libperl-dev \
+        libssl-dev \
+        libxslt1-dev \
+        nginx \
+        tar \
+        wget \
+        zlib1g-dev \
+        $@
+}
+
+install_deps_pacman() {
+    # makepkg will fetch the deps
+    pacman -Syy --needed --noconfirm \
+        sudo \
+        base-devel \
+        git \
+        perl-rename
+}
+
+install_deps_apk() {
+    # without --no-cache because it needs to 
+    # update sources before calling abuild
+    apk add \
+        alpine-sdk \
+        doas
+}
+
+if [ ! -f "/etc/os-release" ]; then
+    echo "Missing /etc/os-release"
+    exit 1
+fi
+
+. /etc/os-release
+
+case $ID in
+    ubuntu)
+        echo "found debian $VERSION_CODENAME"
+        case $VERSION_CODENAME in
+            focal)
+                install_deps_apt_get "libpcre3-dev"
+                ;;
+            jammy)
+                install_deps_apt_get "libpcre3-dev"
+                ;;
+            lunar)
+                sed -i -re 's/([a-z]{2}\.)?archive.ubuntu.com|security.ubuntu.com/old-releases.ubuntu.com/g' /etc/apt/sources.list
+                install_deps_apt_get "libpcre3-dev"
+                ;;
+            mantic)
+                sed -i -re 's/([a-z]{2}\.)?archive.ubuntu.com|security.ubuntu.com/old-releases.ubuntu.com/g' /etc/apt/sources.list
+                install_deps_apt_get "libpcre2-dev"
+                ;;
+            *)
+                echo "Not handled."
+                exit 1
+                ;;
+        esac
+        ;;
+    debian)
+        echo "found debian $VERSION_CODENAME"
+        case $VERSION_CODENAME in
+            bookworm)
+                # required to call apt source
+                chmod 777 .
+                install_deps_apt_get "libpcre3-dev" "libperl-dev" "dpkg-dev"
+                ;;
+            bullseye)
+                install_deps_apt_get "libpcre3-dev" "libperl-dev"
+                ;;
+            buster)
+                install_deps_apt_get "libpcre3-dev" "libperl-dev"
+                ;;
+            *)
+                echo "Not handled."
+                exit 1
+                ;;
+        esac
+        ;;
+    arch)
+        echo "Found arch linux"
+        install_deps_pacman
+        ;;
+    alpine)
+        echo "Found alpine linux"
+        install_deps_apk
+        ;;
+    *)
+        echo "Unknown OS: $ID"
+        exit 1
+        ;;
+esac