CI installation refactoring.

Signed-off-by: fruffy <[email protected]>

.github/workflows/ci-lint.yaml

@@ -35,12 +35,16 @@ jobs:
           tools/ci-build.sh
 
       - name: Run cpplint on C/C++ files.
-        run: cmake --build build --target cpplint
+        run: poetry run cmake --build build --target cpplint
 
       - name: Run clang-format on C/C++ files.
-        run: cmake --build build --target clang-format
+        run: |
+          poetry run clang-format --version
+          poetry run cmake --build build --target clang-format
 
       - name: Run black and isort on Python files.
         run: |
-          cmake --build build --target black
-          cmake --build build --target isort
+          poetry run black --version
+          poetry run cmake --build build --target black
+          poetry run isort --version
+          poetry run cmake --build build --target isort

.github/workflows/ci-p4tools.yml

@@ -40,5 +40,5 @@ jobs:
 
       - name: Run tests (Ubuntu 22.04)
         # Need to use sudo for the eBPF kernel tests.
-        run: sudo -E ctest -R "testgen|smith" --output-on-failure --schedule-random
+        run: sudo -E env PATH="$PATH" poetry run ctest -R "testgen|smith" --output-on-failure --schedule-random
         working-directory: ./build

.github/workflows/ci-ptf-kernels-weekly.yml

@@ -119,4 +119,4 @@ jobs:
 
       - name: Install VM and run PTF tests for eBPF backend
         run: |
-          sudo -E ./tools/ci-ptf/run_test.sh sudo docker run --privileged -v /sys/fs/bpf:/sys/fs/bpf -w /p4c/backends/ebpf/tests p4c ./test.sh
+          sudo -E ./tools/ci-ptf/run_test.sh sudo docker run --privileged -v /sys/fs/bpf:/sys/fs/bpf -w /p4c/backends/ebpf/tests p4c poetry run ./test.sh

.github/workflows/ci-ptf.yml

@@ -53,5 +53,5 @@ jobs:
           tools/ci-build.sh
 
       - name: Run PTF tests for eBPF backend (Ubuntu 20.04)
-        run: sudo -E ./test.sh
+        run: sudo -E env PATH="$PATH" poetry run ./test.sh
         working-directory: ./backends/ebpf/tests

.github/workflows/ci-static-build-test.yml

@@ -48,6 +48,6 @@ jobs:
       - shell: bash
         name: Build (Ubuntu 20.04)
         run: |
-          sudo -E tools/ci-build.sh
+          tools/ci-build.sh
           ./tools/ci-check-static.sh ./build/p4c-bm2-ss ./build/p4c-dpdk ./build/p4c-ebpf \
               ./build/p4c-pna-p4tc ./build/p4c-ubpf ./build/p4test ./build/p4testgen

.github/workflows/ci-test-debian.yml

@@ -14,6 +14,36 @@ concurrency:
   cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
 
 jobs:
+
+  # Build with GCC and test P4C on Ubuntu 24.04.
+  test-ubuntu24:
+    runs-on: ubuntu-24.04
+    env:
+      CTEST_PARALLEL_LEVEL: 4
+      IMAGE_TYPE: test
+      BUILD_GENERATOR: Ninja
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+          fetch-depth: 0
+
+      - name: ccache
+        uses: hendrikmuhs/ccache-action@v1
+        with:
+          key: test-${{ runner.os }}-gcc
+          max-size: 1000M
+
+      - name: Build (Ubuntu 24.04, GCC)
+        run: |
+          tools/ci-build.sh
+
+      - name: Run tests (Ubuntu 24.04)
+        # Need to use sudo for the eBPF kernel tests.
+        run: |
+          sudo -E env PATH="$PATH" poetry run ctest --output-on-failure --schedule-random
+        working-directory: ./build
+
   # Build with GCC and test P4C on Ubuntu 22.04.
   test-ubuntu22:
     runs-on: ubuntu-22.04
@@ -39,7 +69,8 @@ jobs:
 
       - name: Run tests (Ubuntu 22.04)
         # Need to use sudo for the eBPF kernel tests.
-        run: sudo -E ctest --output-on-failure --schedule-random
+        run: |
+          sudo -E env PATH="$PATH" poetry run ctest --output-on-failure --schedule-random
         working-directory: ./build
 
   # Build with GCC and test Tofino backend on Ubuntu 22.04.
@@ -112,6 +143,7 @@ jobs:
 
       - name: Run tests (Ubuntu 20.04)
         # Need to use sudo for the eBPF kernel tests.
-        run: sudo -E ctest --output-on-failure --schedule-random
+        run: |
+          sudo -E env PATH="$PATH" poetry run ctest --output-on-failure --schedule-random
         working-directory: ./build
         if: matrix.unity == 'ON' && matrix.gtest == 'ON'

.github/workflows/ci-test-fedora.yml

@@ -50,5 +50,5 @@ jobs:
 
       - name: Run p4c tests (Fedora Linux)
         run: |
-          export PATH="$HOME/.local/bin:$PATH"; ctest --output-on-failure --schedule-random
+          export PATH="$HOME/.local/bin:$PATH"; poetry run ctest --output-on-failure --schedule-random
         working-directory: ./build

.github/workflows/ci-test-mac.yml

@@ -58,7 +58,7 @@ jobs:
       - name: Run tests (MacOS)
         run: |
           source ~/.bash_profile
-          ctest --output-on-failure --schedule-random -E "bpf|ubpf|testgen|smith"
+          poetry run ctest --output-on-failure --schedule-random -E "bpf|ubpf|testgen|smith"
         working-directory: ./build
 
   # Build and test p4c on MacOS 13 on x86.
@@ -105,5 +105,5 @@ jobs:
       - name: Run tests (MacOS)
         run: |
           source ~/.bash_profile
-          ctest --output-on-failure --schedule-random -E "bpf|ubpf|testgen|smith"
+          poetry run ctest --output-on-failure --schedule-random -E "bpf|ubpf|testgen|smith"
         working-directory: ./build

.github/workflows/ci-ubuntu-18-nightly.yml

@@ -39,4 +39,4 @@ jobs:
         # this is needed to create network namespaces for the ebpf tests.
       - name: Run tests (Ubuntu 18.04)
         run: |
-          sudo -E docker run --privileged -w /p4c/build -e $CTEST_PARALLEL_LEVEL p4c ctest --output-on-failure --schedule-random
+          sudo -E docker run --privileged -w /p4c/build -e $CTEST_PARALLEL_LEVEL p4c  poetry run --output-on-failure --schedule-random

.github/workflows/ci-ubuntu-20-sanitizer-nightly.yml

@@ -40,5 +40,5 @@ jobs:
 
       - name: Run tests (Ubuntu 20.04)
         # Need to use sudo for the eBPF kernel tests.
-        run: sudo -E ctest --output-on-failure --schedule-random
+        run: sudo -E env PATH="$PATH" poetry run ctest --output-on-failure --schedule-random
         working-directory: ./build

.github/workflows/ci-validation-nightly.yml

@@ -38,5 +38,5 @@ jobs:
 
       - name: Validate
         run: |
-          ctest  -R toz3-validate-p4c --output-on-failure --schedule-random
+          poetry run ctest  -R toz3-validate-p4c --output-on-failure --schedule-random
         working-directory: ./build

.gitignore

@@ -52,3 +52,6 @@ docs/doxygen/awesome_css/
 
 # thirdparty
 backends/tofino/third_party/
+
+# Poetry
+poetry.lock

pyproject.toml

@@ -51,3 +51,44 @@ disable = [
   "missing-function-docstring",
   "too-few-public-methods",
 ]
+
+[tool.poetry]
+name = "p4c"
+version = "1.2.5.1"
+description = ""
+authors = ["Your Name <[email protected]>"]
+license = "Apache"
+readme = "README.md"
+package-mode = false
+
+[tool.poetry.dependencies]
+python = "^3.8"
+pyroute2 = "0.7.3"
+ply = "3.11"
+ptf = "0.10.0"
+scapy = "2.5.0"
+clang-format = "18.1.0"
+googleapis-common-protos = "1.53.0"
+protobuf = [
+  {version = "3.20.2", python = ">3.6"},
+  {version = "3.19.2", python = "<=3.6"}
+]
+black = [
+  {version = "24.3.0", python = ">3.6"},
+  {version = "22.8.0", python = "<=3.6"}
+]
+isort = [
+  {version = "5.13.2", python = ">3.6"},
+  {version = "5.10.1", python = "<=3.6"}
+]
+grpcio = [
+  {version = "1.67.0", python = ">3.6"},
+  {version = "1.48.2", python = "<=3.6"}
+]
+# FIXME: Remove and replace with local FetchContent version.
+p4runtime = "1.4.1"
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

tools/ci-build.sh

@@ -104,8 +104,10 @@ fi
 
 sudo apt-get update
 sudo apt-get install -y --no-install-recommends ${P4C_DEPS}
-sudo pip3 install --upgrade pip
-sudo pip3 install -r ${P4C_DIR}/requirements.txt
+# Set up poetry.
+sudo apt-get install -y python3-venv curl
+curl -sSL https://install.python-poetry.org | python3 -
+poetry install
 
 if [ "${BUILD_GENERATOR,,}" == "ninja" ] && [ ! $(command -v ninja) ]
 then
@@ -155,11 +157,6 @@ function build_bmv2() {
   fi
 
   sudo apt-get update && sudo apt-get install -y --no-install-recommends ${P4C_RUNTIME_DEPS}
-
-  if [[ "${DISTRIB_RELEASE}" != "18.04" ]] ; then
-    # To run PTF nanomsg tests. Not available on 18.04.
-    sudo pip3 install nnpy
-  fi
 }
 
 if [[ "${ENABLE_BMV2}" == "ON" ]] ; then