CI installation refactoring.

.github/workflows/ci-lint.yaml

@@ -35,12 +35,16 @@ jobs:
           tools/ci-build.sh
 
       - name: Run cpplint on C/C++ files.
-        run: cmake --build build --target cpplint
+        run: poetry run cmake --build build --target cpplint
 
       - name: Run clang-format on C/C++ files.
-        run: cmake --build build --target clang-format
+        run: |
+          poetry run clang-format --version
+          poetry run cmake --build build --target clang-format
 
       - name: Run black and isort on Python files.
         run: |
-          cmake --build build --target black
-          cmake --build build --target isort
+          poetry run black --version
+          poetry run cmake --build build --target black
+          poetry run isort --version
+          poetry run cmake --build build --target isort

.github/workflows/ci-p4tools.yml

@@ -40,5 +40,5 @@ jobs:
 
       - name: Run tests (Ubuntu 22.04)
         # Need to use sudo for the eBPF kernel tests.
-        run: sudo -E ctest -R "testgen|smith" --output-on-failure --schedule-random
+        run: sudo -E env PATH="$PATH" poetry run ctest -R "testgen|smith" --output-on-failure --schedule-random
         working-directory: ./build

.github/workflows/ci-ptf-kernels-weekly.yml

@@ -119,4 +119,4 @@ jobs:
 
       - name: Install VM and run PTF tests for eBPF backend
         run: |
-          sudo -E ./tools/ci-ptf/run_test.sh sudo docker run --privileged -v /sys/fs/bpf:/sys/fs/bpf -w /p4c/backends/ebpf/tests p4c ./test.sh
+          sudo -E ./tools/ci-ptf/run_test.sh sudo docker run --privileged -v /sys/fs/bpf:/sys/fs/bpf -w /p4c/backends/ebpf/tests p4c poetry run ./test.sh

.github/workflows/ci-ptf.yml

@@ -53,5 +53,5 @@ jobs:
           tools/ci-build.sh
 
       - name: Run PTF tests for eBPF backend (Ubuntu 20.04)
-        run: sudo -E ./test.sh
+        run: sudo -E env PATH="$PATH" poetry run ./test.sh
         working-directory: ./backends/ebpf/tests

.github/workflows/ci-static-build-test.yml

@@ -48,6 +48,6 @@ jobs:
       - shell: bash
         name: Build (Ubuntu 20.04)
         run: |
-          sudo -E tools/ci-build.sh
+          tools/ci-build.sh
           ./tools/ci-check-static.sh ./build/p4c-bm2-ss ./build/p4c-dpdk ./build/p4c-ebpf \
               ./build/p4c-pna-p4tc ./build/p4c-ubpf ./build/p4test ./build/p4testgen

.github/workflows/ci-test-debian.yml

@@ -14,6 +14,36 @@ concurrency:
   cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
 
 jobs:
+
+  # Build with GCC and test P4C on Ubuntu 24.04.
+  test-ubuntu24:
+    runs-on: ubuntu-24.04
+    env:
+      CTEST_PARALLEL_LEVEL: 4
+      IMAGE_TYPE: test
+      BUILD_GENERATOR: Ninja
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+          fetch-depth: 0
+
+      - name: ccache
+        uses: hendrikmuhs/ccache-action@v1
+        with:
+          key: test-${{ runner.os }}-gcc
+          max-size: 1000M
+
+      - name: Build (Ubuntu 24.04, GCC)
+        run: |
+          tools/ci-build.sh
+
+      - name: Run tests (Ubuntu 24.04)
+        # Need to use sudo for the eBPF kernel tests.
+        run: |
+          sudo -E env PATH="$PATH" poetry run ctest --output-on-failure --schedule-random
+        working-directory: ./build
+
   # Build with GCC and test P4C on Ubuntu 22.04.
   test-ubuntu22:
     runs-on: ubuntu-22.04
@@ -39,7 +69,8 @@ jobs:
 
       - name: Run tests (Ubuntu 22.04)
         # Need to use sudo for the eBPF kernel tests.
-        run: sudo -E ctest --output-on-failure --schedule-random
+        run: |
+          sudo -E env PATH="$PATH" poetry run ctest --output-on-failure --schedule-random
         working-directory: ./build
 
   # Build with GCC and test Tofino backend on Ubuntu 22.04.
@@ -112,6 +143,7 @@ jobs:
 
       - name: Run tests (Ubuntu 20.04)
         # Need to use sudo for the eBPF kernel tests.
-        run: sudo -E ctest --output-on-failure --schedule-random
+        run: |
+          sudo -E env PATH="$PATH" poetry run ctest --output-on-failure --schedule-random
         working-directory: ./build
         if: matrix.unity == 'ON' && matrix.gtest == 'ON'

.github/workflows/ci-test-fedora.yml

@@ -35,7 +35,9 @@ jobs:
           submodules: recursive
 
       - name: Install dependencies (Fedora Linux)
-        run: tools/install_fedora_deps.sh
+        run: |
+          export PATH="$HOME/.local/bin:$PATH"
+          tools/install_fedora_deps.sh
 
       - name: ccache
         uses: hendrikmuhs/ccache-action@v1
@@ -45,10 +47,12 @@ jobs:
 
       - name: Build p4c (Fedora Linux)
         run: |
+          export PATH="$HOME/.local/bin:$PATH"
           ./bootstrap.sh -DCMAKE_BUILD_TYPE=Release -DCMAKE_UNITY_BUILD=ON --build-generator "Ninja"
           cmake --build build -- -j $(nproc)
 
       - name: Run p4c tests (Fedora Linux)
         run: |
-          export PATH="$HOME/.local/bin:$PATH"; ctest --output-on-failure --schedule-random
+          export PATH="$HOME/.local/bin:$PATH"
+          poetry run ctest --output-on-failure --schedule-random
         working-directory: ./build

.github/workflows/ci-test-mac.yml

@@ -46,6 +46,7 @@ jobs:
 
       - name: Install dependencies (MacOS)
         run: |
+          source ~/.bash_profile
           tools/install_mac_deps.sh
 
       - name: Build (MacOS)
@@ -58,7 +59,7 @@ jobs:
       - name: Run tests (MacOS)
         run: |
           source ~/.bash_profile
-          ctest --output-on-failure --schedule-random -E "bpf|ubpf|testgen|smith"
+          poetry run ctest --output-on-failure --schedule-random -E "bpf|ubpf|testgen|smith"
         working-directory: ./build
 
   # Build and test p4c on MacOS 13 on x86.
@@ -93,6 +94,7 @@ jobs:
 
       - name: Install dependencies (MacOS)
         run: |
+          source ~/.bash_profile
           tools/install_mac_deps.sh
 
       - name: Build (MacOS)
@@ -105,5 +107,5 @@ jobs:
       - name: Run tests (MacOS)
         run: |
           source ~/.bash_profile
-          ctest --output-on-failure --schedule-random -E "bpf|ubpf|testgen|smith"
+          poetry run ctest --output-on-failure --schedule-random -E "bpf|ubpf|testgen|smith"
         working-directory: ./build

.github/workflows/ci-ubuntu-18-nightly.yml

@@ -39,4 +39,4 @@ jobs:
         # this is needed to create network namespaces for the ebpf tests.
       - name: Run tests (Ubuntu 18.04)
         run: |
-          sudo -E docker run --privileged -w /p4c/build -e $CTEST_PARALLEL_LEVEL p4c ctest --output-on-failure --schedule-random
+          sudo -E docker run --privileged -w /p4c/build -e $CTEST_PARALLEL_LEVEL p4c  poetry run --output-on-failure --schedule-random

.github/workflows/ci-ubuntu-20-sanitizer-nightly.yml

@@ -40,5 +40,5 @@ jobs:
 
       - name: Run tests (Ubuntu 20.04)
         # Need to use sudo for the eBPF kernel tests.
-        run: sudo -E ctest --output-on-failure --schedule-random
+        run: sudo -E env PATH="$PATH" poetry run ctest --output-on-failure --schedule-random
         working-directory: ./build

.github/workflows/ci-validation-nightly.yml

@@ -38,5 +38,5 @@ jobs:
 
       - name: Validate
         run: |
-          ctest  -R toz3-validate-p4c --output-on-failure --schedule-random
+          poetry run ctest  -R toz3-validate-p4c --output-on-failure --schedule-random
         working-directory: ./build

.gitignore

@@ -52,3 +52,6 @@ docs/doxygen/awesome_css/
 
 # thirdparty
 backends/tofino/third_party/
+
+# Poetry
+poetry.lock

Dockerfile

@@ -46,6 +46,7 @@ ARG BUILD_AUTO_VAR_INIT_PATTERN=OFF
 # - Disable leaks detector as p4c uses GC.
 ENV UBSAN_OPTIONS=print_stacktrace=1
 ENV ASAN_OPTIONS=print_stacktrace=1:detect_leaks=0
+ENV PATH="/root/.local/bin:$PATH"
 
 # Delegate the build to tools/ci-build.
 COPY . /p4c/

backends/bmv2/run-bmv2-ptf-test.py

@@ -42,7 +42,7 @@
 PARSER.add_argument(
     "-n",
     "--num-ifaces",
-    default=8,
+    default=10,
     dest="num_ifaces",
     help="How many virtual interfaces to create.",
 )
@@ -91,7 +91,7 @@ class Options:
     # The base directory where tests are executed.
     rootdir: Path = Path(".")
     # The number of interfaces to create for this particular test.
-    num_ifaces: int = 8
+    num_ifaces: int = 10
     # Whether to use nanomsg for packet delivery as opposed to Linux veth interfaces.
     use_nn: bool = False
 
@@ -201,7 +201,7 @@ def run_ptf(self, grpc_port: int, json_name: Path, info_name: Path) -> int:
         )
         # TODO: There is currently a bug where we can not support more than 344 ports at once.
         # The nanomsg test back end simply hangs, the reason is unclear.
-        port_range = "0-8"
+        port_range = f"0-{self.options.num_ifaces - 1}"
         run_ptf_cmd = (
             f"ptf --platform nn --device-socket 0-{{{port_range}}}@ipc://{self.options.testdir}/"
             f"bmv2_packets_1.ipc --pypath {pypath} "
@@ -244,7 +244,7 @@ def run_simple_switch_grpc(self, switchlog: Path, grpc_port: int) -> Optional[su
         thrift_port = testutils.pick_tcp_port(GRPC_ADDRESS, THRIFT_PORT)
         simple_switch_grpc = (
             f"simple_switch_grpc --thrift-port {thrift_port} --device-id 0 --log-file {switchlog} "
-            f"{ifaces} --log-flush -i 0@0 --no-p4 "
+            f"{ifaces} --log-flush --no-p4 "
             f"-- --grpc-server-addr {GRPC_ADDRESS}:{grpc_port}"
         )
         bridge_cmd = self.bridge.get_ns_prefix() + " " + simple_switch_grpc
@@ -271,7 +271,7 @@ def run_ptf(self, grpc_port: int, json_name: Path, info_name: Path) -> int:
         ifaces = self.get_iface_str(num_ifaces=self.options.num_ifaces, prefix="br_")
         test_params = (
             f"grpcaddr='{GRPC_ADDRESS}:{grpc_port}';p4info='{info_name}';config='{json_name}';"
-            f"packet_wait_time='0.1';"
+            f"packet_wait_time='0.5';"
         )
         run_ptf_cmd = (
             f"ptf --pypath {pypath} {ifaces} --log-file {self.options.testdir.joinpath('ptf.log')} "

backends/ebpf/CMakeLists.txt

@@ -21,17 +21,17 @@ if(NOT APPLE)
   set(FETCHCONTENT_QUIET OFF)
   fetchcontent_declare(
     bpfrepo
-    URL https://github.com/libbpf/libbpf/archive/refs/tags/v1.4.1.tar.gz
-    URL_HASH SHA256=cc01a3a05d25e5978c20be7656f14eb8b6fcb120bb1c7e8041e497814fc273cb
-    SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/runtime/contrib/libbpf
+    URL https://github.com/libbpf/bpftool/releases/download/v7.5.0/bpftool-libbpf-v7.5.0-sources.tar.gz
+    # URL_HASH SHA256=cc01a3a05d25e5978c20be7656f14eb8b6fcb120bb1c7e8041e497814fc273cb
+    SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/runtime/contrib/bpftool
     USES_TERMINAL_DOWNLOAD TRUE
     GIT_PROGRESS TRUE
     DOWNLOAD_EXTRACT_TIMESTAMP TRUE
   )
   fetchcontent_makeavailable(bpfrepo)
   set(FETCHCONTENT_QUIET ${FETCHCONTENT_QUIET_PREV})
   # Check if we have already built the libbpf library.
-  find_library(LIBBPF NAMES bpf HINTS "${CMAKE_CURRENT_SOURCE_DIR}/runtime/usr/lib64/")
+  find_library(LIBBPF NAMES bpf HINTS "${CMAKE_CURRENT_SOURCE_DIR}/runtime/install/libbpf/")
   if (NOT LIBBPF)
     message("Building libbpf...")
     execute_process(
@@ -223,7 +223,7 @@ else()
 endif()
 
 # check for the libbpf library
-find_library(LIBBPF NAMES bpf HINTS "${CMAKE_CURRENT_SOURCE_DIR}/runtime/usr/lib64/")
+find_library(LIBBPF NAMES bpf HINTS "${CMAKE_CURRENT_SOURCE_DIR}/runtime/install/libbpf/")
 if (LIBBPF)
   message(STATUS "Found libbpf library at ${LIBBPF}")
 else()

backends/ebpf/build_libbpf

@@ -10,7 +10,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-""" This programs builds a libbpf static library and places it in the runtime
+""" This programs builds a bpftool binary and the libbpf static library and places it in the runtime
     folder. The library and its headers are required by the kernel target.
 """
 
@@ -31,21 +31,24 @@ def main() -> int:
         level=logging.WARN,
         filemode="w",
     )
-    libbpf_dir = FILE_DIR.joinpath("runtime/contrib/libbpf")
-    libbpf_src_dir = libbpf_dir.joinpath("src")
-    libbpf_build_dir = libbpf_src_dir.joinpath("build")
-    libbpf_target_dir = FILE_DIR.joinpath("runtime")
-    # Create the libbpf build directory
-    testutils.check_and_create_dir(libbpf_build_dir)
-    # Build libbpf
-    mk_cmd = f"make -C {libbpf_src_dir} install "
+    bpftool_dir = FILE_DIR.joinpath("runtime/contrib/bpftool")
+    bpftool_src_dir = bpftool_dir.joinpath("src")
+    bpftool_build_dir = bpftool_src_dir.joinpath("build")
+    bpftool_target_dir = FILE_DIR.joinpath("runtime/install/")
+    bpftool_target_dir.mkdir(parents=True, exist_ok=True)
+    # Create the bpftool build directory
+    testutils.check_and_create_dir(bpftool_build_dir)
+    # Build bpftool
+    mk_cmd = f"make -C {bpftool_src_dir} install "
+    mk_cmd += f"OUTPUT={bpftool_target_dir}/ "
     mk_cmd += "BUILD_STATIC_ONLY=y "
-    mk_cmd += f"OBJDIR={libbpf_build_dir} "
-    mk_cmd += f"DESTDIR={libbpf_target_dir} "
+    mk_cmd += f"OBJDIR={bpftool_build_dir} "
+    mk_cmd += f"DESTDIR={bpftool_target_dir} "
     mk_cmd += "EXTRA_CFLAGS=-fPIE"
+    print(mk_cmd)
     result = testutils.exec_process(args=mk_cmd)
     if result.returncode != testutils.SUCCESS:
-        testutils.log.error("Could not build libbpf")
+        testutils.log.error("Could not build bpftool")
     return result.returncode
 
 

backends/ebpf/ebpfProgram.cpp

@@ -102,7 +102,7 @@ void EBPFProgram::emitC(CodeBuilder *builder, const std::filesystem::path &heade
     if (model.arch == ModelArchitecture::XdpSwitch)
         builder->target->emitCodeSection(builder, "xdp"_cs);
     else
-        builder->target->emitCodeSection(builder, "prog"_cs);
+        builder->target->emitCodeSection(builder, "tc"_cs);
     builder->emitIndent();
     builder->target->emitMain(builder, functionName, model.CPacketName.toString());
     builder->blockStart();

backends/ebpf/run-ebpf-test.py

@@ -123,9 +123,9 @@ def __init__(self):
         self.replace = False  # Replace previous outputs.
         self.target = "test"  # The name of the target compiler.
         # Actual location of the test framework.
-        self.testdir = str(FILE_DIR)
+        self.testdir = FILE_DIR
         # The location of the eBPF runtime, some targets may overwrite this.
-        self.runtimedir = str(FILE_DIR.joinpath("runtime"))
+        self.runtimedir = FILE_DIR.joinpath("runtime")
         self.extern = ""  # Path to C file with extern definition.
 
 

backends/ebpf/runtime/.gitignore

@@ -0,0 +1 @@
+install

backends/ebpf/runtime/contrib/.gitignore

@@ -1 +1,2 @@
 libbpf
+bpftool