Try a classic virtualenv approach.

Signed-off-by: fruffy <[email protected]>

.github/workflows/ci-lint.yaml

@@ -35,16 +35,20 @@ jobs:
           tools/ci-build.sh
 
       - name: Run cpplint on C/C++ files.
-        run: poetry run cmake --build build --target cpplint
+        run: |
+          source .venv/bin/activate
+          cmake --build build --target cpplint
 
       - name: Run clang-format on C/C++ files.
         run: |
-          poetry run clang-format --version
-          poetry run cmake --build build --target clang-format
+          source .venv/bin/activate
+          clang-format --version
+          cmake --build build --target clang-format
 
       - name: Run black and isort on Python files.
         run: |
-          poetry run black --version
-          poetry run cmake --build build --target black
-          poetry run isort --version
-          poetry run cmake --build build --target isort
+          source .venv/bin/activate
+          black --version
+          cmake --build build --target black
+          isort --version
+          cmake --build build --target isort

.github/workflows/ci-p4tools.yml

@@ -40,5 +40,7 @@ jobs:
 
       - name: Run tests (Ubuntu 22.04)
         # Need to use sudo for the eBPF kernel tests.
-        run: sudo -E env PATH="$PATH" poetry run ctest -R "testgen|smith" --output-on-failure --schedule-random
+        run: |
+          source ../.venv/bin/activate
+          sudo -E env PATH="$PATH" ctest -R "testgen|smith" --output-on-failure --schedule-random
         working-directory: ./build

.github/workflows/ci-ptf-kernels-weekly.yml

@@ -119,4 +119,4 @@ jobs:
 
       - name: Install VM and run PTF tests for eBPF backend
         run: |
-          sudo -E ./tools/ci-ptf/run_test.sh sudo docker run --privileged -v /sys/fs/bpf:/sys/fs/bpf -w /p4c/backends/ebpf/tests p4c poetry run ./test.sh
+          sudo -E ./tools/ci-ptf/run_test.sh sudo docker run --privileged -v /sys/fs/bpf:/sys/fs/bpf -w /p4c/backends/ebpf/tests p4c ./test.sh

.github/workflows/ci-ptf.yml

@@ -53,5 +53,6 @@ jobs:
           tools/ci-build.sh
 
       - name: Run PTF tests for eBPF backend (Ubuntu 20.04)
-        run: sudo -E env PATH="$PATH" poetry run ./test.sh
-        working-directory: ./backends/ebpf/tests
+        run: |
+          source .venv/bin/activate
+          sudo -E env PATH="$PATH" backends/ebpf/tests/test.sh

.github/workflows/ci-test-debian.yml

@@ -41,7 +41,8 @@ jobs:
       - name: Run tests (Ubuntu 24.04)
         # Need to use sudo for the eBPF kernel tests.
         run: |
-          sudo -E env PATH="$PATH" poetry run ctest --output-on-failure --schedule-random
+          source ../.venv/bin/activate
+          sudo -E env PATH="$PATH" ctest --output-on-failure --schedule-random
         working-directory: ./build
 
   # Build with GCC and test P4C on Ubuntu 22.04.
@@ -70,7 +71,8 @@ jobs:
       - name: Run tests (Ubuntu 22.04)
         # Need to use sudo for the eBPF kernel tests.
         run: |
-          sudo -E env PATH="$PATH" poetry run ctest --output-on-failure --schedule-random
+          source ../.venv/bin/activate
+          sudo -E env PATH="$PATH" ctest --output-on-failure --schedule-random
         working-directory: ./build
 
   # Build with GCC and test Tofino backend on Ubuntu 22.04.
@@ -144,6 +146,7 @@ jobs:
       - name: Run tests (Ubuntu 20.04)
         # Need to use sudo for the eBPF kernel tests.
         run: |
-          sudo -E env PATH="$PATH" poetry run ctest --output-on-failure --schedule-random
+          source ../.venv/bin/activate
+          sudo -E env PATH="$PATH" ctest --output-on-failure --schedule-random
         working-directory: ./build
         if: matrix.unity == 'ON' && matrix.gtest == 'ON'

.github/workflows/ci-test-fedora.yml

@@ -47,12 +47,12 @@ jobs:
 
       - name: Build p4c (Fedora Linux)
         run: |
-          export PATH="$HOME/.local/bin:$PATH"
           ./bootstrap.sh -DCMAKE_BUILD_TYPE=Release -DCMAKE_UNITY_BUILD=ON --build-generator "Ninja"
+          source .venv/bin/activate
           cmake --build build -- -j $(nproc)
 
       - name: Run p4c tests (Fedora Linux)
         run: |
-          export PATH="$HOME/.local/bin:$PATH"
-          poetry run ctest --output-on-failure --schedule-random
+          source ../.venv/bin/activate
+          ctest --output-on-failure --schedule-random
         working-directory: ./build

.github/workflows/ci-test-mac.yml

@@ -52,14 +52,16 @@ jobs:
       - name: Build (MacOS)
         run: |
           source ~/.bash_profile
+          source .venv/bin/activate
           ./bootstrap.sh -DENABLE_GC=ON -DCMAKE_BUILD_TYPE=Release \
               -DCMAKE_UNITY_BUILD=ON -DENABLE_TEST_TOOLS=ON -DENABLE_WERROR=ON --build-generator "Ninja"
           cmake --build build -- -j $(nproc)
 
       - name: Run tests (MacOS)
         run: |
           source ~/.bash_profile
-          poetry run ctest --output-on-failure --schedule-random -E "bpf|ubpf|testgen|smith"
+          source ../.venv/bin/activate
+          ctest --output-on-failure --schedule-random -E "bpf|ubpf|testgen|smith"
         working-directory: ./build
 
   # Build and test p4c on MacOS 13 on x86.
@@ -100,12 +102,14 @@ jobs:
       - name: Build (MacOS)
         run: |
           source ~/.bash_profile
+          source .venv/bin/activate
           ./bootstrap.sh -DENABLE_GC=ON -DCMAKE_BUILD_TYPE=Release \
               -DCMAKE_UNITY_BUILD=ON -DENABLE_TEST_TOOLS=ON -DENABLE_WERROR=ON --build-generator "Ninja"
           cmake --build build -- -j $(nproc)
 
       - name: Run tests (MacOS)
         run: |
           source ~/.bash_profile
-          poetry run ctest --output-on-failure --schedule-random -E "bpf|ubpf|testgen|smith"
+          source ../.venv/bin/activate
+          ctest --output-on-failure --schedule-random -E "bpf|ubpf|testgen|smith"
         working-directory: ./build

.github/workflows/ci-ubuntu-18-nightly.yml

@@ -39,4 +39,4 @@ jobs:
         # this is needed to create network namespaces for the ebpf tests.
       - name: Run tests (Ubuntu 18.04)
         run: |
-          sudo -E docker run --privileged -w /p4c/build -e $CTEST_PARALLEL_LEVEL p4c  poetry run --output-on-failure --schedule-random
+          sudo -E docker run --privileged -w /p4c/build -e $CTEST_PARALLEL_LEVEL p4c  source ../.venv/bin/activate && ctest --output-on-failure --schedule-random

.github/workflows/ci-ubuntu-20-sanitizer-nightly.yml

@@ -40,5 +40,7 @@ jobs:
 
       - name: Run tests (Ubuntu 20.04)
         # Need to use sudo for the eBPF kernel tests.
-        run: sudo -E env PATH="$PATH" poetry run ctest --output-on-failure --schedule-random
+        run: |
+          source ../.venv/bin/activate
+          sudo -E env PATH="$PATH" ctest --output-on-failure --schedule-random
         working-directory: ./build

.github/workflows/ci-validation-nightly.yml

@@ -38,5 +38,6 @@ jobs:
 
       - name: Validate
         run: |
-          poetry run ctest  -R toz3-validate-p4c --output-on-failure --schedule-random
+          source ../.venv/bin/activate
+          ctest  -R toz3-validate-p4c --output-on-failure --schedule-random
         working-directory: ./build

.gitignore

@@ -53,5 +53,7 @@ docs/doxygen/awesome_css/
 # thirdparty
 backends/tofino/third_party/
 
-# Poetry
+# Virtual environments or Poetry
+.venv
+venv
 poetry.lock

requirements.txt

@@ -0,0 +1,32 @@
+# General testing utilities.
+pyroute2==0.7.3
+ply==3.11
+scapy==2.5.0
+
+# Utilities for BMv2 and PTF tests.
+ptf==0.10.0
+googleapis-common-protos==1.53.0
+
+# gRPC and protobuf dependencies.
+grpcio==1.67.0; python_version > "3.6"
+grpcio==1.48.2; python_version <= "3.6"
+thrift==0.21.0
+protobuf==3.20.2; python_version > "3.6"
+protobuf==3.19.2; python_version <= "3.6"
+
+# Linters and formatters.
+clang-format==18.1.0
+black==24.3.0; python_version > "3.6"
+black==22.8.0; python_version <= "3.6"
+isort==5.13.2; python_version > "3.6"
+isort==5.10.1; python_version <= "3.6"
+
+# P4Runtime from Git
+# FIXME: We should figure out a way to synchronize P4Runtime versions across CMake and Poetry. This is the same commit hash as defined in the top-level CMakelists.txt
+git+https://github.com/p4lang/p4runtime.git@ec4eb5ef70dbcbcbf2f8357a4b2b8c2f218845a5#subdirectory=py
+
+# Tools for the Tofino back end.
+# FIXME: The Tofino back end should vendor these tools or use them locally.
+jsl==0.2.4
+pyinstaller==6.11.0
+jsonschema==4.23.0

tools/ci-build.sh

@@ -97,6 +97,7 @@ P4C_DEPS="bison \
           pkg-config \
           python3 \
           python3-pip \
+          python3-virtualenv \
           python3-setuptools \
           tcpdump"
 
@@ -107,10 +108,15 @@ fi
 
 sudo apt-get update
 sudo apt-get install -y --no-install-recommends ${P4C_DEPS}
-# Set up poetry.
-sudo apt-get install -y python3-venv curl
-curl -sSL https://install.python-poetry.org | python3 -
-poetry install -C ${P4C_DIR}
+
+# Install main Python dependencies.
+# TODO: Remove this check once 18.04 is deprecated.
+if [[ "${DISTRIB_RELEASE}" == "18.04" ]] ; then
+  pip3 install virtualenv
+fi
+virtualenv -p python3 ${P4C_DIR}/.venv
+source ${P4C_DIR}/.venv/bin/activate
+pip3 install -r ${P4C_DIR}/requirements.txt
 
 if [ "${BUILD_GENERATOR,,}" == "ninja" ] && [ ! $(command -v ninja) ]
 then
@@ -130,17 +136,17 @@ ccache --set-config max_size=1G
 
 # ! ------  BEGIN BMV2 -----------------------------------------------
 function build_bmv2() {
-  # TODO: Remove this check once 18.04 is deprecated.
-  if [[ "${DISTRIB_RELEASE}" == "18.04" ]] ; then
-    P4C_RUNTIME_DEPS_BOOST="libboost-graph1.65.1 libboost-iostreams1.65.1"
-  fi
-
   P4C_RUNTIME_DEPS="cpp \
-                    ${P4C_RUNTIME_DEPS_BOOST} \
                     libgc1* \
                     libgmp-dev \
+                    python3-dev \
                     libnanomsg-dev"
 
+  # TODO: Remove this check once 18.04 is deprecated.
+  if [[ "${DISTRIB_RELEASE}" == "18.04" ]] ; then
+    P4C_RUNTIME_DEPS+=" libboost-graph1.65.1 libboost-iostreams1.65.1 "
+  fi
+
   # TODO: Remove this check once 18.04 is deprecated.
   if [[ "${DISTRIB_RELEASE}" == "18.04" ]] || [[ "$(which simple_switch 2> /dev/null)" != "" ]] ; then
     # Use GCC 9 from https://launchpad.net/~ubuntu-toolchain-r/+archive/ubuntu/test
@@ -158,6 +164,11 @@ function build_bmv2() {
   fi
 
   sudo apt-get update && sudo apt-get install -y --no-install-recommends ${P4C_RUNTIME_DEPS}
+
+  if [[ "${DISTRIB_RELEASE}" != "18.04" ]] ; then
+    # To run PTF nanomsg tests. Not available on 18.04.
+    pip3 install nnpy==1.4.2
+  fi
 }
 
 if [[ "${ENABLE_BMV2}" == "ON" ]] ; then
@@ -213,8 +224,8 @@ fi
 function build_dpdk() {
   # Replace existing Protobuf with one that works.
   # TODO: Debug protobuf mismatch.
-  sudo -E pip3 uninstall -y protobuf
-  sudo pip3 install protobuf==3.20.3 netaddr==0.9.0
+  pip3 uninstall -y protobuf
+  pip3 install protobuf==3.20.3 netaddr==0.9.0
 }
 
 if [[ "${ENABLE_DPDK}" == "ON" ]]; then
@@ -227,7 +238,7 @@ fi
 function build_tofino() {
     P4C_TOFINO_PACKAGES="rapidjson-dev"
     sudo apt-get install -y --no-install-recommends ${P4C_TOFINO_PACKAGES}
-    sudo pip3 install jsl==0.2.4 pyinstaller==6.11.0
+    pip3 install jsl==0.2.4 pyinstaller==6.11.0
 }
 
 if [[ "${ENABLE_TOFINO}" == "ON" ]]; then
@@ -288,12 +299,12 @@ fi
 if [ -e build ]; then /bin/rm -rf build; fi
 mkdir -p ${P4C_DIR}/build
 cd ${P4C_DIR}/build
-poetry run cmake ${CMAKE_FLAGS} -G "${BUILD_GENERATOR}" ..
+cmake ${CMAKE_FLAGS} -G "${BUILD_GENERATOR}" ..
 
 # If CMAKE_ONLY is active, only run CMake. Do not build.
 if [ "$CMAKE_ONLY" == "OFF" ]; then
-  poetry run cmake --build . -- -j $(nproc)
-  sudo poetry run cmake --install .
+  cmake --build . -- -j $(nproc)
+  sudo cmake --install .
   # Print ccache statistics after building
   ccache -p -s
 fi

tools/install_fedora_deps.sh

@@ -61,10 +61,6 @@ sudo dnf install -y -q \
     zlib-devel \
     ninja-build
 
-# Set up poetry.
-curl -sSL https://install.python-poetry.org | python3 -
-poetry install
-
 MAKEFLAGS="-j$(nproc)"
 export MAKEFLAGS
 
@@ -91,3 +87,7 @@ make -j$((`nproc`+1)) install-strip
 popd
 
 rm -rf "${tmp_dir}"
+
+virtualenv ${P4C_DIR}/.venv
+source ${P4C_DIR}/.venv/bin/activate
+pip3 install -r ${P4C_DIR}/requirements.txt

tools/install_mac_deps.sh

@@ -5,6 +5,9 @@
 set -e  # Exit on error.
 set -x  # Make command execution verbose
 
+THIS_DIR=$( cd -- "$( dirname -- "${0}" )" &> /dev/null && pwd )
+P4C_DIR=$(readlink -f ${THIS_DIR}/..)
+
 # Installation helper.
 brew_install() {
     echo "\nInstalling $1"
@@ -63,6 +66,7 @@ fi
 echo 'export PATH="$HOME/.local/bin:$PATH"' >> ~/.bash_profile
 source ~/.bash_profile
 
-# Set up poetry.
-curl -sSL https://install.python-poetry.org | python3 -
-poetry install
+# Install required pip packages
+virtualenv ${P4C_DIR}/.venv
+source ${P4C_DIR}/.venv/bin/activate
+pip3 install -r ${P4C_DIR}/requirements.txt