From eb6a46800f4d76747a2e05a65d045e8a94bef3eb Mon Sep 17 00:00:00 2001 From: Adam Cozzette Date: Wed, 1 Nov 2023 16:09:54 -0700 Subject: [PATCH] Add CI coverage for utf8_range We will soon be moving utf8_range into the protobuf repo, not as a subtree anymore but as the real source of truth. This change adds CI coverage in advance so that there will not be a lapse in coverage. I also upgraded our pinned versions of rules_fuzzing and rules_python, to fix some errors that came up with Bazel 6 and Python 3.12. I had to patch rules_fuzzing but I am working on upstreaming the fixes. PiperOrigin-RevId: 578671967 --- .github/workflows/test_cpp.yml | 12 ++++++------ WORKSPACE | 20 +++++++++++++++++--- protobuf_deps.bzl | 12 ++++++------ third_party/BUILD.bazel | 1 + third_party/rules_fuzzing.patch | 25 +++++++++++++++++++++++++ 5 files changed, 55 insertions(+), 15 deletions(-) create mode 100644 third_party/rules_fuzzing.patch diff --git a/.github/workflows/test_cpp.yml b/.github/workflows/test_cpp.yml index 55c061bc1b4b..58ffdc191629 100644 --- a/.github/workflows/test_cpp.yml +++ b/.github/workflows/test_cpp.yml @@ -27,14 +27,14 @@ jobs: include: # Set defaults - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/sanitize@sha256:04cd765285bc52cbbf51d66c8c66d8603579cf0f19cc42df26b09d2c270541fb - - targets: //pkg/... //src/... @com_google_protobuf_examples//... + - targets: //pkg/... //src/... @com_google_protobuf_examples//... @utf8_range//... # Override cases with custom images - config: { name: "TCMalloc" } image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/tcmalloc@sha256:bd39119d74b8a3fad4ae335d4cf5294e70384676331b7e19949459fc7a8d8328" - targets: "//src/..." + targets: "//src/... @utf8_range//..." - config: { name: "aarch64" } - targets: "//src/... //src/google/protobuf/compiler:protoc_aarch64_test" + targets: "//src/... //src/google/protobuf/compiler:protoc_aarch64_test @utf8_range//..." image: "us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:aarch64-63dd26c0c7a808d92673a3e52e848189d4ab0f17" name: Linux ${{ matrix.config.name }} runs-on: ${{ matrix.config.runner || 'ubuntu-latest' }} @@ -69,7 +69,7 @@ jobs: image: us-docker.pkg.dev/protobuf-build/containers/test/linux/gcc:${{ matrix.version }}-6.3.0-63dd26c0c7a808d92673a3e52e848189d4ab0f17 credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }} bazel-cache: cpp_linux/gcc-${{ matrix.version }} - bazel: test //pkg/... //src/... @com_google_protobuf_examples//... + bazel: test //pkg/... //src/... @com_google_protobuf_examples//... @utf8_range//... linux-release: strategy: @@ -316,13 +316,13 @@ jobs: - name: MacOS os: macos-12 cache_key: macos-12 - bazel: test //src/... + bazel: test //src/... @utf8_range//... - name: MacOS Apple Silicon (build only) os: macos-12 cache_key: macos-12-arm # Current github runners are all Intel based, so just build/compile # for Apple Silicon to detect issues there. - bazel: build --cpu=darwin_arm64 //src/... + bazel: build --cpu=darwin_arm64 //src/... @utf8_range//... - name: Windows os: windows-2022 cache_key: windows-2022 diff --git a/WORKSPACE b/WORKSPACE index 487c9334fc64..4dd69d25d245 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -17,6 +17,10 @@ load("//:protobuf_deps.bzl", "PROTOBUF_MAVEN_ARTIFACTS", "protobuf_deps") protobuf_deps() +load("@rules_python//python:repositories.bzl", "py_repositories") + +py_repositories() + # Bazel platform rules. http_archive( name = "platforms", @@ -163,15 +167,25 @@ utf8_range_deps() http_archive( name = "rules_fuzzing", - sha256 = "d9002dd3cd6437017f08593124fdd1b13b3473c7b929ceb0e60d317cb9346118", - strip_prefix = "rules_fuzzing-0.3.2", - urls = ["https://github.com/bazelbuild/rules_fuzzing/archive/v0.3.2.zip"], + sha256 = "ff52ef4845ab00e95d29c02a9e32e9eff4e0a4c9c8a6bcf8407a2f19eb3f9190", + strip_prefix = "rules_fuzzing-0.4.1", + urls = ["https://github.com/bazelbuild/rules_fuzzing/releases/download/v0.4.1/rules_fuzzing-0.4.1.zip"], + patches = ["//third_party:rules_fuzzing.patch"], + patch_args = ["-p1"], ) load("@rules_fuzzing//fuzzing:repositories.bzl", "rules_fuzzing_dependencies") rules_fuzzing_dependencies() +load("@rules_fuzzing//fuzzing:init.bzl", "rules_fuzzing_init") + +rules_fuzzing_init() + +load("@fuzzing_py_deps//:requirements.bzl", fuzzing_py_deps_install_deps = "install_deps") + +fuzzing_py_deps_install_deps() + bind( name = "python_headers", actual = "@system_python//:python_headers", diff --git a/protobuf_deps.bzl b/protobuf_deps.bzl index d055eed8c623..ed0d73bbbc1a 100644 --- a/protobuf_deps.bzl +++ b/protobuf_deps.bzl @@ -72,8 +72,8 @@ def protobuf_deps(): _github_archive( name = "utf8_range", repo = "https://github.com/protocolbuffers/utf8_range", - commit = "d863bc33e15cba6d873c878dcca9e6fe52b2f8cb", - sha256 = "568988b5f7261ca181468dba38849fabf59dd9200fb2ed4b2823da187ef84d8c", + commit = "0e378bdb224cc8d4653b0db963b474839c2bb43c", + sha256 = "9b8db33325e5b6105c04369cd04dbbd9186e0577141543714243e66b8f045664", ) if not native.existing_rule("rules_cc"): @@ -102,11 +102,11 @@ def protobuf_deps(): ) if not native.existing_rule("rules_python"): - _github_archive( + http_archive( name = "rules_python", - repo = "https://github.com/bazelbuild/rules_python", - commit = "02b521fce3c7b36b05813aa986d72777cc3ee328", # 0.24.0 - sha256 = "f9e4f6acf82449324d56669bda4bdb28b48688ad2990d8b39fa5b93ed39c9ad1", + sha256 = "9d04041ac92a0985e344235f5d946f71ac543f1b1565f2cdbc9a2aaee8adf55b", + strip_prefix = "rules_python-0.26.0", + url = "https://github.com/bazelbuild/rules_python/releases/download/0.26.0/rules_python-0.26.0.tar.gz", ) if not native.existing_rule("rules_ruby"): diff --git a/third_party/BUILD.bazel b/third_party/BUILD.bazel index d12b3e37f78e..35da5c30f83f 100644 --- a/third_party/BUILD.bazel +++ b/third_party/BUILD.bazel @@ -1,4 +1,5 @@ exports_files([ "BUILD.bazel", + "rules_fuzzing.patch", "zlib.BUILD", ]) diff --git a/third_party/rules_fuzzing.patch b/third_party/rules_fuzzing.patch new file mode 100644 index 000000000000..b522abb73f3a --- /dev/null +++ b/third_party/rules_fuzzing.patch @@ -0,0 +1,25 @@ +diff --git a/fuzzing/tools/validate_dict.py b/fuzzing/tools/validate_dict.py +index d561e68..52cbcb8 100644 +--- a/fuzzing/tools/validate_dict.py ++++ b/fuzzing/tools/validate_dict.py +@@ -19,7 +19,7 @@ Validates and merges a set of fuzzing dictionary files into a single output. + + from absl import app + from absl import flags +-from dict_validation import validate_line ++from fuzzing.tools.dict_validation import validate_line + from sys import stderr + + FLAGS = flags.FLAGS +diff --git a/fuzzing/requirements.txt b/fuzzing/requirements.txt +index 01482d4..4b36f4f 100644 +--- a/fuzzing/requirements.txt ++++ b/fuzzing/requirements.txt +@@ -1,5 +1,5 @@ + # Python requirements for the tools supporting the fuzzing rules. These are + # installed automatically through the WORKSPACE configuration. + +-absl-py==0.11.0 --hash=sha256:b3d9eb5119ff6e0a0125f6dabf2f9fae02f8acae7be70576002fac27235611c5 +-six==1.15.0 --hash=sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced ++absl-py==2.0.0 --hash=sha256:9a28abb62774ae4e8edbe2dd4c49ffcd45a6a848952a5eccc6a49f3f0fc1e2f3 ++six==1.15.0