[DRAFT] Remove --noenable_bzlmod from .bazelrc

Enables bzlmod for most CI tests, but continues to disable bzlmod for python and upb due to use of system vs hermetic python for python/dist rules which will require further work. #test-continuous PiperOrigin-RevId: 705140770
protocolbuffers · Jan 23, 2025 · 752e03a · 752e03a
1 parent 0894265
commit 752e03a
Show file tree

Hide file tree

Showing 11 changed files with 92 additions and 47 deletions.
diff --git a/.bazelrc b/.bazelrc
@@ -31,10 +31,6 @@ build:ubsan --copt=-fno-sanitize=function --copt=-fno-sanitize=vptr
 # Abseil passes nullptr to memcmp with 0 size
 build:ubsan --copt=-fno-sanitize=nonnull-attribute
 
-# TODO: migrate all dependencies from WORKSPACE to MODULE.bazel
-# https://github.com/protocolbuffers/protobuf/issues/14313
-common --noenable_bzlmod
-
 # Important: this flag ensures that we remain compliant with the C++ layering
 # check.
 build --features=layering_check

diff --git a/.github/workflows/test_bazel.yml b/.github/workflows/test_bazel.yml
@@ -58,4 +58,4 @@ jobs:
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           bazel-cache: examples
           version: ${{ matrix.bazelversion }}
-          bash: cd examples && bazel build //... $BAZEL_FLAGS --enable_bzlmod=${{ matrix.bzlmod }} ${{ matrix.toolchain_resolution }}
+          bash: cd examples && bazel build //... $BAZEL_FLAGS --enable_bzlmod=${{ matrix.bzlmod }} --enable_workspace=${{ !matrix.bzlmod }} ${{ matrix.toolchain_resolution }}
diff --git a/.github/workflows/test_cpp.yml b/.github/workflows/test_cpp.yml
@@ -39,14 +39,13 @@ jobs:
           # Set defaults
           - image: us-docker.pkg.dev/protobuf-build/containers/test/linux/sanitize:7.1.2-d9624f2aa83cba3eaf906f751d75b36aacb9aa82
           - targets: //pkg/... //src/... @com_google_protobuf_examples//... //third_party/utf8_range/... //conformance:conformance_framework_tests
-
           # Override cases with custom images
-          - config: { name: "Bazel7", flags: --noenable_bzlmod }
-            cache_key: Bazel7
+          - config: { name: "Bazel7", flags: --cxxopt="-Wno-self-assign-overloaded" }
+            cache_key: Bazel7bzlmod
             image: "us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:7.1.2-27cf7b86212020d7e552bc13b1e084abb971da75"
             targets: "//src/... //third_party/utf8_range/..."
-          - config: { name: "Bazel7 with Bzlmod", flags: --enable_bzlmod --enable_workspace --cxxopt="-Wno-self-assign-overloaded" }
-            cache_key: Bazel7bzlmod
+          - config: { name: "Bazel7 Workspace", flags: --noenable_bzlmod }
+            cache_key: Bazel7nobzlmod
             image: "us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:7.1.2-27cf7b86212020d7e552bc13b1e084abb971da75"
             targets: "//src/... //third_party/utf8_range/..."
           - config: { name: "TCMalloc" }

diff --git a/.github/workflows/test_java.yml b/.github/workflows/test_java.yml
@@ -34,7 +34,9 @@ jobs:
             # TODO: b/318555165 - enable the layering check. Currently it does
             # not work correctly with the toolchain in this Docker image.
             targets: //java/... //java/internal:java_version //compatibility/... --features=-layering_check
-            flags: --java_language_version=8
+            # TODO: Error while fetching artifact with coursier: Error: Could not find or load main class
+            # https://github.com/protocolbuffers/protobuf/actions/runs/12658418098/job/35275386344?pr=19898
+            flags: --java_language_version=8 --noenable_bzlmod
           - name: OpenJDK 11
             cache_key: '11'
             image: us-docker.pkg.dev/protobuf-build/containers/test/linux/java:7.1.2-11-d9624f2aa83cba3eaf906f751d75b36aacb9aa82
@@ -45,15 +47,14 @@ jobs:
             image: us-docker.pkg.dev/protobuf-build/containers/test/linux/java:7.1.2-17-641278a52587c01f44525819b76499db35c2804d
             targets: //java/... //java/internal:java_version //compatibility/...
           - name: Bazel7
-            cache_key: 'bazel7nobzlmod'
+            cache_key: 'bazel7bzlmod'
             image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:7.1.2-27cf7b86212020d7e552bc13b1e084abb971da75
             targets: //java/... //java/internal:java_version //compatibility/...
-            flags: --noenable_bzlmod
-          - name: Bazel7 with Bzlmod
-            cache_key: 'bazel7bzlmod'
+          - name: Bazel7 Workspace
+            cache_key: 'bazel7nobzlmod'
             image: us-docker.pkg.dev/protobuf-build/containers/common/linux/bazel:7.1.2-27cf7b86212020d7e552bc13b1e084abb971da75
             targets: //java/... //java/internal:java_version //compatibility/...
-            flags: --enable_bzlmod --enable_workspace
+            flags: --noenable_bzlmod
           - name: aarch64
             cache_key: 'aarch64'
             image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:7.1.2-aarch64-2920199ab0090ed427413a8e422e62695c8392a8

diff --git a/.github/workflows/test_python.yml b/.github/workflows/test_python.yml
@@ -29,19 +29,16 @@ jobs:
       matrix:
         type: [ Pure, C++]
         version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+        # TODO: ERROR: error loading package 'python/dist': cannot load '@@rules_python~~python~python_3_12//:version.bzl': no such file
+        # https://github.com/protocolbuffers/protobuf/actions/runs/12470144355/job/34804733978?pr=19764
+        bzlmod: [--noenable_bzlmod]
         include:
           - type: Pure
             targets: //python/... //python:python_version_test
             flags: --define=use_fast_cpp_protos=false
           - type: C++
             targets: //python/... //python:python_version_test
             flags: --define=use_fast_cpp_protos=true
-          - type: C++
-            version: aarch64
-            targets: //python/... //python:aarch64_test
-            # TODO Enable this once conformance tests are fixed.
-            flags: --define=use_fast_cpp_protos=true --test_tag_filters=-conformance
-            image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:7.1.2-aarch64-2920199ab0090ed427413a8e422e62695c8392a8
           - version: "3.9"
           - version: "3.10"
             continuous-only: true
@@ -50,6 +47,14 @@ jobs:
           - version: "3.12"
             continuous-only: true
           - version: "3.13"
+          - type: C++
+            version: aarch64
+            python-version: "3.9"
+            targets: //python/... //python:aarch64_test
+            # TODO Enable this once conformance tests are fixed.
+            flags: --define=use_fast_cpp_protos=true --test_tag_filters=-conformance
+            bzlmod: --noenable_bzlmod
+            image: us-docker.pkg.dev/protobuf-build/containers/test/linux/emulation:7.1.2-aarch64-2920199ab0090ed427413a8e422e62695c8392a8
 
     name: ${{ matrix.continuous-only && inputs.continuous-prefix || '' }} Linux ${{ matrix.type }} ${{ matrix.version }}
     runs-on: ubuntu-latest
@@ -66,7 +71,8 @@ jobs:
           image: ${{ matrix.image || format('us-docker.pkg.dev/protobuf-build/containers/test/linux/python:7.1.2-{0}-d9624f2aa83cba3eaf906f751d75b36aacb9aa82', matrix.version) }}
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           bazel-cache: python_linux/${{ matrix.type }}_${{ matrix.version }}
-          bazel: test ${{ matrix.targets }} ${{ matrix.flags }} --test_env=KOKORO_PYTHON_VERSION
+          python-version: ${{ matrix.version }}
+          bazel: test ${{ matrix.targets }} ${{ matrix.flags }} ${{ matrix.bzlmod }} --test_env=SYSTEM_PYTHON_VERSION=${{ matrix.python-version }}
 
 
   macos:
@@ -75,6 +81,9 @@ jobs:
       matrix:
         type: [ Pure, C++]
         version: [ "3.12", "3.13" ]
+        # TODO: ERROR: error loading package 'python/dist': cannot load '@@rules_python~~python~python_3_12//:version.bzl': no such file
+        # https://github.com/protocolbuffers/protobuf/actions/runs/12674986130/job/35325012489?pr=19898
+        bzlmod: [--noenable_bzlmod]
         include:
           - type: Pure
             targets: //python/... //python:python_version_test
@@ -83,6 +92,7 @@ jobs:
             flags: --define=use_fast_cpp_protos=true
           - version: "3.13"
             continuous-only: true
+          - bzlmod: --noenable_bzlmod
 
     name: MacOS ${{ matrix.type }} ${{ matrix.version }}
     runs-on: macos-13
@@ -109,13 +119,13 @@ jobs:
 
       - name: Run tests
         uses: protocolbuffers/protobuf-ci/bazel@v4
-        env:
-          KOKORO_PYTHON_VERSION: ${{ matrix.version }}
+        # env:
+        #   SYSTEM_PYTHON_VERSION: ${{ matrix.version }}
         with:
           version: 7.1.2 # Bazel version
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           bazel-cache: python_macos/${{ matrix.type }}_${{ matrix.version }}
           bazel: >-
-            test ${{ matrix.targets }} ${{ matrix.flags }}
-            --test_env=KOKORO_PYTHON_VERSION=${{ matrix.version }}
+            test ${{ matrix.targets }} ${{ matrix.flags }} ${{ matrix.bzlmod }}
+            --test_env=SYSTEM_PYTHON_VERSION=${{ matrix.version }}
             --macos_minimum_os=11.0
diff --git a/.github/workflows/test_upb.yml b/.github/workflows/test_upb.yml
@@ -52,7 +52,9 @@ jobs:
           image: us-docker.pkg.dev/protobuf-build/containers/test/linux/sanitize:${{ matrix.config.bazel_version || '7.1.2' }}-d9624f2aa83cba3eaf906f751d75b36aacb9aa82
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           bazel-cache: upb-bazel
-          bazel: test --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 //bazel/... //benchmarks/... //lua/... //python/... //upb/... //upb_generator/... ${{ matrix.config.flags }}
+          # TODO: ERROR: error loading package 'python/dist': cannot load '@@rules_python~~python~python_3_12//:version.bzl': no such file
+          # https://github.com/protocolbuffers/protobuf/actions/runs/12470144355/job/34804733978?pr=19764
+          bazel: test --noenable_bzlmod --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 //bazel/... //benchmarks/... //lua/... //python/... //upb/... //upb_generator/... ${{ matrix.config.flags }}
           exclude-targets: ${{ matrix.config.exclude-targets }}
 
   linux-gcc:
@@ -72,7 +74,7 @@ jobs:
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           bazel-cache: "upb-bazel-gcc"
           bazel: >-
-            test --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 -c opt
+            test --noenable_bzlmod --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 -c opt
             --copt="-Wno-error=maybe-uninitialized" --copt="-Wno-error=attributes"
             //bazel/... //benchmarks/... //lua/... //python/... //upb/... //upb_generator/...
 
@@ -96,7 +98,7 @@ jobs:
         with:
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           bazel-cache: "upb-bazel-windows"
-          bazel: test --cxxopt=/std:c++17 --host_cxxopt=/std:c++17 //upb/... //upb_generator/... //python/...
+          bazel: test --noenable_bzlmod --cxxopt=/std:c++17 --host_cxxopt=/std:c++17 //upb/... //upb_generator/... //python/...
           version: 7.1.2
           exclude-targets: -//python:conformance_test -//upb/reflection:def_builder_test
 
@@ -125,7 +127,7 @@ jobs:
         with:
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           bazel-cache: "upb-bazel-macos"
-          bazel: ${{ matrix.config.bazel-command }} --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 ${{ matrix.config.flags }} //bazel/... //benchmarks/... //lua/... //python/... //upb/... //upb_generator/...
+          bazel: ${{ matrix.config.bazel-command }} --noenable_bzlmod --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 ${{ matrix.config.flags }} //bazel/... //benchmarks/... //lua/... //python/... //upb/... //upb_generator/...
           version: 7.1.2
 
   no-python:
@@ -148,7 +150,7 @@ jobs:
             which python3 &&
             mv `which python3` /tmp &&
             ! which python3 &&
-            bazel test $BAZEL_FLAGS --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 //python/... -- -//python/dist:source_wheel -//python:aarch64_test -//python:x86_64_test -//python:google/protobuf/pyext/_message.so -//python:proto_api
+            bazel test $BAZEL_FLAGS --noenable_bzlmod --cxxopt=-std=c++17 --host_cxxopt=-std=c++17 //python/... -- -//python/dist:source_wheel -//python:aarch64_test -//python:x86_64_test -//python:google/protobuf/pyext/_message.so -//python:proto_api
 
   build_wheels:
     name: Build Wheels
@@ -165,7 +167,7 @@ jobs:
           image: us-docker.pkg.dev/protobuf-build/release-containers/linux/apple:7.1.2-d9624f2aa83cba3eaf906f751d75b36aacb9aa82
           credentials: ${{ secrets.GAR_SERVICE_ACCOUNT }}
           bazel-cache: upb-bazel-python
-          bazel: build --crosstool_top=//toolchain:clang_suite --//toolchain:release=true --symlink_prefix=/ -c dbg --incompatible_enable_cc_toolchain_resolution=false //python/dist //python/dist:test_wheel //python/dist:source_wheel
+          bazel: build --noenable_bzlmod --crosstool_top=//toolchain:clang_suite --//toolchain:release=true --symlink_prefix=/ -c dbg --incompatible_enable_cc_toolchain_resolution=false //python/dist //python/dist:test_wheel //python/dist:source_wheel
       - name: Move Wheels
         run: mkdir wheels && find _build/out \( -name 'protobuf*.whl' -o -name 'protobuf-*.tar.gz' \) -exec mv '{}' wheels ';'
       - uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0

diff --git a/MODULE.bazel b/MODULE.bazel
@@ -15,6 +15,7 @@ module(
 bazel_dep(name = "abseil-cpp", version = "20240722.0", repo_name = "com_google_absl")
 bazel_dep(name = "bazel_skylib", version = "1.7.0")
 bazel_dep(name = "jsoncpp", version = "1.9.6")
+bazel_dep(name = "rules_apple", version = "3.13.0", repo_name = "build_bazel_rules_apple")
 bazel_dep(name = "apple_support", version = "1.15.1", repo_name = "build_bazel_apple_support")
 bazel_dep(name = "rules_cc", version = "0.0.16")
 bazel_dep(name = "rules_fuzzing", version = "0.5.2")
@@ -43,21 +44,20 @@ SUPPORTED_PYTHON_VERSIONS = [
     "3.12",
 ]
 
+# TODO: Replace system_python for //python/dist
 python = use_extension("@rules_python//python/extensions:python.bzl", "python")
 
 [
     python.toolchain(
+        # Disable root warning for .pyc cache misses since CI runs as root.
+        # See https://github.com/bazelbuild/rules_python/pull/713
+        ignore_root_user_error = True,
         is_default = python_version == SUPPORTED_PYTHON_VERSIONS[-1],
         python_version = python_version,
     )
     for python_version in SUPPORTED_PYTHON_VERSIONS
 ]
 
-use_repo(
-    python,
-    system_python = "python_{}".format(SUPPORTED_PYTHON_VERSIONS[-1].replace(".", "_")),
-)
-
 pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip", dev_dependency = True)
 
 [

diff --git a/WORKSPACE.bzlmod b/WORKSPACE.bzlmod
@@ -21,8 +21,50 @@ ruby_runtime("system_ruby")
 
 register_toolchains("@system_ruby//:toolchain")
 
+load("@system_ruby//:bundle.bzl", "ruby_bundle")
+
+ruby_bundle(
+    name = "protobuf_bundle",
+    srcs = ["//ruby:google-protobuf.gemspec"],
+    gemfile = "//ruby:Gemfile",
+)
+
+# For testing UPB.
+http_archive(
+    name = "lua",
+    build_file = "//python/dist:lua.BUILD",
+    sha256 = "b9e2e4aad6789b3b63a056d442f7b39f0ecfca3ae0f1fc0ae4e9614401b69f4b",
+    strip_prefix = "lua-5.2.4",
+    urls = [
+        "https://mirror.bazel.build/www.lua.org/ftp/lua-5.2.4.tar.gz",
+        "https://www.lua.org/ftp/lua-5.2.4.tar.gz",
+    ],
+)
+
+http_archive(
+    name = "com_github_google_benchmark",
+    sha256 = "62e2f2e6d8a744d67e4bbc212fcfd06647080de4253c97ad5c6749e09faf2cb0",
+    strip_prefix = "benchmark-0baacde3618ca617da95375e0af13ce1baadea47",
+    urls = ["https://github.com/google/benchmark/archive/0baacde3618ca617da95375e0af13ce1baadea47.zip"],
+)
+
+http_archive(
+    name = "com_google_googleapis",
+    build_file = "//benchmarks:BUILD.googleapis",
+    patch_cmds = ["find google -type f -name BUILD.bazel -delete"],
+    sha256 = "d986023c3d8d2e1b161e9361366669cac9fb97c2a07e656c2548aca389248bb4",
+    strip_prefix = "googleapis-d81d0b9e6993d6ab425dff4d7c3d05fb2e59fa57",
+    urls = ["https://github.com/googleapis/googleapis/archive/d81d0b9e6993d6ab425dff4d7c3d05fb2e59fa57.zip"],
+)
+
 # Following are just needed to run conformance tests, not really needed to support them via MODULE.bazel
 
+# For testing from another repo. TODO: local_path_override(module_name = "com_google_protobuf_examples", path = "examples")
+local_repository(
+    name = "com_google_protobuf_examples",
+    path = "examples",
+)
+
 # For testing runtime against old gencode from a previous major version.
 http_archive(
     name = "com_google_protobuf_v25.0",
@@ -35,7 +77,6 @@ http_archive(
 load("@com_google_protobuf_v25.0//:protobuf_deps.bzl", protobuf_v25_deps="protobuf_deps")
 protobuf_v25_deps()
 
-
 # Needed for checking breaking changes from the previous release version.
 load("//:protobuf_version.bzl", "PROTOBUF_PREVIOUS_RELEASE")
 

diff --git a/ci/common.bazelrc b/ci/common.bazelrc
@@ -76,10 +76,6 @@ build --incompatible_use_host_features
 #   --incompatible_fail_on_unknown_attributes
 #   --incompatible_merge_fixed_and_default_shell_env
 
-# TODO: migrate all dependencies from WORKSPACE to MODULE.bazel
-# https://github.com/protocolbuffers/protobuf/issues/14313
-common --noenable_bzlmod
-
 # For easier debugging of build failures.
 common --announce_rc
 build --verbose_failures

diff --git a/ci/macOS.bazelrc b/ci/macOS.bazelrc
@@ -3,7 +3,7 @@ import common.bazelrc
 build --cxxopt=-std=c++17 --host_cxxopt=-std=c++17
 build --cxxopt="-Woverloaded-virtual"
 build --copt="-Wpointer-arith"
-build --copt="-Werror" --copt="-Wno-sign-compare" --copt="-Wno-sign-conversion" --copt="-Wno-error=sign-conversion" --copt="-Wno-deprecated-declarations"
+build --copt="-Werror" --copt="-Wno-sign-compare" --copt="-Wno-sign-conversion" --copt="-Wno-error=sign-conversion" --copt="-Wno-deprecated-declarations" --copt="-Wno-unknown-warning-option"
 common --xcode_version_config=@com_google_protobuf//.github:host_xcodes
 
 build --apple_crosstool_top=@local_config_apple_cc//:toolchain

diff --git a/python/python_version_test.py b/python/python_version_test.py
@@ -4,7 +4,7 @@
 # Use of this source code is governed by a BSD-style
 # license that can be found in the LICENSE file or at
 # https://developers.google.com/open-source/licenses/bsd
-"""Test that Kokoro is using the expected version of python."""
+"""Test that test runner is using the expected version of python."""
 
 import os
 import sys
@@ -16,9 +16,9 @@ class PythonVersionTest(unittest.TestCase):
   def testPython3(self):
     """Test that we can import nested import public messages."""
 
-    exp = os.getenv('KOKORO_PYTHON_VERSION', '')
+    exp = os.getenv('SYSTEM_PYTHON_VERSION', '')
     if not exp:
-      print('No kokoro python version found, skipping check', file=sys.stderr)
+      print('No system python version found, skipping check', file=sys.stderr)
       return
     self.assertTrue(
         sys.version.startswith(exp),