Skip to content

Commit

Permalink
Bazelize Protobuf
Browse files Browse the repository at this point in the history
Change-Id: I4073b4025d43ec077d77f009f473c2fc32dea911
  • Loading branch information
Alex Zurhake committed Nov 7, 2023
1 parent 17eec29 commit 15fd22f
Show file tree
Hide file tree
Showing 10 changed files with 178 additions and 118 deletions.
9 changes: 9 additions & 0 deletions WORKSPACE
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,8 @@ load(
"MOD_WSGI_VERSION",
"NET_SNMP_SHA256",
"NET_SNMP_VERSION",
"PROTOBUF_SHA256",
"PROTOBUF_VERSION",
)
load("//omd/packages/patch:patch_http.bzl", "patch")

Expand Down Expand Up @@ -272,3 +274,10 @@ netsnmp(
sha256 = NET_SNMP_SHA256,
version_str = NET_SNMP_VERSION,
)

load("//omd/packages/protobuf:protobuf_http.bzl", "protobuf")

protobuf(
sha256 = PROTOBUF_SHA256,
version_str = PROTOBUF_VERSION,
)
Empty file added omd/packages/protobuf/BUILD
Empty file.
94 changes: 94 additions & 0 deletions omd/packages/protobuf/BUILD.protobuf.bazel
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
load("@rules_foreign_cc//foreign_cc:defs.bzl", "configure_make")
load("@omd_packages//:package_versions.bzl", "PROTOBUF_LIB_VERSION", "PYTHON_MAJOR_DOT_MINOR")

# For unknown reasons the version of the libraries differes from the package version
PROTOBUF_LIB_VERSION_ARRAY = PROTOBUF_LIB_VERSION.split(".")

PROTOBUF_LIB_VERSION_MAJOR = PROTOBUF_LIB_VERSION_ARRAY[0]

MAKE_PARALLEL = "-j4"

filegroup(
name = "all_srcs",
srcs = glob(
["**/*"],
exclude = ["bazel-*"],
),
)

configure_make(
name = "protobuf",
args = ["%s" % MAKE_PARALLEL],
configure_in_place = True,
configure_options = [
"-v",
"--disable-option-checking",
"--prefix=/",
],
copts = [
"-I$BUILD_TMPDIR/include",
"-I${EXT_BUILD_DEPS}/python/include/python%s/" % PYTHON_MAJOR_DOT_MINOR,
"-I$EXT_BUILD_DEPS/python/lib/",
],
env = {
"ORIGIN_VAR": "\\\\$$\\$$ORIGIN",
"RANLIB": "ranlib",
"LD_LIBRARY_PATH": "$$EXT_BUILD_DEPS/python/lib/",
"HOME": "/home/\\$$\\$${USER}/",
"CXXFLAGS": "-Wno-stringop-overflow",
},
lib_source = ":all_srcs",
out_binaries = [
"protoc",
],
out_data_dirs = [
"share/man/",
"lib/python%s/site-packages/" % PYTHON_MAJOR_DOT_MINOR,
],
out_shared_libs = [
"libprotobuf-lite.so",
"libprotobuf-lite.so.%s" % PROTOBUF_LIB_VERSION_MAJOR,
"libprotobuf-lite.so.%s" % PROTOBUF_LIB_VERSION,
"libprotobuf.so",
"libprotobuf.so.%s" % PROTOBUF_LIB_VERSION_MAJOR,
"libprotobuf.so.%s" % PROTOBUF_LIB_VERSION,
"libprotoc.so",
"libprotoc.so.%s" % PROTOBUF_LIB_VERSION_MAJOR,
"libprotoc.so.%s" % PROTOBUF_LIB_VERSION,
],
out_static_libs = [
"libprotobuf-lite.a",
"libprotobuf.a",
"libprotoc.a",
],
postfix_script = """
set -ex
# static compilation
# Hack needed for protoc to be linked statically. Tried a lot of different things to make it
# work with the standard Makefile and libtool stuff, but had no luck. It always ended with a
# protoc with dynamic dependencies on libgcc and libstdc++. And we really need to have a
# statically linked binary at the moment. The following is a hand crafted linker command.
# Let me know in case you got a cleaner approach.
cd src
rm protoc
echo -e '\nprotoc-static: $(protoc_OBJECTS) $(protoc_DEPENDENCIES) $(EXTRA_protoc_DEPENDENCIES)\n\tg++ -pthread -DHAVE_PTHREAD=1 -DHAVE_ZLIB=1 -Wall -Wno-sign-compare -static-libgcc -static-libstdc++ -s -o protoc google/protobuf/compiler/main.o -lpthread ./.libs/libprotoc.a ./.libs/libprotobuf.a' >> Makefile && \
make %s protoc-static && \
file protoc | grep ELF >/dev/null && \
ldd protoc | grep -v libstdc++ >/dev/null
cd -
# compile python package
cd python
CPPFLAGS=-I${EXT_BUILD_DEPS}/python/include/python%s/ ${EXT_BUILD_DEPS}/python/bin/python3 setup.py build --cpp_implementation
${EXT_BUILD_DEPS}/python/bin/python3 setup.py install \
--cpp_implementation \
--root=${INSTALLDIR} \
--prefix=''
cd -
# install
make %s DESTDIR=${INSTALLDIR} install
""" % (MAKE_PARALLEL, PYTHON_MAJOR_DOT_MINOR, MAKE_PARALLEL),
targets = [
"", # default target is install, but we only want to build at first
],
deps = ["@python"],
)
3 changes: 0 additions & 3 deletions omd/packages/protobuf/download

This file was deleted.

37 changes: 37 additions & 0 deletions omd/packages/protobuf/patches/04-linking-error.dif
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
# https://github.com/protocolbuffers/protobuf/pull/11032
diff -ruN protobuf-21.9-orig/src/libprotobuf-lite.map protobuf-21.9/src/libprotobuf-lite.map
--- protobuf-21.9-orig/src/libprotobuf-lite.map 2022-12-18 15:56:13.701778705 +0100
+++ protobuf-21.9/src/libprotobuf-lite.map 2022-12-18 15:57:06.973681257 +0100
@@ -3,6 +3,8 @@
extern "C++" {
*google*;
};
+ scc_info_*;
+ descriptor_table_*;

local:
*;
diff -ruN protobuf-21.9-orig/src/libprotobuf.map protobuf-21.9/src/libprotobuf.map
--- protobuf-21.9-orig/src/libprotobuf.map 2022-12-18 15:56:13.702992299 +0100
+++ protobuf-21.9/src/libprotobuf.map 2022-12-18 15:57:11.786965226 +0100
@@ -3,6 +3,8 @@
extern "C++" {
*google*;
};
+ scc_info_*;
+ descriptor_table_*;

local:
*;
diff -ruN protobuf-21.9-orig/src/libprotoc.map protobuf-21.9/src/libprotoc.map
--- protobuf-21.9-orig/src/libprotoc.map 2022-12-18 15:56:13.704384023 +0100
+++ protobuf-21.9/src/libprotoc.map 2022-12-18 15:57:15.907564998 +0100
@@ -3,6 +3,8 @@
extern "C++" {
*google*;
};
+ scc_info_*;
+ descriptor_table_*;

local:
*;
Empty file.
Binary file removed omd/packages/protobuf/protobuf-python-3.20.1.tar.gz
Binary file not shown.
125 changes: 10 additions & 115 deletions omd/packages/protobuf/protobuf.make
Original file line number Diff line number Diff line change
@@ -1,33 +1,12 @@
# This package builds the python protobuf module and also protoc (for tests)
PROTOBUF := protobuf
PROTOBUF_VERS := 3.20.1
PROTOBUF_DIR := $(PROTOBUF)-$(PROTOBUF_VERS)
# Increase this to enforce a recreation of the build cache
PROTOBUF_BUILD_ID := 6
# The cached package contains the python major/minor version, so include this in the cache name in order to trigger
# a rebuild on a python version change.
PROTOBUF_BUILD_ID := $(PROTOBUF_BUILD_ID)-python$(PYTHON_MAJOR_DOT_MINOR)
PROTOBUF_DIR := $(PROTOBUF)

PROTOBUF_PATCHING := $(BUILD_HELPER_DIR)/$(PROTOBUF_DIR)-patching
PROTOBUF_CONFIGURE := $(BUILD_HELPER_DIR)/$(PROTOBUF_DIR)-configure
PROTOBUF_UNPACK := $(BUILD_HELPER_DIR)/$(PROTOBUF_DIR)-unpack
PROTOBUF_BUILD := $(BUILD_HELPER_DIR)/$(PROTOBUF_DIR)-build
PROTOBUF_BUILD_PYTHON := $(BUILD_HELPER_DIR)/$(PROTOBUF_DIR)-build-python
PROTOBUF_BUILD_LIBRARY := $(BUILD_HELPER_DIR)/$(PROTOBUF_DIR)-build-library
PROTOBUF_INTERMEDIATE_INSTALL := $(BUILD_HELPER_DIR)/$(PROTOBUF_DIR)-install-intermediate
PROTOBUF_INTERMEDIATE_INSTALL_PYTHON := $(BUILD_HELPER_DIR)/$(PROTOBUF_DIR)-install-intermediate-python
PROTOBUF_INTERMEDIATE_INSTALL_LIBRARY := $(BUILD_HELPER_DIR)/$(PROTOBUF_DIR)-install-intermediate-library
PROTOBUF_CACHE_PKG_PROCESS := $(BUILD_HELPER_DIR)/$(PROTOBUF_DIR)-cache-pkg-process
PROTOBUF_CACHE_PKG_PROCESS_PYTHON := $(BUILD_HELPER_DIR)/$(PROTOBUF_DIR)-cache-pkg-process-python
PROTOBUF_CACHE_PKG_PROCESS_LIBRARY := $(BUILD_HELPER_DIR)/$(PROTOBUF_DIR)-cache-pkg-process-library
PROTOBUF_INSTALL := $(BUILD_HELPER_DIR)/$(PROTOBUF_DIR)-install
PROTOBUF_INSTALL_PYTHON := $(BUILD_HELPER_DIR)/$(PROTOBUF_DIR)-install-python
PROTOBUF_INSTALL_LIBRARY := $(BUILD_HELPER_DIR)/$(PROTOBUF_DIR)-install-library

PROTOBUF_INSTALL_DIR_PYTHON := $(INTERMEDIATE_INSTALL_BASE)/$(PROTOBUF_DIR)-python
PROTOBUF_INSTALL_DIR_LIBRARY := $(INTERMEDIATE_INSTALL_BASE)/$(PROTOBUF_DIR)-library
PROTOBUF_BUILD_DIR := $(PACKAGE_BUILD_DIR)/$(PROTOBUF_DIR)
#PROTOBUF_WORK_DIR := $(PACKAGE_WORK_DIR)/$(PROTOBUF_DIR)
PROTOBUF_INSTALL_DIR := $(INTERMEDIATE_INSTALL_BASE)/$(PROTOBUF_DIR)

# Used by other OMD packages
PACKAGE_PROTOBUF_DESTDIR := $(PROTOBUF_INSTALL_DIR_LIBRARY)
Expand All @@ -36,95 +15,16 @@ PACKAGE_PROTOBUF_LD_LIBRARY_PATH := $(PACKAGE_PROTOBUF_DESTDIR)/lib
PACKAGE_PROTOBUF_INCLUDE_PATH := $(PACKAGE_PROTOBUF_DESTDIR)/include/google/protobuf
PACKAGE_PROTOBUF_PROTOC_BIN := $(PACKAGE_PROTOBUF_DESTDIR)/bin/protoc

$(PROTOBUF)-build-library: $(BUILD_HELPER_DIR) $(PROTOBUF_CACHE_PKG_PROCESS_LIBRARY)
$(PROTOBUF)-build-library: $(PROTOBUF_INTERMEDIATE_INSTALL)

# We have a globally defined $(PROTOBUF_UNPACK) target, but we need some special
# handling here, because downloaded archive name does not match the omd package name
$(PROTOBUF_UNPACK): $(PACKAGE_DIR)/$(PROTOBUF)/protobuf-python-$(PROTOBUF_VERS).tar.gz
$(RM) -r $(PROTOBUF_BUILD_DIR)
$(MKDIR) $(PACKAGE_BUILD_DIR)
$(TAR_GZ) $< -C $(PACKAGE_BUILD_DIR)
$(MKDIR) $(BUILD_HELPER_DIR)
$(TOUCH) $@
$(PROTOBUF_BUILD):
$(BAZEL_BUILD) @$(PROTOBUF)//:$(PROTOBUF)

# NOTE: We can probably remove the CXXFLAGS hack below when we use a more recent
# protobuf version. Currently -Wall is enabled for builds with GCC, and newer
# GCC versions complain.
$(PROTOBUF_CONFIGURE): $(PROTOBUF_PATCHING)
cd $(PROTOBUF_BUILD_DIR) && \
export LD_LIBRARY_PATH="$(PACKAGE_PYTHON_LD_LIBRARY_PATH)" && \
CXXFLAGS="-Wno-stringop-overflow" ./configure --prefix=""
$(TOUCH) $@
$(PROTOBUF_INTERMEDIATE_INSTALL): $(PROTOBUF_BUILD)
$(MKDIR) $(PROTOBUF_INSTALL_DIR)
$(RSYNC) --chmod=u+w $(BAZEL_BIN_EXT)/$(PROTOBUF)/$(PROTOBUF)/ $(PROTOBUF_INSTALL_DIR)

$(PROTOBUF_BUILD_LIBRARY): $(PROTOBUF_CONFIGURE)
cd $(PROTOBUF_BUILD_DIR) && \
make -j6 && \
`: Hack needed for protoc to be linked statically. Tried a lot of different things to make it ` \
`: work with the standard Makefile and libtool stuff, but had no luck. It always ended with a ` \
`: protoc with dynamic dependencies on libgcc and libstdc++. And we really need to have a ` \
`: statically linked binary at the moment. The following is a hand crafted linker command. ` \
`: Let me know in case you got a cleaner approach. ` \
cd src && \
rm protoc && \
echo -e '\nprotoc-static: $(protoc_OBJECTS) $(protoc_DEPENDENCIES) $(EXTRA_protoc_DEPENDENCIES)\n\tg++ -pthread -DHAVE_PTHREAD=1 -DHAVE_ZLIB=1 -Wall -Wno-sign-compare -static-libgcc -static-libstdc++ -s -o protoc google/protobuf/compiler/main.o -lpthread ./.libs/libprotoc.a ./.libs/libprotobuf.a' >> Makefile && \
make -j6 protoc-static && \
file $(PROTOBUF_BUILD_DIR)/src/protoc | grep ELF >/dev/null && \
ldd $(PROTOBUF_BUILD_DIR)/src/protoc | grep -v libstdc++ >/dev/null
$(TOUCH) $@

$(PROTOBUF_BUILD_PYTHON): $(PROTOBUF_BUILD_LIBRARY) $(PYTHON_CACHE_PKG_PROCESS)
cd $(PROTOBUF_BUILD_DIR)/python && \
export LD_LIBRARY_PATH="$(PACKAGE_PYTHON_LD_LIBRARY_PATH)" && \
$(PACKAGE_PYTHON_EXECUTABLE) setup.py build --cpp_implementation
$(TOUCH) $@

$(PROTOBUF_BUILD): $(PROTOBUF_BUILD_LIBRARY) $(PROTOBUF_BUILD_PYTHON)
file $(PROTOBUF_BUILD_DIR)/src/protoc | grep ELF >/dev/null
ldd $(PROTOBUF_BUILD_DIR)/src/protoc | grep -v libstdc++ >/dev/null
$(TOUCH) $@

$(PROTOBUF_CACHE_PKG_PROCESS): $(PROTOBUF_CACHE_PKG_PROCESS_PYTHON) $(PROTOBUF_CACHE_PKG_PROCESS_LIBRARY)

PROTOBUF_CACHE_PKG_PATH_PYTHON := $(call cache_pkg_path,$(PROTOBUF_DIR)-python,$(PROTOBUF_BUILD_ID))

$(PROTOBUF_CACHE_PKG_PATH_PYTHON):
$(call pack_pkg_archive,$@,$(PROTOBUF_DIR)-python,$(PROTOBUF_BUILD_ID),$(PROTOBUF_INTERMEDIATE_INSTALL_PYTHON))

$(PROTOBUF_CACHE_PKG_PROCESS_PYTHON): $(PROTOBUF_CACHE_PKG_PATH_PYTHON)
$(call unpack_pkg_archive,$(PROTOBUF_CACHE_PKG_PATH_PYTHON),$(PROTOBUF_DIR)-python)
$(call upload_pkg_archive,$(PROTOBUF_CACHE_PKG_PATH_PYTHON),$(PROTOBUF_DIR)-python,$(PROTOBUF_BUILD_ID))
$(TOUCH) $@

PROTOBUF_CACHE_PKG_PATH_LIBRARY := $(call cache_pkg_path,$(PROTOBUF_DIR)-library,$(PROTOBUF_BUILD_ID))

$(PROTOBUF_CACHE_PKG_PATH_LIBRARY):
$(call pack_pkg_archive,$@,$(PROTOBUF_DIR)-library,$(PROTOBUF_BUILD_ID),$(PROTOBUF_INTERMEDIATE_INSTALL_LIBRARY))

$(PROTOBUF_CACHE_PKG_PROCESS_LIBRARY): $(PROTOBUF_CACHE_PKG_PATH_LIBRARY)
$(call unpack_pkg_archive,$(PROTOBUF_CACHE_PKG_PATH_LIBRARY),$(PROTOBUF_DIR)-library)
$(call upload_pkg_archive,$(PROTOBUF_CACHE_PKG_PATH_LIBRARY),$(PROTOBUF_DIR)-library,$(PROTOBUF_BUILD_ID))
$(TOUCH) $@

$(PROTOBUF_INTERMEDIATE_INSTALL): $(PROTOBUF_INTERMEDIATE_INSTALL_PYTHON) $(PROTOBUF_INTERMEDIATE_INSTALL_LIBRARY)

$(PROTOBUF_INTERMEDIATE_INSTALL_LIBRARY): $(PROTOBUF_BUILD_LIBRARY)
file $(PROTOBUF_BUILD_DIR)/src/protoc | grep ELF >/dev/null
ldd $(PROTOBUF_BUILD_DIR)/src/protoc | grep -v libstdc++ >/dev/null
make -C $(PROTOBUF_BUILD_DIR) DESTDIR=$(PROTOBUF_INSTALL_DIR_LIBRARY) install
$(TOUCH) $@

$(PROTOBUF_INTERMEDIATE_INSTALL_PYTHON): $(PROTOBUF_BUILD_PYTHON)
cd $(PROTOBUF_BUILD_DIR)/python && \
export LD_LIBRARY_PATH="$(PACKAGE_PYTHON_LD_LIBRARY_PATH)" && \
$(PACKAGE_PYTHON_EXECUTABLE) setup.py install \
--cpp_implementation \
--root=$(PROTOBUF_INSTALL_DIR_PYTHON) \
--prefix=''
$(TOUCH) $@

$(PROTOBUF_INSTALL): $(PROTOBUF_INSTALL_LIBRARY) $(PROTOBUF_INSTALL_PYTHON)

$(PROTOBUF_INSTALL_LIBRARY): $(PROTOBUF_CACHE_PKG_PROCESS_LIBRARY)
$(PROTOBUF_INSTALL): $(PROTOBUF_INTERMEDIATE_INSTALL)
# Only install the libraries we really need in run time environment. The
# PROTOBUF_INTERMEDIATE_INSTALL_LIBRARY step above installs the libprotobuf.a
# for building the cmc. However, this is not needed later in runtime environment.
Expand All @@ -137,10 +37,5 @@ $(PROTOBUF_INSTALL_LIBRARY): $(PROTOBUF_CACHE_PKG_PROCESS_LIBRARY)
--exclude 'libprotoc*' \
--exclude 'libprotobuf-lite.*' \
--exclude 'protobuf-lite.pc' \
$(PROTOBUF_INSTALL_DIR_LIBRARY)/ $(DESTDIR)$(OMD_ROOT)/
$(TOUCH) $@

$(PROTOBUF_INSTALL_DIR)/ $(DESTDIR)$(OMD_ROOT)/

$(PROTOBUF_INSTALL_PYTHON): $(PROTOBUF_CACHE_PKG_PROCESS_PYTHON)
$(RSYNC) $(PROTOBUF_INSTALL_DIR_PYTHON)/ $(DESTDIR)$(OMD_ROOT)/
$(TOUCH) $@
24 changes: 24 additions & 0 deletions omd/packages/protobuf/protobuf_http.bzl
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
load("//:bazel_variables.bzl", "UPSTREAM_MIRROR_URL")

def protobuf(version_str, sha256):
filename = "protobuf-python-" + version_str + ".tar.gz"
http_archive(
name = "protobuf",
build_file = "@omd_packages//omd/packages/protobuf:BUILD.protobuf.bazel",
strip_prefix = "protobuf-" + version_str,
urls = [
"https://github.com/protocolbuffers/protobuf/releases/download/v" + version_str + "/" + filename,
UPSTREAM_MIRROR_URL + filename,
],
patches = [
"//omd/packages/protobuf/patches:00-unused-parameters.dif",
"//omd/packages/protobuf/patches:01-tweaks-for-iwyu.dif",
"//omd/packages/protobuf/patches:02-Fix-build-with-Python-3.11.dif",
"//omd/packages/protobuf/patches:03-noreturn-fix.dif",
"//omd/packages/protobuf/patches:04-linking-error.dif",
],
patch_args = ["-p1"],
patch_tool = "patch",
sha256 = sha256,
)
4 changes: 4 additions & 0 deletions package_versions.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -68,3 +68,7 @@ MOD_WSGI_SHA256 = "ee926a3fd5675890b908ebc23db1f8f7f03dc3459241abdcf35d46c68e1be

NET_SNMP_VERSION = "5.9.1"
NET_SNMP_SHA256 = "ddbe4d0111a0f1fb4c29751b2794d618565557facdda5315786f0a22472499d3"

PROTOBUF_VERSION = "3.20.2"
PROTOBUF_SHA256 = "c79640b37658dc5e760fc71d33ce22ece90d72e304ec8a5566c9ff859892e86f"
PROTOBUF_LIB_VERSION = "31.0.2"

0 comments on commit 15fd22f

Please sign in to comment.