From ebc135780dbf7720701c7288ad2f632204d1b69f Mon Sep 17 00:00:00 2001 From: Alex Zurhake Date: Mon, 2 Dec 2024 09:57:36 +0100 Subject: [PATCH] Move openssl, python and python3-modules to bazel completly Change-Id: I795c5b02c3f89bd9d7f47a9ae7fa1ef647216bfd --- bazel/rules/package_wheel.bzl | 2 +- omd/BUILD | 35 ++++------- omd/Makefile | 1 - omd/packages/Python/BUILD | 25 -------- omd/packages/Python/Python.make | 22 +------ omd/packages/net-snmp/BUILD.net-snmp.bazel | 2 +- omd/packages/openssl/BUILD | 24 ------- omd/packages/openssl/openssl.make | 20 ------ omd/packages/packages.make | 62 ++++++------------- omd/packages/python3-modules/BUILD | 2 +- .../python3-modules/python3-modules.make | 24 ++----- packages/cmk-werks/BUILD | 5 +- 12 files changed, 43 insertions(+), 181 deletions(-) delete mode 100644 omd/packages/openssl/openssl.make diff --git a/bazel/rules/package_wheel.bzl b/bazel/rules/package_wheel.bzl index 8dd85bdb6f2..8188840e9b0 100644 --- a/bazel/rules/package_wheel.bzl +++ b/bazel/rules/package_wheel.bzl @@ -10,7 +10,7 @@ def package_wheel( # TODO: As soon as we have all omd/packages in bazel, we won't need the intermediate install target anymore. # Then we can also completely remove the "python3-modules" prefix - currently it will be deployed to $(DESTDIR) via # $(PYTHON3_MODULES_INSTALL) in python3-modules.make - intermediate_prefix = "python3-modules/"): + intermediate_prefix = ""): """Packages a python wheel into our omd site-packages. Args: diff --git a/omd/BUILD b/omd/BUILD index 59f4422641f..c61a5fac9c9 100644 --- a/omd/BUILD +++ b/omd/BUILD @@ -2,17 +2,24 @@ load("@bazel_skylib//rules:write_file.bzl", "write_file") load("@rules_pkg//pkg:tar.bzl", "pkg_tar") load("//:bazel_variables.bzl", "edition_deps", "edition_skel_permissions") -pkg_tar( - name = "intermediate_install", +[pkg_tar( + name = "deps_install_" + edition, srcs = [ - "//omd/packages/Python:python_files_with_prefix_pkg", - "//omd/packages/openssl:openssl_files_with_prefix_pkg", + "//omd/packages/freetds:freetds_pkg", + "//omd/packages/protobuf", + "//packages/site/check-http:check_http_pkg", ], extension = "tar.gz", - visibility = ["//visibility:public"], deps = [ + #added here instead of above because of disappearing file + "//omd/packages/Python:python_tar", + "//omd/packages/net-snmp:net-snmp_tar", + "//omd/packages/openssl:openssl_tar", + "//omd/packages/perl-modules", "//omd/packages/python3-modules:python3-modules.tar", + "//omd/packages/rrdtool:rrdtool_perl_bindings", "//omd/packages/rrdtool:rrdtool_python_tar", + "//omd/packages/rrdtool:rrdtool_tar", "//packages/cmk-agent-based:pkg_tar", "//packages/cmk-agent-receiver:pkg_tar", "//packages/cmk-ccc:pkg_tar", @@ -27,24 +34,6 @@ pkg_tar( "//packages/cmk-server-side-calls:pkg_tar", "//packages/cmk-trace:pkg_tar", "//packages/cmk-werks:pkg_tar", - ], -) - -[pkg_tar( - name = "deps_install_" + edition, - srcs = [ - "//omd/packages/freetds:freetds_pkg", - "//omd/packages/protobuf", - "//packages/site/check-http:check_http_pkg", - ], - extension = "tar.gz", - deps = [ - #added here instead of above because of disappearing file - "//omd/packages/perl-modules", - "//omd/packages/rrdtool:rrdtool_tar", - "//omd/packages/net-snmp:net-snmp_tar", - "//omd/packages/rrdtool:rrdtool_perl_bindings", - "//omd/packages/rrdtool:rrdtool_python_tar", ] + edition_deps[edition], ) for edition in [ "cre", diff --git a/omd/Makefile b/omd/Makefile index 22a5c3ad49e..645f54dbfd4 100644 --- a/omd/Makefile +++ b/omd/Makefile @@ -25,7 +25,6 @@ include $(BUILD_BASE_DIR)/build-helper-files.make # PACKAGES variable there to customzie the list of packages. PACKAGES := \ - openssl \ erlang \ redis \ libgsf \ diff --git a/omd/packages/Python/BUILD b/omd/packages/Python/BUILD index 58200fe35a2..c2fa2e729b5 100644 --- a/omd/packages/Python/BUILD +++ b/omd/packages/Python/BUILD @@ -1,4 +1,3 @@ -load("@omd_packages//:package_versions.bzl", "PYTHON_VERSION") load("@omd_packages//omd/packages/Python:version.bzl", "PYTHON_MAJOR_DOT_MINOR", "PYTHON_VERSION_MAJOR") load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files", "pkg_mklink") load("@rules_pkg//pkg:tar.bzl", "pkg_tar") @@ -216,27 +215,3 @@ pkg_tar( package_file_name = "python.tar", visibility = ["//visibility:public"], ) - -# only necessary for intermediate install -# can be removed once that is obsolete -pkg_filegroup( - name = "python_files_with_prefix_pkg", - srcs = [ - ":python_bin_pkg", - ":python_bin_symlinks_pkg", - ":python_lib_dynload_pkg", - ":python_lib_symlinks_pkg", - ":python_pkgconfig_symlinks_pkg", - ":python_rest_pkg", - ":python_share_symlinks_pkg", - ], - prefix = "Python-%s" % PYTHON_VERSION, - visibility = ["//visibility:public"], -) - -pkg_tar( - name = "python_tar_with_prefix", - srcs = [":python_files_with_prefix_pkg"], - package_file_name = "python.tar", - visibility = ["//visibility:public"], -) diff --git a/omd/packages/Python/Python.make b/omd/packages/Python/Python.make index 7e7d810a1b6..0d5ebfff6f7 100644 --- a/omd/packages/Python/Python.make +++ b/omd/packages/Python/Python.make @@ -4,31 +4,11 @@ include $(REPO_PATH)/defines.make PYTHON := Python PYTHON_DIR := $(PYTHON)-$(PYTHON_VERSION) -PYTHON_INSTALL := $(BUILD_HELPER_DIR)/$(PYTHON_DIR)-install -PYTHON_SYSCONFIGDATA := _sysconfigdata__linux_x86_64-linux-gnu.py - -PYTHON_INSTALL_DIR := $(INTERMEDIATE_INSTALL_BASE)/$(PYTHON_DIR) - # Used by other OMD packages -PACKAGE_PYTHON_DESTDIR := $(PYTHON_INSTALL_DIR) +PACKAGE_PYTHON_DESTDIR := $(DESTDIR)$(OMD_ROOT) PACKAGE_PYTHON_PYTHONPATH := $(PACKAGE_PYTHON_DESTDIR)/lib/python$(PYTHON_MAJOR_DOT_MINOR) PACKAGE_PYTHON_LDFLAGS := -L$(PACKAGE_PYTHON_DESTDIR)/lib -L$(PACKAGE_PYTHON_PYTHONPATH)/config PACKAGE_PYTHON_LD_LIBRARY_PATH := $(PACKAGE_PYTHON_DESTDIR)/lib PACKAGE_PYTHON_INCLUDE_PATH := $(PACKAGE_PYTHON_DESTDIR)/include/python$(PYTHON_MAJOR_DOT_MINOR) PACKAGE_PYTHON_BIN := $(PACKAGE_PYTHON_DESTDIR)/bin PACKAGE_PYTHON_EXECUTABLE := $(PACKAGE_PYTHON_BIN)/python$(PYTHON_VERSION_MAJOR) -PACKAGE_PYTHON_SYSCONFIGDATA := $(PACKAGE_PYTHON_PYTHONPATH)/$(PYTHON_SYSCONFIGDATA) - -$(PYTHON_INSTALL): $(INTERMEDIATE_INSTALL_BAZEL) - $(RSYNC) -rl --perms $(PYTHON_INSTALL_DIR)/ $(DESTDIR)$(OMD_ROOT)/ - $(SED) -i "s|$(PACKAGE_PYTHON_DESTDIR)|$(OMD_ROOT)|g" \ - $(DESTDIR)/$(OMD_ROOT)/lib/python$(PYTHON_MAJOR_DOT_MINOR)/$(PYTHON_SYSCONFIGDATA) - # pre-compile pyc files enforcing `checked-hash` invalidation - # note: this is a workaround and should be handled in according Bazel project - $(PACKAGE_PYTHON_EXECUTABLE) -m compileall \ - -f \ - --invalidation-mode=checked-hash \ - -s "$(DESTDIR)/$(OMD_ROOT)/lib/python$(PYTHON_MAJOR_DOT_MINOR)/" \ - -x "bad_coding|badsyntax|test/test_lib2to3/data" \ - "$(DESTDIR)/$(OMD_ROOT)/lib/python$(PYTHON_MAJOR_DOT_MINOR)/" - diff --git a/omd/packages/net-snmp/BUILD.net-snmp.bazel b/omd/packages/net-snmp/BUILD.net-snmp.bazel index ad60c1220b9..adec9626978 100644 --- a/omd/packages/net-snmp/BUILD.net-snmp.bazel +++ b/omd/packages/net-snmp/BUILD.net-snmp.bazel @@ -45,7 +45,7 @@ configure_make( "PERL_MODULES_TAR": "$(execpath @omd_packages//omd/packages/perl-modules:perl-modules.tar)", "PYTHON_MODULES_TAR": "$(execpath @omd_packages//omd/packages/python3-modules:python3-modules.tar)", "PERL_DIR": "$$(dirname $(execpath @omd_packages//omd/packages/perl-modules:perl-modules.tar))/perl-modules", - "PYTHONPATH": "$$(dirname $(execpath @omd_packages//omd/packages/python3-modules:python3-modules.tar))/python3-modules/lib/python%s/site-packages/" % PYTHON_MAJOR_DOT_MINOR, + "PYTHONPATH": "$$(dirname $(execpath @omd_packages//omd/packages/python3-modules:python3-modules.tar))/lib/python%s/site-packages/" % PYTHON_MAJOR_DOT_MINOR, }, lib_source = ":all_srcs", out_binaries = [ diff --git a/omd/packages/openssl/BUILD b/omd/packages/openssl/BUILD index 2825559ec43..a4c4dee9777 100644 --- a/omd/packages/openssl/BUILD +++ b/omd/packages/openssl/BUILD @@ -146,7 +146,6 @@ pkg_filegroup( ":openssl_empty_dirs", ":openssl_executables_pkg", ":openssl_lib_symlinks_pkg", - ":openssl_lib_with_rpath_pkg", ":openssl_rest_pkg", ":openssl_skel_symlinks_pkg", ], @@ -159,26 +158,3 @@ pkg_tar( package_file_name = "openssl.tar", visibility = ["//visibility:public"], ) - -# The prefixed version will be obsolete as soon as we remove -# intermediate install (protobuf still depends on it) -pkg_filegroup( - name = "openssl_files_with_prefix_pkg", - srcs = [ - ":openssl_bin_pkg", - ":openssl_empty_dirs", - ":openssl_executables_pkg", - ":openssl_lib_symlinks_pkg", - ":openssl_rest_pkg", - ":openssl_skel_symlinks_pkg", - ], - prefix = "openssl", - visibility = ["//visibility:public"], -) - -pkg_tar( - name = "openssl_with_prefix_tar", - srcs = [":openssl_files_with_prefix_pkg"], - package_file_name = "openssl_with_prefix.tar", - visibility = ["//visibility:public"], -) diff --git a/omd/packages/openssl/openssl.make b/omd/packages/openssl/openssl.make deleted file mode 100644 index 4958e2d2219..00000000000 --- a/omd/packages/openssl/openssl.make +++ /dev/null @@ -1,20 +0,0 @@ -OPENSSL := openssl -# also set in package_versions.bzl -OPENSSL_DIR := $(OPENSSL) - -OPENSSL_INSTALL := $(BUILD_HELPER_DIR)/$(OPENSSL_DIR)-install - -# externally required variables -OPENSSL_INSTALL_DIR := $(INTERMEDIATE_INSTALL_BASE)/$(OPENSSL_DIR) - -$(OPENSSL)-build-library: $(BUILD_HELPER_DIR) $(OPENSSL_CACHE_PKG_PROCESS) - -# Used by Python/Python.make -PACKAGE_OPENSSL_DESTDIR := $(OPENSSL_INSTALL_DIR) -PACKAGE_OPENSSL_LDFLAGS := -L$(PACKAGE_OPENSSL_DESTDIR)/lib -PACKAGE_OPENSSL_LD_LIBRARY_PATH := $(PACKAGE_OPENSSL_DESTDIR)/lib -PACKAGE_OPENSSL_INCLUDE_PATH := $(PACKAGE_OPENSSL_DESTDIR)/include - -.PHONY: $(OPENSSL_INSTALL) -$(OPENSSL_INSTALL): $(INTERMEDIATE_INSTALL_BAZEL) - $(RSYNC) --recursive --links --perms "$(OPENSSL_INSTALL_DIR)/" "$(DESTDIR)$(OMD_ROOT)/" diff --git a/omd/packages/packages.make b/omd/packages/packages.make index d2cd6ad483d..4ac430a4c63 100644 --- a/omd/packages/packages.make +++ b/omd/packages/packages.make @@ -22,29 +22,35 @@ BAZEL_CMD ?= $(realpath ../scripts/run-bazel.sh) # intermediate_install used to be necessary to link external dependecies with each other. # This is now done inside of Bazel # This target can be removed once `dest` is created inside of Bazel -INTERMEDIATE_INSTALL_BAZEL := '$(BUILD_HELPER_DIR)/intermediate_install_bazel' DEPS_INSTALL_BAZEL := '$(BUILD_HELPER_DIR)/deps_install_bazel' # Human make target -.PHONY: intermediate_install_bazel -intermediate_install_bazel: $(INTERMEDIATE_INSTALL_BAZEL) +.PHONY: deps_install_bazel deps_install_bazel: $(DEPS_INSTALL_BAZEL) -NET_SNMP_PYTHONPATH := $(DESTDIR)$(OMD_ROOT)/lib/python$(PYTHON_MAJOR_DOT_MINOR)/site-packages - -$(INTERMEDIATE_INSTALL_BAZEL): +$(DEPS_INSTALL_BAZEL): # NOTE: this might result in unexpected build behavior, when dependencies of //omd:intermediate_install # are built somewhere else without --define git-ssl-no-verify=true being specified, likely # resulting in different builds $(BAZEL_CMD) build \ $(if $(filter sles15%,$(DISTRO_CODE)),--define git-ssl-no-verify=true) \ - //omd:intermediate_install - mkdir -p $(INTERMEDIATE_INSTALL_BASE) - tar -C $(INTERMEDIATE_INSTALL_BASE) -xf $(BAZEL_BIN)/omd/intermediate_install.tar.gz + //omd:deps_install_$(EDITION_SHORT) + $(MKDIR) $(DESTDIR)$(OMD_ROOT) + tar -C $(DESTDIR)$(OMD_ROOT) -xf $(BAZEL_BIN)/omd/deps_install_$(EDITION_SHORT).tar.gz #TODO: The following code should be executed by Bazel instead of make # Fix sysconfigdata - $(SED) -i "s|/replace-me|$(PACKAGE_PYTHON_DESTDIR)|g" $(PACKAGE_PYTHON_SYSCONFIGDATA) + $(SED) -i "s|/replace-me|$(OMD_ROOT)|g" \ + $(DESTDIR)/$(OMD_ROOT)/lib/python$(PYTHON_MAJOR_DOT_MINOR)/_sysconfigdata__linux_x86_64-linux-gnu.py + + # pre-compile pyc files enforcing `checked-hash` invalidation + # note: this is a workaround and should be handled in according Bazel project + $(DESTDIR)$(OMD_ROOT)/bin/python3 -m compileall \ + -f \ + --invalidation-mode=checked-hash \ + -s "$(DESTDIR)/$(OMD_ROOT)/lib/python$(PYTHON_MAJOR_DOT_MINOR)/" \ + -x "bad_coding|badsyntax|test/test_lib2to3/data" \ + "$(DESTDIR)/$(OMD_ROOT)/lib/python$(PYTHON_MAJOR_DOT_MINOR)/" # This will replace forced absolute paths determined at build time by # Bazel/foreign_cc. Note that this step depends on $OMD_ROOT which is different @@ -58,36 +64,7 @@ $(INTERMEDIATE_INSTALL_BAZEL): --inplace \ "/home/.*?/openssl.build_tmpdir/openssl/" \ "$(OMD_ROOT)/" \ - "$(OPENSSL_INSTALL_DIR)/lib/libcrypto.so.3" - - mkdir -p $(BUILD_HELPER_DIR)/ - touch $@ - -$(DEPS_INSTALL_BAZEL): - # NOTE: this might result in unexpected build behavior, when dependencies of //omd:intermediate_install - # are built somewhere else without --define git-ssl-no-verify=true being specified, likely - # resulting in different builds - $(BAZEL_CMD) build \ - $(if $(filter sles15%,$(DISTRO_CODE)),--define git-ssl-no-verify=true) \ - //omd:deps_install_$(EDITION_SHORT) - $(MKDIR) $(DESTDIR)$(OMD_ROOT) - tar -C $(DESTDIR)$(OMD_ROOT) -xf $(BAZEL_BIN)/omd/deps_install_$(EDITION_SHORT).tar.gz - - # compile pyc files explicitly selecting `checked-hash` invalidation mode - @if [ "$(EDITION_SHORT)" != "cre" ]; then \ - $(DESTDIR)$(OMD_ROOT)/bin/python3 -m compileall\ - -f \ - --invalidation-mode=checked-hash \ - -s "$(DESTDIR)$(OMD_ROOT)/lib/python3/" \ - "$(DESTDIR)$(OMD_ROOT)/lib/python3/cmc_proto"; \ - fi - - $(DESTDIR)$(OMD_ROOT)/bin/python3 -m compileall \ - -f \ - --invalidation-mode=checked-hash \ - -s "$(NET_SNMP_PYTHONPATH)/" \ - -o 0 -o 1 -o 2 -j0 \ - "$(NET_SNMP_PYTHONPATH)/netsnmp/" + "$(DESTDIR)$(OMD_ROOT)/lib/libcrypto.so.3" mkdir -p $(BUILD_HELPER_DIR)/ touch $@ @@ -202,7 +179,6 @@ debug: # Include rules to make packages include \ - packages/openssl/openssl.make \ packages/erlang/erlang.make \ packages/redis/redis.make \ packages/apache-omd/apache-omd.make \ @@ -219,13 +195,13 @@ include \ packages/lcab/lcab.make \ packages/msitools/msitools.make \ packages/nagios/nagios.make \ + packages/Python/Python.make \ + packages/python3-modules/python3-modules.make \ packages/heirloom-mailx/heirloom-mailx.make \ packages/navicli/navicli.make \ packages/nrpe/nrpe.make \ packages/patch/patch.make \ packages/pnp4nagios/pnp4nagios.make \ - packages/Python/Python.make \ - packages/python3-modules/python3-modules.make \ packages/omd/omd.make \ packages/mod_wsgi/mod_wsgi.make \ packages/mk-livestatus/mk-livestatus.make \ diff --git a/omd/packages/python3-modules/BUILD b/omd/packages/python3-modules/BUILD index e9e47d06bb8..fb76eeaa33c 100644 --- a/omd/packages/python3-modules/BUILD +++ b/omd/packages/python3-modules/BUILD @@ -127,7 +127,7 @@ genrule( $$(echo $$F | sed 's|[^/]*/|../|g' | sed 's|\\.\\./|\\$$ORIGIN/|' | sed 's|[^/]*$$||') $$F ; done cd - - tar cf $@ python3-modules + tar cf $@ -C python3-modules . """.format(pyMajMin = PYTHON_MAJOR_DOT_MINOR), visibility = ["//visibility:public"], ) diff --git a/omd/packages/python3-modules/python3-modules.make b/omd/packages/python3-modules/python3-modules.make index ab8981ec4d4..7ff72a97834 100644 --- a/omd/packages/python3-modules/python3-modules.make +++ b/omd/packages/python3-modules/python3-modules.make @@ -2,30 +2,14 @@ include $(REPO_PATH)/defines.make PYTHON3_MODULES := python3-modules -PYTHON3_MODULES_INSTALL_DIR := $(INTERMEDIATE_INSTALL_BASE)/$(PYTHON3_MODULES) -PYTHON3_MODULES_BUILD_DIR := $(BAZEL_BIN)/omd/packages/$(PYTHON3_MODULES)/$(PYTHON3_MODULES) - -PYTHON3_MODULES_INTERMEDIATE_INSTALL := $(BUILD_HELPER_DIR)/$(PYTHON3_MODULES)-install-intermediate -PYTHON3_MODULES_INSTALL := $(BUILD_HELPER_DIR)/$(PYTHON3_MODULES)-install - -PACKAGE_PYTHON3_MODULES_PYTHON_DEPS := $(OPENSSL_CACHE_PKG_PROCESS) $(PYTHON_CACHE_PKG_PROCESS) $(PYTHON3_MODULES_CACHE_PKG_PROCESS) - # Used by other OMD packages -PACKAGE_PYTHON3_MODULES_DESTDIR := $(PYTHON3_MODULES_INSTALL_DIR) -PACKAGE_PYTHON3_MODULES_PYTHONPATH := $(PACKAGE_PYTHON3_MODULES_DESTDIR)/lib/python$(PYTHON_MAJOR_DOT_MINOR)/site-packages +PACKAGE_PYTHON3_MODULES_PYTHON_DEPS := deps_install_bazel + +PACKAGE_PYTHON3_MODULES_PYTHONPATH := $(DESTDIR)$(OMD_ROOT)/lib/python$(PYTHON_MAJOR_DOT_MINOR)/site-packages # May be used during omd package build time. Call sites have to use the target # dependency "$(PACKAGE_PYTHON3_MODULES_PYTHON_DEPS)" to have everything needed in place. PACKAGE_PYTHON3_MODULES_PYTHON := \ PYTHONPATH="$$PYTHONPATH:$(PACKAGE_PYTHON3_MODULES_PYTHONPATH):$(PACKAGE_PYTHON_PYTHONPATH)" \ LDFLAGS="$$LDFLAGS $(PACKAGE_PYTHON_LDFLAGS)" \ - LD_LIBRARY_PATH="$$LD_LIBRARY_PATH:$(PACKAGE_PYTHON_LD_LIBRARY_PATH):$(PACKAGE_OPENSSL_LD_LIBRARY_PATH)" \ + LD_LIBRARY_PATH="$$LD_LIBRARY_PATH:$(PACKAGE_PYTHON_LD_LIBRARY_PATH)" \ $(PACKAGE_PYTHON_EXECUTABLE) - -$(PYTHON3_MODULES_INSTALL): $(INTERMEDIATE_INSTALL_BAZEL) - $(PACKAGE_PYTHON_EXECUTABLE) -m compileall \ - -f \ - --invalidation-mode=checked-hash \ - -s "$(PACKAGE_PYTHON3_MODULES_PYTHONPATH)/" \ - "$(PACKAGE_PYTHON3_MODULES_PYTHONPATH)/" - $(RSYNC) --times $(PYTHON3_MODULES_INSTALL_DIR)/ $(DESTDIR)$(OMD_ROOT)/ - diff --git a/packages/cmk-werks/BUILD b/packages/cmk-werks/BUILD index 6f091476c92..e16a0afa936 100644 --- a/packages/cmk-werks/BUILD +++ b/packages/cmk-werks/BUILD @@ -26,7 +26,10 @@ py_library( "cmk/werks/validate.py", ], imports = ["."], - visibility = ["//cmk:__pkg__"], + visibility = [ + "//cmk:__pkg__", + "//omd:__pkg__", + ], deps = [ requirement("gitpython"), requirement("lxml"),