Move openssl, python and python3-modules to bazel completly

Change-Id: I795c5b02c3f89bd9d7f47a9ae7fa1ef647216bfd

bazel/rules/package_wheel.bzl

@@ -10,7 +10,7 @@ def package_wheel(
         # TODO: As soon as we have all omd/packages in bazel, we won't need the intermediate install target anymore.
         # Then we can also completely remove the "python3-modules" prefix - currently it will be deployed to $(DESTDIR) via
         # $(PYTHON3_MODULES_INSTALL) in python3-modules.make
-        intermediate_prefix = "python3-modules/"):
+        intermediate_prefix = ""):
     """Packages a python wheel into our omd site-packages.
 
     Args:

omd/BUILD

@@ -2,17 +2,24 @@ load("@bazel_skylib//rules:write_file.bzl", "write_file")
 load("@rules_pkg//pkg:tar.bzl", "pkg_tar")
 load("//:bazel_variables.bzl", "edition_deps", "edition_skel_permissions")
 
-pkg_tar(
-    name = "intermediate_install",
+[pkg_tar(
+    name = "deps_install_" + edition,
     srcs = [
-        "//omd/packages/Python:python_files_with_prefix_pkg",
-        "//omd/packages/openssl:openssl_files_with_prefix_pkg",
+        "//omd/packages/freetds:freetds_pkg",
+        "//omd/packages/protobuf",
+        "//packages/site/check-http:check_http_pkg",
     ],
     extension = "tar.gz",
-    visibility = ["//visibility:public"],
     deps = [
+        #added here instead of above because of disappearing file
+        "//omd/packages/Python:python_tar",
+        "//omd/packages/net-snmp:net-snmp_tar",
+        "//omd/packages/openssl:openssl_tar",
+        "//omd/packages/perl-modules",
         "//omd/packages/python3-modules:python3-modules.tar",
+        "//omd/packages/rrdtool:rrdtool_perl_bindings",
         "//omd/packages/rrdtool:rrdtool_python_tar",
+        "//omd/packages/rrdtool:rrdtool_tar",
         "//packages/cmk-agent-based:pkg_tar",
         "//packages/cmk-agent-receiver:pkg_tar",
         "//packages/cmk-ccc:pkg_tar",
@@ -27,24 +34,6 @@ pkg_tar(
         "//packages/cmk-server-side-calls:pkg_tar",
         "//packages/cmk-trace:pkg_tar",
         "//packages/cmk-werks:pkg_tar",
-    ],
-)
-
-[pkg_tar(
-    name = "deps_install_" + edition,
-    srcs = [
-        "//omd/packages/freetds:freetds_pkg",
-        "//omd/packages/protobuf",
-        "//packages/site/check-http:check_http_pkg",
-    ],
-    extension = "tar.gz",
-    deps = [
-        #added here instead of above because of disappearing file
-        "//omd/packages/perl-modules",
-        "//omd/packages/rrdtool:rrdtool_tar",
-        "//omd/packages/net-snmp:net-snmp_tar",
-        "//omd/packages/rrdtool:rrdtool_perl_bindings",
-        "//omd/packages/rrdtool:rrdtool_python_tar",
     ] + edition_deps[edition],
 ) for edition in [
     "cre",

omd/Makefile

@@ -25,7 +25,6 @@ include $(BUILD_BASE_DIR)/build-helper-files.make
 # PACKAGES variable there to customzie the list of packages.
 
 PACKAGES := \
-    openssl \
     erlang \
     redis \
     libgsf \

omd/packages/Python/BUILD

@@ -1,4 +1,3 @@
-load("@omd_packages//:package_versions.bzl", "PYTHON_VERSION")
 load("@omd_packages//omd/packages/Python:version.bzl", "PYTHON_MAJOR_DOT_MINOR", "PYTHON_VERSION_MAJOR")
 load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files", "pkg_mklink")
 load("@rules_pkg//pkg:tar.bzl", "pkg_tar")
@@ -216,27 +215,3 @@ pkg_tar(
     package_file_name = "python.tar",
     visibility = ["//visibility:public"],
 )
-
-# only necessary for intermediate install
-# can be removed once that is obsolete
-pkg_filegroup(
-    name = "python_files_with_prefix_pkg",
-    srcs = [
-        ":python_bin_pkg",
-        ":python_bin_symlinks_pkg",
-        ":python_lib_dynload_pkg",
-        ":python_lib_symlinks_pkg",
-        ":python_pkgconfig_symlinks_pkg",
-        ":python_rest_pkg",
-        ":python_share_symlinks_pkg",
-    ],
-    prefix = "Python-%s" % PYTHON_VERSION,
-    visibility = ["//visibility:public"],
-)
-
-pkg_tar(
-    name = "python_tar_with_prefix",
-    srcs = [":python_files_with_prefix_pkg"],
-    package_file_name = "python.tar",
-    visibility = ["//visibility:public"],
-)

omd/packages/Python/Python.make

@@ -4,31 +4,11 @@ include $(REPO_PATH)/defines.make
 PYTHON := Python
 PYTHON_DIR := $(PYTHON)-$(PYTHON_VERSION)
 
-PYTHON_INSTALL := $(BUILD_HELPER_DIR)/$(PYTHON_DIR)-install
-PYTHON_SYSCONFIGDATA := _sysconfigdata__linux_x86_64-linux-gnu.py
-
-PYTHON_INSTALL_DIR := $(INTERMEDIATE_INSTALL_BASE)/$(PYTHON_DIR)
-
 # Used by other OMD packages
-PACKAGE_PYTHON_DESTDIR         := $(PYTHON_INSTALL_DIR)
+PACKAGE_PYTHON_DESTDIR         := $(DESTDIR)$(OMD_ROOT)
 PACKAGE_PYTHON_PYTHONPATH      := $(PACKAGE_PYTHON_DESTDIR)/lib/python$(PYTHON_MAJOR_DOT_MINOR)
 PACKAGE_PYTHON_LDFLAGS         := -L$(PACKAGE_PYTHON_DESTDIR)/lib -L$(PACKAGE_PYTHON_PYTHONPATH)/config
 PACKAGE_PYTHON_LD_LIBRARY_PATH := $(PACKAGE_PYTHON_DESTDIR)/lib
 PACKAGE_PYTHON_INCLUDE_PATH    := $(PACKAGE_PYTHON_DESTDIR)/include/python$(PYTHON_MAJOR_DOT_MINOR)
 PACKAGE_PYTHON_BIN             := $(PACKAGE_PYTHON_DESTDIR)/bin
 PACKAGE_PYTHON_EXECUTABLE      := $(PACKAGE_PYTHON_BIN)/python$(PYTHON_VERSION_MAJOR)
-PACKAGE_PYTHON_SYSCONFIGDATA := $(PACKAGE_PYTHON_PYTHONPATH)/$(PYTHON_SYSCONFIGDATA)
-
-$(PYTHON_INSTALL): $(INTERMEDIATE_INSTALL_BAZEL)
-	$(RSYNC) -rl --perms $(PYTHON_INSTALL_DIR)/ $(DESTDIR)$(OMD_ROOT)/
-	$(SED) -i "s|$(PACKAGE_PYTHON_DESTDIR)|$(OMD_ROOT)|g" \
-	    $(DESTDIR)/$(OMD_ROOT)/lib/python$(PYTHON_MAJOR_DOT_MINOR)/$(PYTHON_SYSCONFIGDATA)
-	# pre-compile pyc files enforcing `checked-hash` invalidation
-	# note: this is a workaround and should be handled in according Bazel project
-	$(PACKAGE_PYTHON_EXECUTABLE) -m compileall \
-	    -f \
-	    --invalidation-mode=checked-hash \
-	    -s "$(DESTDIR)/$(OMD_ROOT)/lib/python$(PYTHON_MAJOR_DOT_MINOR)/" \
-	    -x "bad_coding|badsyntax|test/test_lib2to3/data" \
-	    "$(DESTDIR)/$(OMD_ROOT)/lib/python$(PYTHON_MAJOR_DOT_MINOR)/"
-

omd/packages/net-snmp/BUILD.net-snmp.bazel

@@ -45,7 +45,7 @@ configure_make(
         "PERL_MODULES_TAR": "$(execpath @omd_packages//omd/packages/perl-modules:perl-modules.tar)",
         "PYTHON_MODULES_TAR": "$(execpath @omd_packages//omd/packages/python3-modules:python3-modules.tar)",
         "PERL_DIR": "$$(dirname $(execpath @omd_packages//omd/packages/perl-modules:perl-modules.tar))/perl-modules",
-        "PYTHONPATH": "$$(dirname $(execpath @omd_packages//omd/packages/python3-modules:python3-modules.tar))/python3-modules/lib/python%s/site-packages/" % PYTHON_MAJOR_DOT_MINOR,
+        "PYTHONPATH": "$$(dirname $(execpath @omd_packages//omd/packages/python3-modules:python3-modules.tar))/lib/python%s/site-packages/" % PYTHON_MAJOR_DOT_MINOR,
     },
     lib_source = ":all_srcs",
     out_binaries = [

omd/packages/openssl/BUILD

@@ -146,7 +146,6 @@ pkg_filegroup(
         ":openssl_empty_dirs",
         ":openssl_executables_pkg",
         ":openssl_lib_symlinks_pkg",
-        ":openssl_lib_with_rpath_pkg",
         ":openssl_rest_pkg",
         ":openssl_skel_symlinks_pkg",
     ],
@@ -159,26 +158,3 @@ pkg_tar(
     package_file_name = "openssl.tar",
     visibility = ["//visibility:public"],
 )
-
-# The prefixed version will be obsolete as soon as we remove
-# intermediate install (protobuf still depends on it)
-pkg_filegroup(
-    name = "openssl_files_with_prefix_pkg",
-    srcs = [
-        ":openssl_bin_pkg",
-        ":openssl_empty_dirs",
-        ":openssl_executables_pkg",
-        ":openssl_lib_symlinks_pkg",
-        ":openssl_rest_pkg",
-        ":openssl_skel_symlinks_pkg",
-    ],
-    prefix = "openssl",
-    visibility = ["//visibility:public"],
-)
-
-pkg_tar(
-    name = "openssl_with_prefix_tar",
-    srcs = [":openssl_files_with_prefix_pkg"],
-    package_file_name = "openssl_with_prefix.tar",
-    visibility = ["//visibility:public"],
-)

omd/packages/packages.make

@@ -22,29 +22,35 @@ BAZEL_CMD ?= $(realpath ../scripts/run-bazel.sh)
 # intermediate_install used to be necessary to link external dependecies with each other.
 # This is now done inside of Bazel
 # This target can be removed once `dest` is created inside of Bazel
-INTERMEDIATE_INSTALL_BAZEL := '$(BUILD_HELPER_DIR)/intermediate_install_bazel'
 DEPS_INSTALL_BAZEL := '$(BUILD_HELPER_DIR)/deps_install_bazel'
 
 # Human make target
-.PHONY: intermediate_install_bazel
-intermediate_install_bazel: $(INTERMEDIATE_INSTALL_BAZEL)
+.PHONY: deps_install_bazel
 deps_install_bazel: $(DEPS_INSTALL_BAZEL)
 
-NET_SNMP_PYTHONPATH := $(DESTDIR)$(OMD_ROOT)/lib/python$(PYTHON_MAJOR_DOT_MINOR)/site-packages
-
-$(INTERMEDIATE_INSTALL_BAZEL):
+$(DEPS_INSTALL_BAZEL):
 	# NOTE: this might result in unexpected build behavior, when dependencies of //omd:intermediate_install
 	#       are built somewhere else without --define git-ssl-no-verify=true being specified, likely
 	#       resulting in different builds
 	$(BAZEL_CMD) build \
 	    $(if $(filter sles15%,$(DISTRO_CODE)),--define git-ssl-no-verify=true) \
-	    //omd:intermediate_install
-	mkdir -p $(INTERMEDIATE_INSTALL_BASE)
-	tar -C $(INTERMEDIATE_INSTALL_BASE) -xf $(BAZEL_BIN)/omd/intermediate_install.tar.gz
+	    //omd:deps_install_$(EDITION_SHORT)
+	$(MKDIR) $(DESTDIR)$(OMD_ROOT)
+	tar -C $(DESTDIR)$(OMD_ROOT) -xf $(BAZEL_BIN)/omd/deps_install_$(EDITION_SHORT).tar.gz
 
 	#TODO: The following code should be executed by Bazel instead of make
 	# Fix sysconfigdata
-	$(SED) -i "s|/replace-me|$(PACKAGE_PYTHON_DESTDIR)|g" $(PACKAGE_PYTHON_SYSCONFIGDATA)
+	$(SED) -i "s|/replace-me|$(OMD_ROOT)|g" \
+	    $(DESTDIR)/$(OMD_ROOT)/lib/python$(PYTHON_MAJOR_DOT_MINOR)/_sysconfigdata__linux_x86_64-linux-gnu.py
+
+	# pre-compile pyc files enforcing `checked-hash` invalidation
+	# note: this is a workaround and should be handled in according Bazel project
+	$(DESTDIR)$(OMD_ROOT)/bin/python3 -m compileall \
+	    -f \
+	    --invalidation-mode=checked-hash \
+	    -s "$(DESTDIR)/$(OMD_ROOT)/lib/python$(PYTHON_MAJOR_DOT_MINOR)/" \
+	    -x "bad_coding|badsyntax|test/test_lib2to3/data" \
+	    "$(DESTDIR)/$(OMD_ROOT)/lib/python$(PYTHON_MAJOR_DOT_MINOR)/"
 
 	# This will replace forced absolute paths determined at build time by
 	# Bazel/foreign_cc. Note that this step depends on $OMD_ROOT which is different
@@ -58,36 +64,7 @@ $(INTERMEDIATE_INSTALL_BAZEL):
 	    --inplace \
 	    "/home/.*?/openssl.build_tmpdir/openssl/" \
 	    "$(OMD_ROOT)/" \
-	    "$(OPENSSL_INSTALL_DIR)/lib/libcrypto.so.3"
-
-	mkdir -p $(BUILD_HELPER_DIR)/
-	touch $@
-
-$(DEPS_INSTALL_BAZEL):
-	# NOTE: this might result in unexpected build behavior, when dependencies of //omd:intermediate_install
-	#       are built somewhere else without --define git-ssl-no-verify=true being specified, likely
-	#       resulting in different builds
-	$(BAZEL_CMD) build \
-	    $(if $(filter sles15%,$(DISTRO_CODE)),--define git-ssl-no-verify=true) \
-	    //omd:deps_install_$(EDITION_SHORT)
-	$(MKDIR) $(DESTDIR)$(OMD_ROOT)
-	tar -C $(DESTDIR)$(OMD_ROOT) -xf $(BAZEL_BIN)/omd/deps_install_$(EDITION_SHORT).tar.gz
-
-	# compile pyc files explicitly selecting `checked-hash` invalidation mode
-	@if [ "$(EDITION_SHORT)" != "cre" ]; then \
-	$(DESTDIR)$(OMD_ROOT)/bin/python3 -m compileall\
-	    -f \
-	    --invalidation-mode=checked-hash \
-	    -s "$(DESTDIR)$(OMD_ROOT)/lib/python3/" \
-	    "$(DESTDIR)$(OMD_ROOT)/lib/python3/cmc_proto"; \
-	fi
-
-	$(DESTDIR)$(OMD_ROOT)/bin/python3 -m compileall \
-	    -f \
-	    --invalidation-mode=checked-hash \
-	    -s "$(NET_SNMP_PYTHONPATH)/" \
-	    -o 0 -o 1 -o 2 -j0 \
-	    "$(NET_SNMP_PYTHONPATH)/netsnmp/"
+	    "$(DESTDIR)$(OMD_ROOT)/lib/libcrypto.so.3"
 
 	mkdir -p $(BUILD_HELPER_DIR)/
 	touch $@
@@ -202,7 +179,6 @@ debug:
 
 # Include rules to make packages
 include \
-    packages/openssl/openssl.make \
     packages/erlang/erlang.make \
     packages/redis/redis.make \
     packages/apache-omd/apache-omd.make \
@@ -219,13 +195,13 @@ include \
     packages/lcab/lcab.make \
     packages/msitools/msitools.make \
     packages/nagios/nagios.make \
+    packages/Python/Python.make \
+    packages/python3-modules/python3-modules.make \
     packages/heirloom-mailx/heirloom-mailx.make \
     packages/navicli/navicli.make \
     packages/nrpe/nrpe.make \
     packages/patch/patch.make \
     packages/pnp4nagios/pnp4nagios.make \
-    packages/Python/Python.make \
-    packages/python3-modules/python3-modules.make \
     packages/omd/omd.make \
     packages/mod_wsgi/mod_wsgi.make \
     packages/mk-livestatus/mk-livestatus.make \

omd/packages/python3-modules/BUILD

@@ -127,7 +127,7 @@ genrule(
             $$(echo $$F | sed 's|[^/]*/|../|g' | sed 's|\\.\\./|\\$$ORIGIN/|' | sed 's|[^/]*$$||') $$F ;
         done
         cd -
-	tar cf $@ python3-modules
+	tar cf $@ -C python3-modules .
     """.format(pyMajMin = PYTHON_MAJOR_DOT_MINOR),
     visibility = ["//visibility:public"],
 )

omd/packages/python3-modules/python3-modules.make

@@ -2,30 +2,14 @@ include $(REPO_PATH)/defines.make
 
 PYTHON3_MODULES := python3-modules
 
-PYTHON3_MODULES_INSTALL_DIR := $(INTERMEDIATE_INSTALL_BASE)/$(PYTHON3_MODULES)
-PYTHON3_MODULES_BUILD_DIR := $(BAZEL_BIN)/omd/packages/$(PYTHON3_MODULES)/$(PYTHON3_MODULES)
-
-PYTHON3_MODULES_INTERMEDIATE_INSTALL := $(BUILD_HELPER_DIR)/$(PYTHON3_MODULES)-install-intermediate
-PYTHON3_MODULES_INSTALL := $(BUILD_HELPER_DIR)/$(PYTHON3_MODULES)-install
-
-PACKAGE_PYTHON3_MODULES_PYTHON_DEPS := $(OPENSSL_CACHE_PKG_PROCESS) $(PYTHON_CACHE_PKG_PROCESS) $(PYTHON3_MODULES_CACHE_PKG_PROCESS)
-
 # Used by other OMD packages
-PACKAGE_PYTHON3_MODULES_DESTDIR    := $(PYTHON3_MODULES_INSTALL_DIR)
-PACKAGE_PYTHON3_MODULES_PYTHONPATH := $(PACKAGE_PYTHON3_MODULES_DESTDIR)/lib/python$(PYTHON_MAJOR_DOT_MINOR)/site-packages
+PACKAGE_PYTHON3_MODULES_PYTHON_DEPS := deps_install_bazel
+
+PACKAGE_PYTHON3_MODULES_PYTHONPATH := $(DESTDIR)$(OMD_ROOT)/lib/python$(PYTHON_MAJOR_DOT_MINOR)/site-packages
 # May be used during omd package build time. Call sites have to use the target
 # dependency "$(PACKAGE_PYTHON3_MODULES_PYTHON_DEPS)" to have everything needed in place.
 PACKAGE_PYTHON3_MODULES_PYTHON         := \
 	PYTHONPATH="$$PYTHONPATH:$(PACKAGE_PYTHON3_MODULES_PYTHONPATH):$(PACKAGE_PYTHON_PYTHONPATH)" \
 	LDFLAGS="$$LDFLAGS $(PACKAGE_PYTHON_LDFLAGS)" \
-	LD_LIBRARY_PATH="$$LD_LIBRARY_PATH:$(PACKAGE_PYTHON_LD_LIBRARY_PATH):$(PACKAGE_OPENSSL_LD_LIBRARY_PATH)" \
+	LD_LIBRARY_PATH="$$LD_LIBRARY_PATH:$(PACKAGE_PYTHON_LD_LIBRARY_PATH)" \
 	$(PACKAGE_PYTHON_EXECUTABLE)
-
-$(PYTHON3_MODULES_INSTALL): $(INTERMEDIATE_INSTALL_BAZEL)
-	$(PACKAGE_PYTHON_EXECUTABLE) -m compileall \
-	    -f \
-	    --invalidation-mode=checked-hash \
-	    -s "$(PACKAGE_PYTHON3_MODULES_PYTHONPATH)/" \
-	    "$(PACKAGE_PYTHON3_MODULES_PYTHONPATH)/"
-	$(RSYNC) --times $(PYTHON3_MODULES_INSTALL_DIR)/ $(DESTDIR)$(OMD_ROOT)/
-

packages/cmk-werks/BUILD

@@ -26,7 +26,10 @@ py_library(
         "cmk/werks/validate.py",
     ],
     imports = ["."],
-    visibility = ["//cmk:__pkg__"],
+    visibility = [
+        "//cmk:__pkg__",
+        "//omd:__pkg__",
+    ],
     deps = [
         requirement("gitpython"),
         requirement("lxml"),