Add C++ API documentation

PiperOrigin-RevId: 691926109
Change-Id: I578e679ae4d2d34b7fca6d142beb199a7681279d

WORKSPACE

@@ -23,3 +23,25 @@ rules_proto_toolchains()
 load("@build_bazel_apple_support//crosstool:setup.bzl", "apple_cc_configure")
 
 apple_cc_configure()
+
+# Define LLVM toolchain used for extracting C++ API documentation information
+load("@toolchains_llvm//toolchain:rules.bzl", "llvm_toolchain")
+
+llvm_toolchain(
+    name = "llvm_toolchain",
+    # https://github.com/bazel-contrib/toolchains_llvm/blob/master/toolchain/internal/llvm_distributions.bzl
+    llvm_versions = {
+        # Note: Older versions are built against older glibc, which is needed
+        # for compatibility with manylinux containers.
+        "": "15.0.6",
+        "darwin-aarch64": "15.0.7",
+        "darwin-x86_64": "15.0.7",
+    },
+    extra_target_compatible_with = {
+        "": ["@//docs:docs_toolchain_value"],
+    },
+)
+
+load("@llvm_toolchain//:toolchains.bzl", "llvm_register_toolchains")
+
+llvm_register_toolchains()

docs/BUILD

@@ -1,11 +1,35 @@
+load("@bazel_skylib//rules:common_settings.bzl", "bool_flag")
 load("//bazel:pytest.bzl", "tensorstore_pytest_test")
 load("//bazel:pytype.bzl", "pytype_strict_binary", "pytype_strict_test")
+load("//bazel:tensorstore.bzl", "tensorstore_cc_library")
+load("//docs:defs.bzl", "cc_preprocessed_output")
 load("//docs:doctest.bzl", "doctest_test")
 
 package(default_visibility = ["//visibility:public"])
 
 licenses(["notice"])
 
+# To exclude the Python API documentation from the generation documentation,
+# specify:
+#
+# bazel build --//docs:exclude_python_api
+#
+# This significantly speeds up the documentation build since the Python
+# extension module (which has a large dependency tree) does not have to be
+# built.
+bool_flag(
+    name = "exclude_python_api",
+    build_setting_default = False,
+)
+
+config_setting(
+    name = "exclude_python_api_setting",
+    flag_values = {
+        ":exclude_python_api": "True",
+    },
+    visibility = ["//visibility:private"],
+)
+
 filegroup(
     name = "doc_sources",
     srcs = [
@@ -50,12 +74,17 @@ pytype_strict_test(
         # Keep going after the first warning.
         "--keep-going",
     ],
-    data = [":doc_sources"] + glob(
+    data = [
+        "cpp_api.json",
+        ":doc_sources",
+        "@pypa_clang_format//:clang-format_binary",
+    ] + glob(
         ["cached_external_resources/**"],
         allow_empty = True,
     ),
     env = {
         "TENSORSTORE_SPECIAL_CPU_USER_LIMITS": "forge-00=4",
+        "SPHINX_CLANG_FORMAT": "$(location @pypa_clang_format//:clang-format_binary)",
     },
     python_version = "PY3",
     tags = [
@@ -66,11 +95,14 @@ pytype_strict_test(
     ],
     deps = [
         "//docs/tensorstore_sphinx_ext:doctest",
-        "//python/tensorstore",
+        "@pypa_libclang//:libclang",  # buildcleaner: keep
         "@pypa_pyyaml//:pyyaml",  # buildcleaner: keep
         "@pypa_sphinx//:sphinx",
         "@pypa_sphinx_immaterial//:sphinx_immaterial",
-    ],
+    ] + select({
+        ":exclude_python_api_setting": [],
+        "//conditions:default": ["//python/tensorstore:core"],
+    }),
 )
 
 pytype_strict_binary(
@@ -113,3 +145,131 @@ doctest_test(
     name = "doctest_test",
     srcs = glob(["python/**/*.rst"]),
 )
+
+tensorstore_cc_library(
+    name = "cpp_api_include",
+    testonly = True,
+    srcs = ["cpp_api_include.cc"],
+    copts = [
+        # Generated preprocessed output rather than object file.
+        "-E",
+        # Retain comments.
+        "-C",
+        # GCC/clang flag to output macro definitions.
+        "-dD",
+    ],
+    features = [
+        "-use_header_modules",
+        "-layering_check",
+    ],
+    linkstatic = True,
+    local_defines = [
+        "TENSORSTORE_CPP_DOC_GENERATION",
+    ],
+    tags = ["manual"],
+    deps = [
+        "//tensorstore",
+        "//tensorstore:array",
+        "//tensorstore:cast",
+        "//tensorstore:data_type",
+        "//tensorstore:downsample",
+        "//tensorstore:open",
+        "//tensorstore:rank",
+        "//tensorstore/index_space:alignment",
+        "//tensorstore/index_space:dim_expression",
+        "//tensorstore/index_space:index_transform",
+        "//tensorstore/index_space:transformed_array",
+        "//tensorstore/util:byte_strided_pointer",
+        "//tensorstore/util:element_pointer",
+        "//tensorstore/util:element_traits",
+        "//tensorstore/util:future",
+        "//tensorstore/util:result",
+        "//tensorstore/util:status_testutil",
+    ],
+)
+
+# Define a special constraint_setting that will be matched in order to select a
+# hermetic clang toolchain for preprocessing the C++ API headers used for the
+# API documentation.  This is used when the default toolchain is not clang.
+#
+# The C++ API documentation is extracted using libclang. libstdc++ (typically
+# used on Linux) is compatible with clang but only when preprocessed with clang.
+# If it is first preprocessed by GCC, then the resultant output contains
+# GCC-specific builtins, etc. and is not compatible with libclang.
+constraint_setting(
+    name = "docs_toolchain_setting",
+    visibility = ["//visibility:public"],
+)
+
+constraint_value(
+    name = "docs_toolchain_value",
+    constraint_setting = ":docs_toolchain_setting",
+    visibility = ["//visibility:public"],
+)
+
+platform(
+    name = "docs_toolchain_platform",
+    constraint_values = [":docs_toolchain_value"],
+    parents = ["@platforms//host"],
+    visibility = ["//visibility:public"],
+)
+
+cc_preprocessed_output(
+    name = "cpp_api_preprocessed.cc",
+    testonly = True,
+    cpp_compiler_constraint = select({
+        "//:compiler_clang": "@platforms//host",
+        "//conditions:default": ":docs_toolchain_platform",
+    }),
+    flags_output = "compiler_flags.json",
+    tags = ["manual"],
+    target = ":cpp_api_include",
+)
+
+pytype_strict_binary(
+    name = "generate_cpp_api",
+    srcs = ["generate_cpp_api.py"],
+    deps = [
+        "@pypa_libclang//:libclang",  # buildcleaner: keep
+        "@pypa_sphinx_immaterial//:sphinx_immaterial",
+    ],
+)
+
+pytype_strict_binary(
+    name = "cpp_api_shell",
+    testonly = True,
+    srcs = ["generate_cpp_api.py"],
+    args = [
+        "--source=$(location :cpp_api_preprocessed.cc)",
+        "--flags-file=$(location :compiler_flags.json)",
+        "--interactive",
+    ],
+    data = [
+        ":compiler_flags.json",
+        ":cpp_api_preprocessed.cc",
+    ],
+    main = "generate_cpp_api.py",
+    tags = ["manual"],
+    deps = [
+        "@pypa_libclang//:libclang",  # buildcleaner: keep
+        "@pypa_sphinx_immaterial//:sphinx_immaterial",
+    ],
+)
+
+genrule(
+    name = "genrule_cpp_api.json",
+    testonly = True,
+    srcs = [
+        ":cpp_api_preprocessed.cc",
+        ":compiler_flags.json",
+    ],
+    outs = ["cpp_api.json"],
+    cmd = ("$(location :generate_cpp_api) " +
+           "--source=$(location :cpp_api_preprocessed.cc) " +
+           "--flags-file=$(location :compiler_flags.json) " +
+           "--output=$@"),
+    tags = ["manual"],
+    tools = [
+        ":generate_cpp_api",
+    ],
+)

docs/build_docs.py

@@ -21,6 +21,7 @@
 import pathlib
 import platform
 import re
+import shutil
 import sys
 import tempfile
 from typing import List
@@ -182,7 +183,7 @@ def _write_third_party_libraries_summary(runfiles_dir: str, output_path: str):
 
 
 @contextlib.contextmanager
-def _prepare_source_tree(runfiles_dir: str):
+def _prepare_source_tree(runfiles_dir: str, excluded: List[str]):
   with tempfile.TemporaryDirectory() as temp_src_dir:
     _write_third_party_libraries_summary(
         runfiles_dir=runfiles_dir,
@@ -238,6 +239,18 @@ def create_symlinks(source_dir, target_dir):
         zipfile.ZipFile(cache_zip).extractall(zip_path)
         os.environ[cache_env_key] = zip_path
 
+    for excluded_glob in excluded:
+      if excluded_glob.startswith('/'):
+        excluded_glob = excluded_glob[1:]
+      matching_paths = glob.glob(
+          os.path.join(temp_src_dir, excluded_glob), recursive=True
+      )
+      matching_paths.reverse()
+      for matching_path in matching_paths:
+        if os.path.islink(matching_path):
+          os.remove(matching_path)
+        else:
+          shutil.rmtree(matching_path)
     yield temp_src_dir
 
 
@@ -294,7 +307,9 @@ def run(args: argparse.Namespace, unknown: List[str]):
       os.getenv('BUILD_WORKING_DIRECTORY', os.getcwd()), args.output
   )
   os.makedirs(output_dir, exist_ok=True)
-  with _prepare_source_tree(runfiles_dir) as temp_src_dir:
+  with _prepare_source_tree(
+      runfiles_dir, excluded=args.exclude
+  ) as temp_src_dir:
     # Use a separate temporary directory for the doctrees, since we don't want
     # them mixed into the output directory.
     with tempfile.TemporaryDirectory() as doctree_dir: