Merge branch 'master' into aws-credentials-expiration

google · Nov 13, 2023 · 39ccfe0 · 39ccfe0
2 parents e5ceaf3 + 95c37e0
commit 39ccfe0
Show file tree

Hide file tree

Showing 88 changed files with 2,144 additions and 1,155 deletions.
diff --git a/docs/environment.rst b/docs/environment.rst
@@ -91,6 +91,18 @@ Proxy configuration
 Debugging
 ^^^^^^^^^
 
+.. envvar:: TENSORSTORE_VERBOSE_LOGGING
+
+   Enables debug logging for tensorstore internal subsystems.  Set to comma
+   separated list of values, where each value is one of ``name=int`` or just
+   ``name``. When ``all`` is, present, then verbose logging will be enabled for
+   each subsytem, otherwise logging is set only for those subsystems present in
+   the list.
+
+   Verbose flag values include: ``curl``, ``distributed``, ``gcs_grpc``, 
+   ``gcs_http``, ``gfile``, ``http_kvstore``, ``kvstore_cache``, ``ocdbt``,
+   ``s3``, ``test``, ``zip``, ``zip_details``.
+
 .. envvar:: TENSORSTORE_CURL_VERBOSE
 
    If set to any value, verbose debugging information will be printed to stderr

diff --git a/tensorstore/internal/BUILD b/tensorstore/internal/BUILD
@@ -1,5 +1,5 @@
-load("//bazel:tensorstore.bzl", "tensorstore_cc_library", "tensorstore_cc_test")
 load("@bazel_skylib//rules:common_settings.bzl", "bool_flag")
+load("//bazel:tensorstore.bzl", "tensorstore_cc_library", "tensorstore_cc_test")
 
 package(default_visibility = ["//tensorstore:internal_packages"])
 
@@ -1083,7 +1083,6 @@ tensorstore_cc_library(
         "//tensorstore/util:iterate",
         "//tensorstore/util:result",
         "//tensorstore/util:status",
-        "@com_google_absl//absl/container:fixed_array",
         "@com_google_absl//absl/status",
     ],
 )
@@ -1554,17 +1553,6 @@ tensorstore_cc_library(
     hdrs = ["non_compile_bypass.h"],
 )
 
-tensorstore_cc_library(
-    name = "os_error_code",
-    srcs = ["os_error_code.cc"],
-    hdrs = ["os_error_code.h"],
-    tags = ["msvc"],
-    deps = [
-        "//tensorstore/util:str_cat",
-        "@com_google_absl//absl/status",
-    ],
-)
-
 tensorstore_cc_library(
     name = "parse_json_matches",
     testonly = 1,
@@ -1712,42 +1700,6 @@ tensorstore_cc_test(
     ],
 )
 
-tensorstore_cc_library(
-    name = "subprocess",
-    testonly = True,
-    srcs = ["subprocess.cc"],
-    hdrs = ["subprocess.h"],
-    tags = ["msvc"],
-    deps = [
-        ":os_error_code",
-        "//tensorstore/util:result",
-        "//tensorstore/util:status",
-        "@com_google_absl//absl/container:flat_hash_map",
-        "@com_google_absl//absl/log:absl_check",
-        "@com_google_absl//absl/log:absl_log",
-        "@com_google_absl//absl/status",
-        "@com_google_absl//absl/strings",
-    ],
-)
-
-tensorstore_cc_test(
-    name = "subprocess_test",
-    srcs = ["subprocess_test.cc"],
-    tags = ["msvc"],
-    deps = [
-        ":env",
-        ":subprocess",
-        "//tensorstore/util:result",
-        "//tensorstore/util:status_testutil",
-        "@com_google_absl//absl/container:flat_hash_map",
-        "@com_google_absl//absl/log:absl_log",
-        "@com_google_absl//absl/status",
-        "@com_google_absl//absl/strings",
-        "@com_google_absl//absl/time",
-        "@com_google_googletest//:gtest",
-    ],
-)
-
 tensorstore_cc_library(
     name = "tagged_ptr",
     hdrs = ["tagged_ptr.h"],
@@ -1776,17 +1728,15 @@ tensorstore_cc_library(
     ],
     deps = [
         ":env",
-        ":os_error_code",
         ":path",
         ":source_location",
+        "//tensorstore/internal/os:filesystem",
         "//tensorstore/kvstore/file:file_util",
         "//tensorstore/util:result",
         "//tensorstore/util:status",
         "//tensorstore/util:str_cat",
-        "@com_google_absl//absl/log:absl_check",
         "@com_google_absl//absl/log:absl_log",
         "@com_google_absl//absl/random",
-        "@com_google_absl//absl/status",
         "@com_google_absl//absl/strings",
         "@com_google_googletest//:gtest",
     ],

diff --git a/tensorstore/internal/benchmark/kvstore_duration.cc b/tensorstore/internal/benchmark/kvstore_duration.cc
@@ -17,14 +17,15 @@
 ///
 /* Examples
 
-bazel run -c opt //tensorstore/internal/benchmark:kvstore_duration
--- \
+bazel run -c opt \
+  //tensorstore/internal/benchmark:kvstore_duration -- \
   --kvstore_spec='"file:///tmp/kvstore"' --duration=1m
 */
 
+#include <stddef.h>
+#include <stdint.h>
+
 #include <atomic>
-#include <cstddef>
-#include <cstdint>
 #include <cstring>
 #include <iostream>
 #include <string>
@@ -82,7 +83,6 @@ auto& read_throughput = internal_metrics::Value<double>::New(
     "/tensorstore/kvstore_benchmark/read_throughput",
     "the read throughput in this test");
 
-
 struct ReadState : public internal::AtomicReferenceCount<ReadState> {
   std::vector<std::string> keys;
   tensorstore::KvStore kvstore;

diff --git a/tensorstore/internal/benchmark/ts_benchmark.cc b/tensorstore/internal/benchmark/ts_benchmark.cc
@@ -16,7 +16,8 @@
 
 # Sequential reads, 1G tensorstore, in-memory, n5, 256k chunks, 2G total reads
 
-bazel run -c opt //tensorstore/internal/benchmark:ts_benchmark -- \
+bazel run -c opt \
+  //tensorstore/internal/benchmark:ts_benchmark -- \
   --alsologtostderr       \
   --strategy=sequential   \
   --total_read_bytes=-10  \
@@ -28,7 +29,8 @@ bazel run -c opt //tensorstore/internal/benchmark:ts_benchmark -- \
 
 # Random reads, 1G tensorstore, in-memory, n5, 256k chunks, 2G total reads
 
-bazel run -c opt //tensorstore/internal/benchmark:ts_benchmark -- \
+bazel run -c opt \
+  //tensorstore/internal/benchmark:ts_benchmark -- \
   --alsologtostderr       \
   --strategy=random       \
   --total_read_bytes=-10  \
@@ -40,7 +42,8 @@ bazel run -c opt //tensorstore/internal/benchmark:ts_benchmark -- \
 
 # As above, with context specified:
 
-bazel run -c opt //tensorstore/internal/benchmark:ts_benchmark -- \
+bazel run -c opt \
+  //tensorstore/internal/benchmark:ts_benchmark -- \
   --alsologtostderr       \
   --strategy=random       \
   --total_read_bytes=-10  \
@@ -64,7 +67,8 @@ bazel run -c opt //tensorstore/internal/benchmark:ts_benchmark -- \
 
 # Using a file driver and mis-aligned chunks
 
-bazel run -c opt //tensorstore/internal/benchmark:ts_benchmark -- \
+bazel run -c opt \
+  /third_party/tensorstore/internal/benchmark:ts_benchmark -- \
   --alsologtostderr              \
   --strategy=random              \
   --total_read_bytes=-10         \

diff --git a/tensorstore/internal/compression/BUILD b/tensorstore/internal/compression/BUILD
@@ -216,10 +216,12 @@ tensorstore_cc_library(
     srcs = ["zip_details.cc"],
     hdrs = ["zip_details.h"],
     deps = [
+        "//tensorstore/internal/log:verbose_flag",
         "//tensorstore/internal/riegeli:find",
         "//tensorstore/util:result",
         "//tensorstore/util:status",
         "//tensorstore/util:str_cat",
+        "@com_google_absl//absl/base:core_headers",
         "@com_google_absl//absl/log",
         "@com_google_absl//absl/log:absl_log",
         "@com_google_absl//absl/status",

diff --git a/tensorstore/internal/compression/zip_details.cc b/tensorstore/internal/compression/zip_details.cc
@@ -26,6 +26,7 @@
 #include <utility>
 #include <variant>
 
+#include "absl/base/attributes.h"
 #include "absl/log/absl_log.h"
 #include "absl/log/log.h"
 #include "absl/status/status.h"
@@ -40,15 +41,12 @@
 #include "riegeli/xz/xz_reader.h"
 #include "riegeli/zlib/zlib_reader.h"
 #include "riegeli/zstd/zstd_reader.h"
+#include "tensorstore/internal/log/verbose_flag.h"
 #include "tensorstore/internal/riegeli/find.h"
 #include "tensorstore/util/result.h"
 #include "tensorstore/util/status.h"
 #include "tensorstore/util/str_cat.h"
 
-#ifndef TENSORSTORE_ZIP_DETAILS_LOGGING
-#define TENSORSTORE_ZIP_DETAILS_LOGGING 0
-#endif
-
 namespace tensorstore {
 namespace internal_zip {
 namespace {
@@ -58,6 +56,8 @@ using ::riegeli::ReadLittleEndian32;
 using ::riegeli::ReadLittleEndian64;
 using ::riegeli::ReadLittleEndianSigned64;
 
+ABSL_CONST_INIT internal_log::VerboseFlag zip_logging("zip_details");
+
 // Windows epoch 1601-01-01T00:00:00Z is 11644473600 seconds before
 // Unix epoch 1970-01-01T00:00:00Z.
 const absl::Time kWindowsEpoch =
@@ -201,7 +201,7 @@ absl::Status ReadExtraField(riegeli::Reader &reader, ZipEntry &entry) {
         !ReadLittleEndian16(reader, tag_size)) {
       return absl::OkStatus();
     }
-    ABSL_LOG_IF(INFO, TENSORSTORE_ZIP_DETAILS_LOGGING)
+    ABSL_LOG_IF(INFO, zip_logging)
         << std::hex << "extra tag " << tag << " size " << tag_size;
     auto pos = reader.pos();
     switch (tag) {
@@ -250,8 +250,7 @@ absl::Status ReadEOCD64Locator(riegeli::Reader &reader,
   ReadLittleEndianSigned64(reader, locator.cd_offset);
   ReadLittleEndian32(reader, ignored32);
   if (locator.cd_offset < 0) {
-    ABSL_LOG_IF(INFO, TENSORSTORE_ZIP_DETAILS_LOGGING && !reader.ok())
-        << reader.status();
+    ABSL_LOG_IF(INFO, zip_logging && !reader.ok()) << reader.status();
     return absl::InvalidArgumentError(
         "Failed to read ZIP64 End of Central Directory Locator");
   }

diff --git a/tensorstore/internal/http/BUILD b/tensorstore/internal/http/BUILD
@@ -42,13 +42,16 @@ tensorstore_cc_library(
     deps = [
         ":curl_factory",
         ":curl_handle",
+        ":curl_wrappers",
         ":http",
         "//tensorstore/internal:cord_util",
         "//tensorstore/internal:env",
         "//tensorstore/internal:no_destructor",
         "//tensorstore/internal:thread",
+        "//tensorstore/internal/log:verbose_flag",
         "//tensorstore/internal/metrics",
         "//tensorstore/util:future",
+        "@com_google_absl//absl/base:core_headers",
         "@com_google_absl//absl/log:absl_log",
         "@com_google_absl//absl/strings:cord",
         "@com_google_absl//absl/synchronization",

diff --git a/tensorstore/internal/http/curl_transport.cc b/tensorstore/internal/http/curl_transport.cc
@@ -16,14 +16,22 @@
 
 #include <stddef.h>
 #include <stdint.h>
-#include <stdlib.h>
 
+#include <algorithm>
+#include <atomic>
+#include <cassert>
+#include <cstdio>
+#include <cstdlib>
+#include <limits>
 #include <memory>
 #include <optional>
 #include <string>
 #include <string_view>
 #include <utility>
+#include <vector>
 
+#include "absl/base/attributes.h"
+#include "absl/base/const_init.h"
 #include "absl/log/absl_log.h"
 #include "absl/strings/cord.h"
 #include "absl/synchronization/mutex.h"
@@ -33,8 +41,11 @@
 #include "tensorstore/internal/env.h"
 #include "tensorstore/internal/http/curl_factory.h"
 #include "tensorstore/internal/http/curl_handle.h"
+#include "tensorstore/internal/http/curl_wrappers.h"
 #include "tensorstore/internal/http/http_request.h"
 #include "tensorstore/internal/http/http_response.h"
+#include "tensorstore/internal/http/http_transport.h"
+#include "tensorstore/internal/log/verbose_flag.h"
 #include "tensorstore/internal/metrics/counter.h"
 #include "tensorstore/internal/metrics/gauge.h"
 #include "tensorstore/internal/metrics/histogram.h"
@@ -81,6 +92,8 @@ auto& http_first_byte_latency_us =
         "/tensorstore/http/first_byte_latency_us",
         "HTTP first byte received latency (us)");
 
+ABSL_CONST_INIT internal_log::VerboseFlag curl_logging("curl");
+
 // Concurrent CURL HTTP/2 streams.
 int32_t GetMaxHttp2ConcurrentStreams() {
   auto limit = internal::GetEnvValue<int32_t>(
@@ -96,8 +109,8 @@ int32_t GetMaxHttp2ConcurrentStreams() {
 
 // Cached configuration from environment variables.
 struct CurlConfig {
-  bool verbose =
-      internal::GetEnvValue<bool>("TENSORSTORE_CURL_VERBOSE").value_or(false);
+  bool verbose = internal::GetEnvValue<bool>("TENSORSTORE_CURL_VERBOSE")
+                     .value_or(curl_logging.Level(0));
   std::optional<std::string> ca_path = internal::GetEnv("TENSORSTORE_CA_PATH");
   std::optional<std::string> ca_bundle =
       internal::GetEnv("TENSORSTORE_CA_BUNDLE");
@@ -281,17 +294,17 @@ struct CurlRequestState {
     handle_.SetOption(CURLOPT_FORBID_REUSE, 1);
   }
 
-  static std::size_t CurlWriteCallback(void* contents, std::size_t size,
-                                       std::size_t nmemb, void* userdata) {
+  static size_t CurlWriteCallback(void* contents, size_t size, size_t nmemb,
+                                  void* userdata) {
     auto* self = static_cast<CurlRequestState*>(userdata);
     auto data =
         std::string_view(static_cast<char const*>(contents), size * nmemb);
     self->response_.payload.Append(data);
     return data.size();
   }
 
-  static std::size_t CurlReadCallback(void* contents, std::size_t size,
-                                      std::size_t nmemb, void* userdata) {
+  static size_t CurlReadCallback(void* contents, size_t size, size_t nmemb,
+                                 void* userdata) {
     auto* self = static_cast<CurlRequestState*>(userdata);
     size_t n = std::min(size * nmemb, self->payload_remaining_);
     internal::CopyCordToSpan(self->payload_it_, {static_cast<char*>(contents),
@@ -319,8 +332,8 @@ struct CurlRequestState {
     return CURL_SEEKFUNC_OK;
   }
 
-  static std::size_t CurlHeaderCallback(void* contents, std::size_t size,
-                                        std::size_t nmemb, void* userdata) {
+  static size_t CurlHeaderCallback(void* contents, size_t size, size_t nmemb,
+                                   void* userdata) {
     auto* self = static_cast<CurlRequestState*>(userdata);
     auto data =
         std::string_view(static_cast<char const*>(contents), size * nmemb);

diff --git a/tensorstore/internal/log/BUILD b/tensorstore/internal/log/BUILD
@@ -0,0 +1,31 @@
+load("//bazel:tensorstore.bzl", "tensorstore_cc_library", "tensorstore_cc_test")
+
+package(default_visibility = ["//tensorstore:internal_packages"])
+
+licenses(["notice"])
+
+tensorstore_cc_library(
+    name = "verbose_flag",
+    srcs = ["verbose_flag.cc"],
+    hdrs = ["verbose_flag.h"],
+    deps = [
+        "//tensorstore/internal:env",
+        "//tensorstore/internal:no_destructor",
+        "@com_google_absl//absl/base:core_headers",
+        "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/flags:flag",
+        "@com_google_absl//absl/log:absl_log",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/synchronization",
+    ],
+)
+
+tensorstore_cc_test(
+    name = "verbose_flag_test",
+    srcs = ["verbose_flag_test.cc"],
+    deps = [
+        ":verbose_flag",
+        "@com_google_absl//absl/base:core_headers",
+        "@com_google_googletest//:gtest_main",
+    ],
+)