Enable "s3" kvstore by default.

"s3" is not feature complete, but it's in a place where it's useful. PiperOrigin-RevId: 580701544 Change-Id: I44797058865418057f2bfec6bb40cd5db3e19e20
google · Nov 9, 2023 · 6cd6cb5 · 6cd6cb5
1 parent c952b34
commit 6cd6cb5
Show file tree

Hide file tree

Showing 4 changed files with 40 additions and 37 deletions.
diff --git a/tensorstore/kvstore/BUILD b/tensorstore/kvstore/BUILD
@@ -1,5 +1,5 @@
-load("//bazel:tensorstore.bzl", "tensorstore_cc_binary", "tensorstore_cc_library", "tensorstore_cc_test")
 load("@bazel_skylib//rules:common_settings.bzl", "bool_flag")
+load("//bazel:tensorstore.bzl", "tensorstore_cc_binary", "tensorstore_cc_library", "tensorstore_cc_test")
 
 package(default_visibility = ["//visibility:public"])
 
@@ -12,6 +12,7 @@ DRIVER_DOCS = [
     "memory",
     "neuroglancer_uint64_sharded",
     "ocdbt",
+    "s3",
     "zarr3_sharding_indexed",
     "zip",
 ]

diff --git a/tensorstore/kvstore/s3/BUILD b/tensorstore/kvstore/s3/BUILD
@@ -1,3 +1,4 @@
+# Placeholder: load py_binary
 load("@bazel_skylib//rules:common_settings.bzl", "bool_flag")
 load("//bazel:tensorstore.bzl", "tensorstore_cc_library", "tensorstore_cc_test")
 
@@ -302,8 +303,10 @@ tensorstore_cc_test(
         "--binary_mode=moto",
     ],
     data = [":moto_server"],
+    flaky = 1,  # Spawning the test process can be flaky.
     tags = [
-        "skip-cmake",  # moto is python, which will not work in bazel_to_cmake.
+        "requires-net:loopback",
+        "skip-cmake",
     ],
     deps = [
         ":aws_credential_provider",

diff --git a/tensorstore/kvstore/s3/index.rst b/tensorstore/kvstore/s3/index.rst
@@ -61,23 +61,28 @@ without credentials.  Otherwise amazon credentials are required:
 .. envvar:: AWS_SHARED_CREDENTIALS_FILE
 
    Specifies the location of the file that the AWS CLI uses to store access keys.
-   The default path is `~/.aws/credentials`.
+   The default path is :file:`~/.aws/credentials`.
    See <https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html>
 
 .. envvar:: AWS_PROFILE
 
   Specifies the name of the AWS CLI profile with the credentials and options to
   use. This can be the name of a profile stored in a credentials or config file,
-  or the value `default`` to use the default profile.
+  or the value ``default`` to use the default profile.
 
   If defined, this environment variable overrides the behavior of using the
-  profile named `[default]` in the credentials file.
+  profile named ``[default]`` in the credentials file.
   See <https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-envvars.html>
 
 .. envvar:: AWS_EC2_METADATA_SERVICE_ENDPOINT
 
   Overrides the default EC2 Instance Metadata Service (IMDS) endpoint of 
-  `http://169.254.169.254`. This must be a valid uri, and should respond to the
-  AWS IMDS api endpoints.
+  ``http://169.254.169.254``. This must be a valid uri, and should respond to
+  the AWS IMDS api endpoints.
   See <https://docs.aws.amazon.com/sdkref/latest/guide/feature-imds-credentials.html>
 
+.. envvar:: TENSORSTORE_S3_REQUEST_CONCURRENCY
+
+   Specifies the concurrency level used by the shared Context
+   :json:schema:`Context.s3_request_concurrency` resource. Defaults to 32.
+
diff --git a/tensorstore/kvstore/s3/schema.yml b/tensorstore/kvstore/s3/schema.yml
@@ -12,57 +12,52 @@ allOf:
     requester_pays:
       type: boolean
       title: Permit requester-pays requests.
-      description: |
+      description: |-
         This option must be enabled in order for any operations to succeed if the bucket has
         Requester Pays enabled and the supplied credentials are not for an owner of the bucket.
       default: false
     aws_region:
       type: string
       title: AWS region identifier to use in signatures.
-      description: |
-        If `.endpoint` is not specified, the region of the `.bucket` is determined automatically.
+      description: |-
+        If :json:schema:`.endpoint` is not specified, the region of the :json:schema:`.bucket` is
+        determined automatically.
     endpoint:
       type: string
       title: S3 server endpoint to use in place of the public Amazon S3 endpoints.
-      description: |
-        Must be an http or https URL. 
+      description: |-
+        Must be an http or https URL.
       examples:
       - "http://localhost:1234"
     host_header:
       type: string
       title: Override HTTP host header to send in requests.
-      description: |
-        May only be specified in conjunction with `.endpoint`, to send a different host than
-        specified in `.endpoint`.  This may be useful for testing with
+      description: |-
+        May only be specified in conjunction with :json:schema:`.endpoint`, to send a different host
+        than specified in :json:schema:`.endpoint`.  This may be useful for testing with
         `localstack <https://localstack.cloud/>`__."
       examples:
       - "mybucket.s3.af-south-1.localstack.localhost.com"
     aws_credentials:
       $ref: ContextResource
       description: |-
-        Specifies or references a previously defined
-        `Context.aws_credentials`.
+        Specifies or references a previously defined `Context.aws_credentials`.
     s3_request_concurrency:
       $ref: ContextResource
       description: |-
-        Specifies or references a previously defined
-        `Context.s3_request_concurrency`.
+        Specifies or references a previously defined `Context.s3_request_concurrency`.
     s3_request_retries:
       $ref: ContextResource
       description: |-
-        Specifies or references a previously defined
-        `Context.s3_request_retries`.
+        Specifies or references a previously defined `Context.s3_request_retries`.
     experimental_s3_rate_limiter:
       $ref: ContextResource
       description: |-
-        Specifies or references a previously defined
-        `Context.experimental_s3_rate_limiter`.
+        Specifies or references a previously defined `Context.experimental_s3_rate_limiter`.
     data_copy_concurrency:
       $ref: ContextResource
       description: |-
-        Specifies or references a previously defined
-        `Context.data_copy_concurrency`.  It is normally more convenient to
-        specify a default `~Context.data_copy_concurrency` in the `.context`.
+        Specifies or references a previously defined `Context.data_copy_concurrency`.
       default: data_copy_concurrency
   required:
   - bucket
@@ -80,13 +75,13 @@ definitions:
         - const: "shared"
         description: |-
           The maximum number of concurrent requests.  If the special value of
-          ``"shared"`` is specified, a shared global limit specified by
+          :json:"shared" is specified, a shared global limit specified by
           environment variable :envvar:`TENSORSTORE_S3_REQUEST_CONCURRENCY`,
           which defaults to 32.
         default: "shared"
   s3_request_retries:
     $id: Context.s3_request_retries
-    description: |
+    description: |-
       Specifies retry parameters for handling transient network errors.
       An exponential delay is added between consecutive retry attempts. The
       default values are appropriate for S3.
@@ -115,10 +110,10 @@ definitions:
     type: object
     properties:
       profile:
-      type: string
-      description: |
-        The profile name in the :file:`~/.aws/credentials` file, when used.
-        Overrides the :envvar:`AWS_PROFILE` environment variables.
+        type: string
+        description: |-
+          The profile name in the :file:`~/.aws/credentials` file, when used.
+          Overrides the :envvar:`AWS_PROFILE` environment variables.
   experimental_s3_rate_limiter:
     $id: Context.experimental_s3_rate_limiter
     description: |-
@@ -135,7 +130,7 @@ definitions:
           The maximum rate of write and/or delete calls issued per second.
       doubling_time:
         type: string
-        description:
+        description: |-
           The time interval over which the initial rates scale to 2x. The cases
           where this setting is useful depend on details to the storage buckets.
         default: "0"
@@ -144,12 +139,11 @@ definitions:
     allOf:
     - $ref: KvStoreUrl
     - type: string
-    title: |
-      :literal:`s3://` KvStore URL scheme
+    title: :literal:`s3://` KvStore URL scheme
     description: |
       AWS S3 key-value stores may be specified using the
-      :file:`s3://{bucket}/{path}` URL syntax, as supported by `aws s3
-      <https://docs.aws.amazon.com/cli/latest/reference/s3/>`__.
+      :file:`s3://{bucket}/{path}` URL syntax, as supported by
+      `aws s3 <https://docs.aws.amazon.com/cli/latest/reference/s3/>`__.
 
       .. admonition:: Examples
          :class: example