FASTER key/value store improvements (#356)

* Checkpointing and key iteration improvements

Add option for creating incremental checkpoints at a separate interval to full checkpoints.

Use FASTER's "push" key iteration (https://microsoft.github.io/FASTER/docs/fasterkv-basics/#key-iteration).

Use source-generated logging methods.

src/DataCore.Adapter.Abstractions/PublicAPI.Unshipped.txt

@@ -75,8 +75,12 @@ DataCore.Adapter.Services.RawKeyValueStore<TOptions>
 DataCore.Adapter.Services.RawKeyValueStore<TOptions>.RawKeyValueStore(TOptions? options, Microsoft.Extensions.Logging.ILogger? logger = null) -> void
 DataCore.Adapter.Services.ScopedKeyValueStore.ReadAsync<T>(DataCore.Adapter.Services.KVKey key) -> System.Threading.Tasks.ValueTask<T?>
 DataCore.Adapter.Services.ScopedKeyValueStore.WriteAsync<T>(DataCore.Adapter.Services.KVKey key, T value) -> System.Threading.Tasks.ValueTask
+override DataCore.Adapter.Services.KVKey.ToString() -> string!
 override sealed DataCore.Adapter.Services.KeyValueStore<TOptions>.GetCompressionLevel() -> System.IO.Compression.CompressionLevel
 override sealed DataCore.Adapter.Services.KeyValueStore<TOptions>.GetSerializer() -> DataCore.Adapter.Services.IKeyValueStoreSerializer!
 static DataCore.Adapter.AdapterExtensions.CreateExtendedAdapterDescriptorBuilder(this DataCore.Adapter.IAdapter! adapter) -> DataCore.Adapter.Common.AdapterDescriptorBuilder!
 static DataCore.Adapter.Services.JsonKeyValueStoreSerializer.Default.get -> DataCore.Adapter.Services.IKeyValueStoreSerializer!
+static DataCore.Adapter.Services.KeyValueStoreExtensions.GetKeysAsStringsAsync(this DataCore.Adapter.Services.IKeyValueStore! store) -> System.Collections.Generic.IAsyncEnumerable<string!>!
+static DataCore.Adapter.Services.KeyValueStoreExtensions.GetKeysAsStringsAsync(this DataCore.Adapter.Services.IKeyValueStore! store, DataCore.Adapter.Services.KVKey? prefix) -> System.Collections.Generic.IAsyncEnumerable<string!>!
+static DataCore.Adapter.Services.KVKey.implicit operator string?(DataCore.Adapter.Services.KVKey value) -> string?
 ~static DataCore.Adapter.AbstractionsResources.Error_KeyValueStore_RawWritesDisabled.get -> string

src/DataCore.Adapter.Abstractions/Services/KVKey.cs

@@ -77,6 +77,21 @@ public bool Equals(KVKey other) {
             return true;
         }
 
+
+        /// <inheritdoc/>
+        public override string ToString() {
+            if (Length == 0) {
+                return string.Empty;
+            }
+
+            try {
+                return Encoding.UTF8.GetString(Value);
+            }
+            catch {
+                return BitConverter.ToString(Value);
+            }
+        }
+
 #pragma warning disable CS1591 // Missing XML comment for publicly visible type or member
 
         public static bool operator ==(KVKey left, KVKey right) => left.Equals(right);
@@ -98,6 +113,7 @@ public bool Equals(KVKey other) {
         public static implicit operator KVKey(uint value) => new KVKey(BitConverter.GetBytes(value));
         public static implicit operator KVKey(ulong value) => new KVKey(BitConverter.GetBytes(value));
         public static implicit operator byte[](KVKey value) => value.Value;
+        public static implicit operator string?(KVKey value) => value.Length == 0 ? null : value.ToString();
 
 #pragma warning restore CS1591 // Missing XML comment for publicly visible type or member
 

src/DataCore.Adapter.Abstractions/Services/KeyValueStoreExtensions.cs

@@ -80,20 +80,16 @@ public static IKeyValueStore CreateScopedStore(this IKeyValueStore store, KVKey
         ///   key will be converted to a string using <see cref="BitConverter.ToString(byte[])"/> 
         ///   instead.
         /// </remarks>
-        public static async IAsyncEnumerable<string> GetKeysAsStrings(this IKeyValueStore store, KVKey? prefix) {
+        public static async IAsyncEnumerable<string> GetKeysAsStringsAsync(this IKeyValueStore store, KVKey? prefix) {
             if (store == null) {
                 throw new ArgumentNullException(nameof(store));
             }
 
             await foreach (var key in store.GetKeysAsync(prefix).ConfigureAwait(false)) {
-                string result;
-                try {
-                    result = Encoding.UTF8.GetString(key);
+                if (key.Length == 0) {
+                    continue;
                 }
-                catch {
-                    result = BitConverter.ToString(key);
-                }
-                yield return result;
+                yield return key.ToString();
             }
         }
 
@@ -104,6 +100,9 @@ public static async IAsyncEnumerable<string> GetKeysAsStrings(this IKeyValueStor
         /// <param name="store">
         ///   The <see cref="IKeyValueStore"/>.
         /// </param>
+        /// <param name="prefix">
+        ///   Only keys with this prefix will be returned.
+        /// </param>
         /// <returns>
         ///   The keys, converted to strings.
         /// </returns>
@@ -116,9 +115,51 @@ public static async IAsyncEnumerable<string> GetKeysAsStrings(this IKeyValueStor
         ///   key will be converted to a string using <see cref="BitConverter.ToString(byte[])"/> 
         ///   instead.
         /// </remarks>
-        public static IAsyncEnumerable<string> GetKeysAsStrings(this IKeyValueStore store) {
-            return store.GetKeysAsStrings(default);
-        }
+        [Obsolete("Use GetKeysAsStringsAsync instead.", false)]
+        public static IAsyncEnumerable<string> GetKeysAsStrings(this IKeyValueStore store, KVKey? prefix) => store.GetKeysAsStringsAsync(prefix);
+
+
+        /// <summary>
+        /// Gets the keys that are defined in the store, converted to <see cref="string"/>.
+        /// </summary>
+        /// <param name="store">
+        ///   The <see cref="IKeyValueStore"/>.
+        /// </param>
+        /// <returns>
+        ///   The keys, converted to strings.
+        /// </returns>
+        /// <exception cref="ArgumentNullException">
+        ///   <paramref name="store"/> is <see langword="null"/>.
+        /// </exception>
+        /// <remarks>
+        ///   Each key is converted to a string by calling <see cref="Encoding.GetString(byte[])"/> 
+        ///   on <see cref="Encoding.UTF8"/>. If an exception is thrown during this conversion, the 
+        ///   key will be converted to a string using <see cref="BitConverter.ToString(byte[])"/> 
+        ///   instead.
+        /// </remarks>
+        public static IAsyncEnumerable<string> GetKeysAsStringsAsync(this IKeyValueStore store) => store.GetKeysAsStringsAsync(default);
+
+
+        /// <summary>
+        /// Gets the keys that are defined in the store, converted to <see cref="string"/>.
+        /// </summary>
+        /// <param name="store">
+        ///   The <see cref="IKeyValueStore"/>.
+        /// </param>
+        /// <returns>
+        ///   The keys, converted to strings.
+        /// </returns>
+        /// <exception cref="ArgumentNullException">
+        ///   <paramref name="store"/> is <see langword="null"/>.
+        /// </exception>
+        /// <remarks>
+        ///   Each key is converted to a string by calling <see cref="Encoding.GetString(byte[])"/> 
+        ///   on <see cref="Encoding.UTF8"/>. If an exception is thrown during this conversion, the 
+        ///   key will be converted to a string using <see cref="BitConverter.ToString(byte[])"/> 
+        ///   instead.
+        /// </remarks>
+        [Obsolete("Use GetKeysAsStringsAsync instead.", false)]
+        public static IAsyncEnumerable<string> GetKeysAsStrings(this IKeyValueStore store) => store.GetKeysAsStringsAsync(default);
 
 
         /// <summary>

src/DataCore.Adapter.KeyValueStore.FASTER/DataCore.Adapter.KeyValueStore.FASTER.csproj

@@ -11,6 +11,7 @@
 
   <ItemGroup>
     <PackageReference Include="Microsoft.FASTER.Core" />
+    <PackageReference Include="System.Threading.Channels" />
   </ItemGroup>
 
   <ItemGroup>