From de9bd806086f291f443dc79d22b11936d2448f03 Mon Sep 17 00:00:00 2001 From: Graham Watts Date: Thu, 26 Oct 2023 09:45:13 +0300 Subject: [PATCH] FASTER key/value store improvements (#356) * Checkpointing and key iteration improvements Add option for creating incremental checkpoints at a separate interval to full checkpoints. Use FASTER's "push" key iteration (https://microsoft.github.io/FASTER/docs/fasterkv-basics/#key-iteration). Use source-generated logging methods. --- .../PublicAPI.Unshipped.txt | 4 + .../Services/KVKey.cs | 16 ++ .../Services/KeyValueStoreExtensions.cs | 63 ++++- ...taCore.Adapter.KeyValueStore.FASTER.csproj | 1 + .../FasterKeyValueStore.cs | 230 ++++++++++++++---- .../FasterKeyValueStoreOptions.cs | 27 +- .../PublicAPI.Unshipped.txt | 3 + .../Resources.Designer.cs | 63 ----- .../Resources.resx | 21 -- .../KeyValueStoreTests.cs | 2 +- 10 files changed, 276 insertions(+), 154 deletions(-) diff --git a/src/DataCore.Adapter.Abstractions/PublicAPI.Unshipped.txt b/src/DataCore.Adapter.Abstractions/PublicAPI.Unshipped.txt index 2c85920f..c43e849b 100644 --- a/src/DataCore.Adapter.Abstractions/PublicAPI.Unshipped.txt +++ b/src/DataCore.Adapter.Abstractions/PublicAPI.Unshipped.txt @@ -75,8 +75,12 @@ DataCore.Adapter.Services.RawKeyValueStore DataCore.Adapter.Services.RawKeyValueStore.RawKeyValueStore(TOptions? options, Microsoft.Extensions.Logging.ILogger? logger = null) -> void DataCore.Adapter.Services.ScopedKeyValueStore.ReadAsync(DataCore.Adapter.Services.KVKey key) -> System.Threading.Tasks.ValueTask DataCore.Adapter.Services.ScopedKeyValueStore.WriteAsync(DataCore.Adapter.Services.KVKey key, T value) -> System.Threading.Tasks.ValueTask +override DataCore.Adapter.Services.KVKey.ToString() -> string! override sealed DataCore.Adapter.Services.KeyValueStore.GetCompressionLevel() -> System.IO.Compression.CompressionLevel override sealed DataCore.Adapter.Services.KeyValueStore.GetSerializer() -> DataCore.Adapter.Services.IKeyValueStoreSerializer! static DataCore.Adapter.AdapterExtensions.CreateExtendedAdapterDescriptorBuilder(this DataCore.Adapter.IAdapter! adapter) -> DataCore.Adapter.Common.AdapterDescriptorBuilder! static DataCore.Adapter.Services.JsonKeyValueStoreSerializer.Default.get -> DataCore.Adapter.Services.IKeyValueStoreSerializer! +static DataCore.Adapter.Services.KeyValueStoreExtensions.GetKeysAsStringsAsync(this DataCore.Adapter.Services.IKeyValueStore! store) -> System.Collections.Generic.IAsyncEnumerable! +static DataCore.Adapter.Services.KeyValueStoreExtensions.GetKeysAsStringsAsync(this DataCore.Adapter.Services.IKeyValueStore! store, DataCore.Adapter.Services.KVKey? prefix) -> System.Collections.Generic.IAsyncEnumerable! +static DataCore.Adapter.Services.KVKey.implicit operator string?(DataCore.Adapter.Services.KVKey value) -> string? ~static DataCore.Adapter.AbstractionsResources.Error_KeyValueStore_RawWritesDisabled.get -> string diff --git a/src/DataCore.Adapter.Abstractions/Services/KVKey.cs b/src/DataCore.Adapter.Abstractions/Services/KVKey.cs index c86041da..1e4aeda8 100644 --- a/src/DataCore.Adapter.Abstractions/Services/KVKey.cs +++ b/src/DataCore.Adapter.Abstractions/Services/KVKey.cs @@ -77,6 +77,21 @@ public bool Equals(KVKey other) { return true; } + + /// + public override string ToString() { + if (Length == 0) { + return string.Empty; + } + + try { + return Encoding.UTF8.GetString(Value); + } + catch { + return BitConverter.ToString(Value); + } + } + #pragma warning disable CS1591 // Missing XML comment for publicly visible type or member public static bool operator ==(KVKey left, KVKey right) => left.Equals(right); @@ -98,6 +113,7 @@ public bool Equals(KVKey other) { public static implicit operator KVKey(uint value) => new KVKey(BitConverter.GetBytes(value)); public static implicit operator KVKey(ulong value) => new KVKey(BitConverter.GetBytes(value)); public static implicit operator byte[](KVKey value) => value.Value; + public static implicit operator string?(KVKey value) => value.Length == 0 ? null : value.ToString(); #pragma warning restore CS1591 // Missing XML comment for publicly visible type or member diff --git a/src/DataCore.Adapter.Abstractions/Services/KeyValueStoreExtensions.cs b/src/DataCore.Adapter.Abstractions/Services/KeyValueStoreExtensions.cs index cde454ed..3ed973f8 100644 --- a/src/DataCore.Adapter.Abstractions/Services/KeyValueStoreExtensions.cs +++ b/src/DataCore.Adapter.Abstractions/Services/KeyValueStoreExtensions.cs @@ -80,20 +80,16 @@ public static IKeyValueStore CreateScopedStore(this IKeyValueStore store, KVKey /// key will be converted to a string using /// instead. /// - public static async IAsyncEnumerable GetKeysAsStrings(this IKeyValueStore store, KVKey? prefix) { + public static async IAsyncEnumerable GetKeysAsStringsAsync(this IKeyValueStore store, KVKey? prefix) { if (store == null) { throw new ArgumentNullException(nameof(store)); } await foreach (var key in store.GetKeysAsync(prefix).ConfigureAwait(false)) { - string result; - try { - result = Encoding.UTF8.GetString(key); + if (key.Length == 0) { + continue; } - catch { - result = BitConverter.ToString(key); - } - yield return result; + yield return key.ToString(); } } @@ -104,6 +100,9 @@ public static async IAsyncEnumerable GetKeysAsStrings(this IKeyValueStor /// /// The . /// + /// + /// Only keys with this prefix will be returned. + /// /// /// The keys, converted to strings. /// @@ -116,9 +115,51 @@ public static async IAsyncEnumerable GetKeysAsStrings(this IKeyValueStor /// key will be converted to a string using /// instead. /// - public static IAsyncEnumerable GetKeysAsStrings(this IKeyValueStore store) { - return store.GetKeysAsStrings(default); - } + [Obsolete("Use GetKeysAsStringsAsync instead.", false)] + public static IAsyncEnumerable GetKeysAsStrings(this IKeyValueStore store, KVKey? prefix) => store.GetKeysAsStringsAsync(prefix); + + + /// + /// Gets the keys that are defined in the store, converted to . + /// + /// + /// The . + /// + /// + /// The keys, converted to strings. + /// + /// + /// is . + /// + /// + /// Each key is converted to a string by calling + /// on . If an exception is thrown during this conversion, the + /// key will be converted to a string using + /// instead. + /// + public static IAsyncEnumerable GetKeysAsStringsAsync(this IKeyValueStore store) => store.GetKeysAsStringsAsync(default); + + + /// + /// Gets the keys that are defined in the store, converted to . + /// + /// + /// The . + /// + /// + /// The keys, converted to strings. + /// + /// + /// is . + /// + /// + /// Each key is converted to a string by calling + /// on . If an exception is thrown during this conversion, the + /// key will be converted to a string using + /// instead. + /// + [Obsolete("Use GetKeysAsStringsAsync instead.", false)] + public static IAsyncEnumerable GetKeysAsStrings(this IKeyValueStore store) => store.GetKeysAsStringsAsync(default); /// diff --git a/src/DataCore.Adapter.KeyValueStore.FASTER/DataCore.Adapter.KeyValueStore.FASTER.csproj b/src/DataCore.Adapter.KeyValueStore.FASTER/DataCore.Adapter.KeyValueStore.FASTER.csproj index ffe8f6fc..57418588 100644 --- a/src/DataCore.Adapter.KeyValueStore.FASTER/DataCore.Adapter.KeyValueStore.FASTER.csproj +++ b/src/DataCore.Adapter.KeyValueStore.FASTER/DataCore.Adapter.KeyValueStore.FASTER.csproj @@ -11,6 +11,7 @@ + diff --git a/src/DataCore.Adapter.KeyValueStore.FASTER/FasterKeyValueStore.cs b/src/DataCore.Adapter.KeyValueStore.FASTER/FasterKeyValueStore.cs index 3cacde6c..1c1479ac 100644 --- a/src/DataCore.Adapter.KeyValueStore.FASTER/FasterKeyValueStore.cs +++ b/src/DataCore.Adapter.KeyValueStore.FASTER/FasterKeyValueStore.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.IO; using System.Threading; +using System.Threading.Channels; using System.Threading.Tasks; using DataCore.Adapter.Services; @@ -16,7 +17,7 @@ namespace DataCore.Adapter.KeyValueStore.FASTER { /// /// Default implementation. /// - public class FasterKeyValueStore : RawKeyValueStore, IDisposable, IAsyncDisposable { + public partial class FasterKeyValueStore : RawKeyValueStore, IDisposable, IAsyncDisposable { /// /// Flags if the object has been disposed. @@ -29,11 +30,6 @@ public class FasterKeyValueStore : RawKeyValueStore, /// private readonly CancellationTokenSource _disposedTokenSource = new CancellationTokenSource(); - /// - /// Indicates if trace logging can be performed. - /// - private readonly bool _logTrace; - /// /// The underlying FASTER store. /// @@ -69,7 +65,13 @@ public class FasterKeyValueStore : RawKeyValueStore, /// Flags if the periodic checkpoint loop should create a new checkpoint the next /// time it runs. /// - private int _checkpointIsRequired; + private int _fullCheckpointIsRequired; + + /// + /// Lock used to ensure that incremental and full checkpoints cannot be ongoing at the + /// same time. + /// + private readonly Nito.AsyncEx.AsyncLock _checkpointLock = new Nito.AsyncEx.AsyncLock(); /// /// The size (in bytes) that the read-only portion of the FASTER log must reach before @@ -113,15 +115,13 @@ public FasterKeyValueStore(FasterKeyValueStoreOptions options, ILogger( @@ -141,14 +141,19 @@ public FasterKeyValueStore(FasterKeyValueStoreOptions options, ILogger TimeSpan.Zero) { - // Start periodic checkpoint task. - _ = RunCheckpointCreationLoopAsync(options.CheckpointInterval.Value, _disposedTokenSource.Token); + // Start periodic full checkpoint task. + _ = RunFullCheckpointLoopAsync(options.CheckpointInterval.Value, _disposedTokenSource.Token); + } + + if (_canRecordCheckpoints && options.IncrementalCheckpointInterval.HasValue && options.IncrementalCheckpointInterval.Value > TimeSpan.Zero) { + // Start periodic incremental checkpoint task. + _ = RunIncrementalCheckpointLoopAsync(options.IncrementalCheckpointInterval.Value, _disposedTokenSource.Token); } } @@ -243,7 +248,7 @@ IDevice CreateDefaultDevice() { /// A that will create a snapshot of the FASTER log at the specified /// . /// - private async Task RunCheckpointCreationLoopAsync(TimeSpan interval, CancellationToken cancellationToken) { + private async Task RunFullCheckpointLoopAsync(TimeSpan interval, CancellationToken cancellationToken) { while (!_disposed && !cancellationToken.IsCancellationRequested) { try { await Task.Delay(interval, cancellationToken).ConfigureAwait(false); @@ -251,14 +256,42 @@ private async Task RunCheckpointCreationLoopAsync(TimeSpan interval, Cancellatio } catch (OperationCanceledException) { } catch (Exception e) { - Logger.LogError(e, Resources.Log_ErrorWhileCreatingCheckpoint); + LogErrorWhileCreatingFullCheckpoint(Logger, e); + } + } + } + + + /// + /// Long-running task that will periodically take an incremental snapshot checkpoint of the FASTER + /// log and persist it using the configured . + /// + /// + /// The snapshot interval. + /// + /// + /// The cancellation token for the operation. + /// + /// + /// A that will create a snapshot of the FASTER log at the specified + /// . + /// + private async Task RunIncrementalCheckpointLoopAsync(TimeSpan interval, CancellationToken cancellationToken) { + while (!_disposed && !cancellationToken.IsCancellationRequested) { + try { + await Task.Delay(interval, cancellationToken).ConfigureAwait(false); + await TakeIncrementalCheckpointAsync(cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) { } + catch (Exception e) { + LogErrorWhileCreatingIncrementalCheckpoint(Logger, e); } } } /// - /// Takes a full snapshot checkpoint. + /// Takes a full snapshot checkpoint of both the FASTER index and the log. /// /// /// The cancellation token for the operation. @@ -279,18 +312,45 @@ public async ValueTask TakeFullCheckpointAsync(CancellationToken cancellat throw new InvalidOperationException(Resources.Error_CheckpointsAreDisabled); } - if (Interlocked.CompareExchange(ref _checkpointIsRequired, 0, 1) != 1) { + if (Interlocked.CompareExchange(ref _fullCheckpointIsRequired, 0, 1) != 1) { // No checkpoint pending. return false; } - try { - var (success, token) = await _fasterKVStore.TakeFullCheckpointAsync(CheckpointType.Snapshot, cancellationToken).ConfigureAwait(false); - return success; + using (await _checkpointLock.LockAsync(cancellationToken).ConfigureAwait(false)) { + try { + var (success, token) = await _fasterKVStore.TakeFullCheckpointAsync(CheckpointType.Snapshot, cancellationToken).ConfigureAwait(false); + return success; + } + catch { + Interlocked.Exchange(ref _fullCheckpointIsRequired, 1); + throw; + } } - catch { - Interlocked.Exchange(ref _checkpointIsRequired, 1); - throw; + } + + + /// + /// Takes an incremental checkpoint of the FASTER log. + /// + /// + /// The cancellation token for the operation. + /// + /// + /// A that will return a flag indicating if a + /// checkpoint was created. + /// + /// + /// Checkpoint management is disabled. + /// + public async ValueTask TakeIncrementalCheckpointAsync(CancellationToken cancellationToken = default) { + if (!_canRecordCheckpoints) { + throw new InvalidOperationException(Resources.Error_CheckpointsAreDisabled); + } + + using (await _checkpointLock.LockAsync(cancellationToken).ConfigureAwait(false)) { + var(success, token) = await _fasterKVStore.TakeHybridLogCheckpointAsync(CheckpointType.Snapshot, tryIncremental: true, cancellationToken: cancellationToken).ConfigureAwait(false); + return success; } } @@ -318,13 +378,7 @@ private async Task RunLogCompactionLoopAsync(TimeSpan interval, CancellationToke // (oldest entries here) BeginAddress <= HeadAddress (where the in-memory region begins) <= SafeReadOnlyAddress (entries between here and tail updated in-place) < TailAddress (entries added here) var safeReadOnlyRegionByteSize = logAccessor.SafeReadOnlyAddress - logAccessor.BeginAddress; if (safeReadOnlyRegionByteSize < _logCompactionThresholdBytes) { - if (_logTrace) { - Logger.LogTrace( - Resources.Log_SkippingLogCompaction, - safeReadOnlyRegionByteSize, - _logCompactionThresholdBytes - ); - } + LogSkippingCompaction(Logger, safeReadOnlyRegionByteSize, _logCompactionThresholdBytes); _numConsecutiveLogCompactions = 0; continue; } @@ -334,38 +388,24 @@ private async Task RunLogCompactionLoopAsync(TimeSpan interval, CancellationToke try { session.Compact(compactUntilAddress, CompactionType.Scan); // Log has been modified; we need a new checkpoint to be created. - Interlocked.Exchange(ref _checkpointIsRequired, 1); + Interlocked.Exchange(ref _fullCheckpointIsRequired, 1); } finally { ReturnPooledSession(session); } _numConsecutiveLogCompactions++; - - if (_logTrace) { - Logger.LogTrace( - Resources.Log_LogCompacted, - safeReadOnlyRegionByteSize, - logAccessor.SafeReadOnlyAddress - logAccessor.BeginAddress, - _numConsecutiveLogCompactions - ); - } + LogCompactionCompleted(Logger, safeReadOnlyRegionByteSize, logAccessor.SafeReadOnlyAddress - logAccessor.BeginAddress, _numConsecutiveLogCompactions); if (_numConsecutiveLogCompactions >= ConsecutiveCompactOperationsBeforeThresholdIncrease) { _logCompactionThresholdBytes *= 2; - if (_logTrace) { - Logger.LogTrace( - Resources.Log_LogCompactionThresholdIncreased, - _logCompactionThresholdBytes / 2, - _logCompactionThresholdBytes - ); - } + LogCompactionThresholdIncreased(Logger, _logCompactionThresholdBytes / 2, _logCompactionThresholdBytes); _numConsecutiveLogCompactions = 0; } } catch (OperationCanceledException) { } catch (Exception e) { - Logger.LogError(e, Resources.Log_CompactionError); + LogCompactionError(Logger, e); } } } @@ -469,7 +509,7 @@ private async ValueTask WriteCoreAsync(KVKey key, byte[] value) { } // Mark the cache as dirty. - Interlocked.Exchange(ref _checkpointIsRequired, 1); + Interlocked.Exchange(ref _fullCheckpointIsRequired, 1); } finally { ReturnPooledSession(session); @@ -549,7 +589,7 @@ protected override async ValueTask DeleteAsync(KVKey key) { if (result.Status.Found) { // Mark the cache as dirty. - Interlocked.Exchange(ref _checkpointIsRequired, 1); + Interlocked.Exchange(ref _fullCheckpointIsRequired, 1); } return result.Status.IsCompletedSuccessfully; @@ -565,10 +605,17 @@ protected override async IAsyncEnumerable GetKeysAsync(KVKey? prefix) { await Task.Yield(); var session = GetPooledSession(); try { - using (var iterator = session.Iterate()) { - while (iterator.GetNext(out var recordInfo)) { - var key = iterator.GetKey().AsSpan().ToArray(); + var channel = Channel.CreateUnbounded(new UnboundedChannelOptions() { + SingleReader = true, + SingleWriter = true, + AllowSynchronousContinuations = false + }); + var funcs = new ScanIteratorFunctions(channel); + + session.Iterate(ref funcs); + while (await channel.Reader.WaitToReadAsync().ConfigureAwait(false)) { + while (channel.Reader.TryRead(out var key)) { if (prefix == null || prefix.Value.Length == 0 || StartsWithPrefix(prefix.Value, key)) { yield return key; } @@ -635,7 +682,7 @@ protected virtual void Dispose(bool disposing) { await TakeFullCheckpointAsync().ConfigureAwait(false); } catch (Exception e) { - Logger.LogError(e, Resources.Log_ErrorWhileCreatingCheckpoint); + LogErrorWhileCreatingFullCheckpoint(Logger, e); } finally { @lock.Set(); @@ -665,5 +712,80 @@ protected virtual async ValueTask DisposeAsyncCore() { DisposeCommon(); } + + [LoggerMessage(100, LogLevel.Information, "Checkpoint management is disabled; backup and restore of data will not be performed.")] + static partial void LogCheckpointManagerIsDisabled(ILogger logger); + + [LoggerMessage(101, LogLevel.Warning, "Error while recovering the FASTER log from the latest checkpoint. This message can be ignored the first time the FASTER store is used as there will be no checkpoint available to recover from.")] + static partial void LogErrorWhileRecoveringCheckpoint(ILogger logger, Exception e); + + [LoggerMessage(102, LogLevel.Error, "Error while creating an incremental recovery checkpoint for the FASTER log.")] + static partial void LogErrorWhileCreatingIncrementalCheckpoint(ILogger logger, Exception e); + + [LoggerMessage(103, LogLevel.Error, "Error while creating a full recovery checkpoint for the FASTER log.")] + static partial void LogErrorWhileCreatingFullCheckpoint(ILogger logger, Exception e); + + [LoggerMessage(104, LogLevel.Trace, "Skipping FASTER log compaction. Safe read-only region size: {safeReadOnlyRegionByteSize} bytes. Threshold: {logCompactionThresholdBytes} bytes.")] + static partial void LogSkippingCompaction(ILogger logger, long safeReadOnlyRegionByteSize, long logCompactionThresholdBytes); + + [LoggerMessage(105, LogLevel.Trace, "FASTER log compaction completed. Safe read-only region size before: {sizeBeforeBytes} bytes. Size after: {sizeAfterBytes} bytes. Consecutive compactions: {consecutiveCompactions}")] + static partial void LogCompactionCompleted(ILogger logger, long sizeBeforeBytes, long sizeAfterBytes, byte consecutiveCompactions); + + [LoggerMessage(106, LogLevel.Trace, "Increasing FASTER log compaction threshold. Previous threshold: {oldThresholdBytes} bytes. New threshold: {newThresholdBytes} bytes.")] + static partial void LogCompactionThresholdIncreased(ILogger logger, long oldThresholdBytes, long newThresholdBytes); + + [LoggerMessage(107, LogLevel.Error, "Error while performing FASTER log compaction.")] + static partial void LogCompactionError(ILogger logger, Exception e); + + + /// + /// implementation that allows us to use + /// FASTER's "push" key iteration: https://microsoft.github.io/FASTER/docs/fasterkv-basics/#key-iteration + /// + private struct ScanIteratorFunctions : IScanIteratorFunctions { + + /// + /// The channel to publish keys to as they are iterated over. + /// + private readonly ChannelWriter _channel; + + /// + /// Creates a new instance. + /// + /// + /// The channel to publish keys to as they are iterated over. + /// + public ScanIteratorFunctions(ChannelWriter channel) { + _channel = channel; + } + + + /// + public bool OnStart(long beginAddress, long endAddress) => true; + + + /// + public bool SingleReader(ref SpanByte key, ref SpanByte value, RecordMetadata recordMetadata, long numberOfRecords) { + return _channel.TryWrite(key.AsSpan().ToArray()); + } + + + /// + public bool ConcurrentReader(ref SpanByte key, ref SpanByte value, RecordMetadata recordMetadata, long numberOfRecords) => SingleReader(ref key, ref value, recordMetadata, numberOfRecords); + + + /// + public void OnStop(bool completed, long numberOfRecords) { + _channel.TryComplete(); + } + + + /// + public void OnException(Exception exception, long numberOfRecords) { + _channel.TryComplete(exception); + } + + } + } } diff --git a/src/DataCore.Adapter.KeyValueStore.FASTER/FasterKeyValueStoreOptions.cs b/src/DataCore.Adapter.KeyValueStore.FASTER/FasterKeyValueStoreOptions.cs index 3d41ab8e..c257b461 100644 --- a/src/DataCore.Adapter.KeyValueStore.FASTER/FasterKeyValueStoreOptions.cs +++ b/src/DataCore.Adapter.KeyValueStore.FASTER/FasterKeyValueStoreOptions.cs @@ -95,16 +95,35 @@ public class FasterKeyValueStoreOptions : Services.KeyValueStoreOptions { public int SegmentSizeBits { get; set; } = 27; /// - /// The interval at which log checkpoints will be saved to - /// created by . + /// The interval at which full (index + log) checkpoints will be saved to /// /// - /// Note that checkpoints will not be created if no is - /// supplied by , or if + /// Note that full checkpoints will not be created if no + /// is supplied by , or if /// is or less than or equal to . /// public TimeSpan? CheckpointInterval { get; set; } + /// + /// The interval at which incremental log checkpoints will be saved to + /// + /// + /// + /// + /// Note that incremental checkpoints will not be created if no + /// is supplied by , or if + /// is or less than or equal to . + /// + /// + /// + /// When both full and incremental checkpoints are enabled, the + /// should be configured so that incremental checkpoints are taken more frequently than + /// full checkpoints. + /// + /// + /// + public TimeSpan? IncrementalCheckpointInterval { get; set; } + /// /// The interval at which the store will check if the log should be compacted by /// removing expired items. diff --git a/src/DataCore.Adapter.KeyValueStore.FASTER/PublicAPI.Unshipped.txt b/src/DataCore.Adapter.KeyValueStore.FASTER/PublicAPI.Unshipped.txt index 93fda986..8b7675ec 100644 --- a/src/DataCore.Adapter.KeyValueStore.FASTER/PublicAPI.Unshipped.txt +++ b/src/DataCore.Adapter.KeyValueStore.FASTER/PublicAPI.Unshipped.txt @@ -1,6 +1,9 @@ #nullable enable +DataCore.Adapter.KeyValueStore.FASTER.FasterKeyValueStore.TakeIncrementalCheckpointAsync(System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) -> System.Threading.Tasks.ValueTask DataCore.Adapter.KeyValueStore.FASTER.FasterKeyValueStoreOptions.EnableRawWrites.get -> bool DataCore.Adapter.KeyValueStore.FASTER.FasterKeyValueStoreOptions.EnableRawWrites.set -> void +DataCore.Adapter.KeyValueStore.FASTER.FasterKeyValueStoreOptions.IncrementalCheckpointInterval.get -> System.TimeSpan? +DataCore.Adapter.KeyValueStore.FASTER.FasterKeyValueStoreOptions.IncrementalCheckpointInterval.set -> void override DataCore.Adapter.KeyValueStore.FASTER.FasterKeyValueStore.AllowRawWrites.get -> bool override DataCore.Adapter.KeyValueStore.FASTER.FasterKeyValueStore.ReadAsync(DataCore.Adapter.Services.KVKey key) -> System.Threading.Tasks.ValueTask override DataCore.Adapter.KeyValueStore.FASTER.FasterKeyValueStore.ReadRawAsync(DataCore.Adapter.Services.KVKey key) -> System.Threading.Tasks.ValueTask diff --git a/src/DataCore.Adapter.KeyValueStore.FASTER/Resources.Designer.cs b/src/DataCore.Adapter.KeyValueStore.FASTER/Resources.Designer.cs index e38743b7..add26f5e 100644 --- a/src/DataCore.Adapter.KeyValueStore.FASTER/Resources.Designer.cs +++ b/src/DataCore.Adapter.KeyValueStore.FASTER/Resources.Designer.cs @@ -77,68 +77,5 @@ internal static string Error_StoreIsReadOnly { return ResourceManager.GetString("Error_StoreIsReadOnly", resourceCulture); } } - - /// - /// Looks up a localized string similar to An error occurred while compacting the FASTER log.. - /// - internal static string Log_CompactionError { - get { - return ResourceManager.GetString("Log_CompactionError", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to An error occurred while creating a recovery checkpoint for the FASTER log.. - /// - internal static string Log_ErrorWhileCreatingCheckpoint { - get { - return ResourceManager.GetString("Log_ErrorWhileCreatingCheckpoint", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to An error occurred while recovering the FASTER log from the last checkpoint.. - /// - internal static string Log_ErrorWhileRecoveringCheckpoint { - get { - return ResourceManager.GetString("Log_ErrorWhileRecoveringCheckpoint", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to FASTER log compacted. Safe read-only region size before: {SizeBefore} bytes. After: {SizeAfter} bytes. Consecutive compactions: {ConsecutiveIterations}.. - /// - internal static string Log_LogCompacted { - get { - return ResourceManager.GetString("Log_LogCompacted", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to FASTER log compaction threshold increased. Before: {SizeBefore} bytes. After: {SizeAfter} bytes.. - /// - internal static string Log_LogCompactionThresholdIncreased { - get { - return ResourceManager.GetString("Log_LogCompactionThresholdIncreased", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Checkpoint management is disabled; backup and restore of data will not be performed.. - /// - internal static string Log_NoCheckpointManagerProvided { - get { - return ResourceManager.GetString("Log_NoCheckpointManagerProvided", resourceCulture); - } - } - - /// - /// Looks up a localized string similar to Skipping FASTER log compaction. Safe read-only region size: {0} bytes. Threshold: {Size} bytes. - /// - internal static string Log_SkippingLogCompaction { - get { - return ResourceManager.GetString("Log_SkippingLogCompaction", resourceCulture); - } - } } } diff --git a/src/DataCore.Adapter.KeyValueStore.FASTER/Resources.resx b/src/DataCore.Adapter.KeyValueStore.FASTER/Resources.resx index 46e2a362..6970dd3d 100644 --- a/src/DataCore.Adapter.KeyValueStore.FASTER/Resources.resx +++ b/src/DataCore.Adapter.KeyValueStore.FASTER/Resources.resx @@ -123,25 +123,4 @@ This operation cannot be performed on a read-only store. - - An error occurred while compacting the FASTER log. - - - An error occurred while creating a recovery checkpoint for the FASTER log. - - - An error occurred while recovering the FASTER log from the last checkpoint. - - - FASTER log compacted. Safe read-only region size before: {SizeBefore} bytes. After: {SizeAfter} bytes. Consecutive compactions: {ConsecutiveIterations}. - - - FASTER log compaction threshold increased. Before: {SizeBefore} bytes. After: {SizeAfter} bytes. - - - Checkpoint management is disabled; backup and restore of data will not be performed. - - - Skipping FASTER log compaction. Safe read-only region size: {0} bytes. Threshold: {Size} bytes - \ No newline at end of file diff --git a/test/DataCore.Adapter.Tests/KeyValueStoreTests.cs b/test/DataCore.Adapter.Tests/KeyValueStoreTests.cs index 9bd17d81..78714ffa 100644 --- a/test/DataCore.Adapter.Tests/KeyValueStoreTests.cs +++ b/test/DataCore.Adapter.Tests/KeyValueStoreTests.cs @@ -104,7 +104,7 @@ public async Task ShouldDeleteValueFromStore(CompressionLevel compressionLevel) public async Task ShouldListKeys(CompressionLevel compressionLevel) { var store = CreateStore(compressionLevel); try { - var keys = Enumerable.Range(1, 10).Select(x => $"key:{x}").ToArray(); + var keys = Enumerable.Range(1, 100).Select(x => $"key:{x}").ToArray(); foreach (var key in keys) { await store.WriteAsync(key, 0);