From 2bc2a0ecca22b2f7109531457e6ca6c2a90186ba Mon Sep 17 00:00:00 2001 From: Graham Watts Date: Fri, 7 Jan 2022 13:33:32 +0200 Subject: [PATCH 1/6] Refactor key-value store functionality **Contains breaking changes!** - `KVKey` is a new type that represents a key or prefix on an `IKeyValueStore` that supports implicit conversion from `byte[]`, `string`, `int`, etc. - `IKeyValueStore` no longer supports generic read/write methods. Read/write methods now just accept `byte[]` as the value. - New extension methods in DataCore.Adapter provide conversion to/from JSON when writing or reading values from a store. - `KeyValueStore` is a base implementation of `IKeyValueStore` that other implementations inherit from. - New projects for a file system-based store, and a Sqlite store. --- DataCore.Adapter.sln | 14 + build/Dependencies.props | 2 + .../AbstractionsResources.Designer.cs | 11 +- .../AbstractionsResources.resx | 3 + .../Services/IKeyValueStore.cs | 29 +- .../Services/InMemoryKeyValueStore.cs | 47 +- .../Services/KVKey.cs | 109 ++ .../Services/KeyValueStore.cs | 314 +++++ .../Services/KeyValueStoreExtensions.cs | 1234 +---------------- .../Services/KeyValueStoreReadResult.cs | 34 + .../Services/ScopedKeyValueStore.cs | 156 +-- .../DataCore.Adapter.Http.Proxy.csproj | 8 - ...taCore.Adapter.KeyValueStore.FASTER.csproj | 1 - .../FasterKeyValueStore.cs | 45 +- .../FasterKeyValueStoreOptions.cs | 6 - .../IFasterKeyValueStoreSerializer.cs | 41 - .../JsonFasterKeyValueStoreSerializer.cs | 45 - ...re.Adapter.KeyValueStore.FileSystem.csproj | 34 + .../KeyValueFileStore.cs | 337 +++++ .../KeyValueFileStoreOptions.cs | 25 + .../Resources.Designer.cs | 90 ++ .../Resources.resx | 129 ++ ...taCore.Adapter.KeyValueStore.Sqlite.csproj | 34 + .../Resources.Designer.cs | 90 ++ .../Resources.resx | 129 ++ .../SqliteKeyValueStore.cs | 194 +++ .../SqliteKeyValueStoreOptions.cs | 20 + .../AssetModel/AssetModelManager.cs | 19 +- .../RealTimeData/SnapshotTagValueManager.cs | 16 +- .../Services/KVStoreExtensions.cs | 78 ++ src/DataCore.Adapter/Tags/TagManager.cs | 17 +- .../DataCore.Adapter.Tests.csproj | 6 + .../FasterKeyValueStoreTests.cs | 8 +- .../KeyValueFileStoreTests.cs | 67 + .../KeyValueStoreTests.cs | 78 +- .../SqliteKeyValueStoreTests.cs | 72 + 36 files changed, 1989 insertions(+), 1553 deletions(-) create mode 100644 src/DataCore.Adapter.Abstractions/Services/KVKey.cs create mode 100644 src/DataCore.Adapter.Abstractions/Services/KeyValueStore.cs delete mode 100644 src/DataCore.Adapter.KeyValueStore.FASTER/IFasterKeyValueStoreSerializer.cs delete mode 100644 src/DataCore.Adapter.KeyValueStore.FASTER/JsonFasterKeyValueStoreSerializer.cs create mode 100644 src/DataCore.Adapter.KeyValueStore.FileSystem/DataCore.Adapter.KeyValueStore.FileSystem.csproj create mode 100644 src/DataCore.Adapter.KeyValueStore.FileSystem/KeyValueFileStore.cs create mode 100644 src/DataCore.Adapter.KeyValueStore.FileSystem/KeyValueFileStoreOptions.cs create mode 100644 src/DataCore.Adapter.KeyValueStore.FileSystem/Resources.Designer.cs create mode 100644 src/DataCore.Adapter.KeyValueStore.FileSystem/Resources.resx create mode 100644 src/DataCore.Adapter.KeyValueStore.Sqlite/DataCore.Adapter.KeyValueStore.Sqlite.csproj create mode 100644 src/DataCore.Adapter.KeyValueStore.Sqlite/Resources.Designer.cs create mode 100644 src/DataCore.Adapter.KeyValueStore.Sqlite/Resources.resx create mode 100644 src/DataCore.Adapter.KeyValueStore.Sqlite/SqliteKeyValueStore.cs create mode 100644 src/DataCore.Adapter.KeyValueStore.Sqlite/SqliteKeyValueStoreOptions.cs create mode 100644 src/DataCore.Adapter/Services/KVStoreExtensions.cs create mode 100644 test/DataCore.Adapter.Tests/KeyValueFileStoreTests.cs create mode 100644 test/DataCore.Adapter.Tests/SqliteKeyValueStoreTests.cs diff --git a/DataCore.Adapter.sln b/DataCore.Adapter.sln index af96e88a..26caaa1b 100644 --- a/DataCore.Adapter.sln +++ b/DataCore.Adapter.sln @@ -139,6 +139,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ExampleHostedAdapter", "exa EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DataCore.Adapter.KeyValueStore.FASTER", "src\DataCore.Adapter.KeyValueStore.FASTER\DataCore.Adapter.KeyValueStore.FASTER.csproj", "{E2C09B9E-FCE1-4C61-A547-16ADFA76B7F6}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataCore.Adapter.KeyValueStore.Sqlite", "src\DataCore.Adapter.KeyValueStore.Sqlite\DataCore.Adapter.KeyValueStore.Sqlite.csproj", "{22DD3CC9-034B-4C43-A143-6240E8E833CC}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DataCore.Adapter.KeyValueStore.FileSystem", "src\DataCore.Adapter.KeyValueStore.FileSystem\DataCore.Adapter.KeyValueStore.FileSystem.csproj", "{8C225A69-794C-4451-B5CE-91EAD5269C87}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -269,6 +273,14 @@ Global {E2C09B9E-FCE1-4C61-A547-16ADFA76B7F6}.Debug|Any CPU.Build.0 = Debug|Any CPU {E2C09B9E-FCE1-4C61-A547-16ADFA76B7F6}.Release|Any CPU.ActiveCfg = Release|Any CPU {E2C09B9E-FCE1-4C61-A547-16ADFA76B7F6}.Release|Any CPU.Build.0 = Release|Any CPU + {22DD3CC9-034B-4C43-A143-6240E8E833CC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {22DD3CC9-034B-4C43-A143-6240E8E833CC}.Debug|Any CPU.Build.0 = Debug|Any CPU + {22DD3CC9-034B-4C43-A143-6240E8E833CC}.Release|Any CPU.ActiveCfg = Release|Any CPU + {22DD3CC9-034B-4C43-A143-6240E8E833CC}.Release|Any CPU.Build.0 = Release|Any CPU + {8C225A69-794C-4451-B5CE-91EAD5269C87}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {8C225A69-794C-4451-B5CE-91EAD5269C87}.Debug|Any CPU.Build.0 = Debug|Any CPU + {8C225A69-794C-4451-B5CE-91EAD5269C87}.Release|Any CPU.ActiveCfg = Release|Any CPU + {8C225A69-794C-4451-B5CE-91EAD5269C87}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -314,6 +326,8 @@ Global {B5692AC0-AA11-4062-8F65-8A120C8846FA} = {7701E410-A9F0-4F50-80E4-51576DABB306} {CA874402-DD93-40C0-893A-BC8D1D961A43} = {92FD0A48-0AE7-44A6-86BC-C4E47E014C17} {E2C09B9E-FCE1-4C61-A547-16ADFA76B7F6} = {B158078F-D217-46FE-A9B8-7BD1B99922BA} + {22DD3CC9-034B-4C43-A143-6240E8E833CC} = {B158078F-D217-46FE-A9B8-7BD1B99922BA} + {8C225A69-794C-4451-B5CE-91EAD5269C87} = {B158078F-D217-46FE-A9B8-7BD1B99922BA} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {323415F4-D394-4EAA-B259-7C6834BFD819} diff --git a/build/Dependencies.props b/build/Dependencies.props index e0bf9280..5d29e3e0 100644 --- a/build/Dependencies.props +++ b/build/Dependencies.props @@ -17,6 +17,7 @@ 1.1.0 5.2.7 6.0.0 + 6.0.1 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + Error while deleting file '{FilePath}'. + + + Error while reading from file '{FilePath}'. + + + Error while writing to file '{FilePath}'. + + \ No newline at end of file diff --git a/src/DataCore.Adapter.KeyValueStore.Sqlite/DataCore.Adapter.KeyValueStore.Sqlite.csproj b/src/DataCore.Adapter.KeyValueStore.Sqlite/DataCore.Adapter.KeyValueStore.Sqlite.csproj new file mode 100644 index 00000000..ef1a8c1a --- /dev/null +++ b/src/DataCore.Adapter.KeyValueStore.Sqlite/DataCore.Adapter.KeyValueStore.Sqlite.csproj @@ -0,0 +1,34 @@ + + + + net461;netstandard2.0 + DataCore.Adapter.KeyValueStore.Sqlite + $(PackagePrefix).Adapter.KeyValueStore.Sqlite + Sqlite key-value store for App Store Connect adapters. + true + + + + + + + + + + + + + True + True + Resources.resx + + + + + + ResXFileCodeGenerator + Resources.Designer.cs + + + + diff --git a/src/DataCore.Adapter.KeyValueStore.Sqlite/Resources.Designer.cs b/src/DataCore.Adapter.KeyValueStore.Sqlite/Resources.Designer.cs new file mode 100644 index 00000000..40863f63 --- /dev/null +++ b/src/DataCore.Adapter.KeyValueStore.Sqlite/Resources.Designer.cs @@ -0,0 +1,90 @@ +//------------------------------------------------------------------------------ +// +// This code was generated by a tool. +// Runtime Version:4.0.30319.42000 +// +// Changes to this file may cause incorrect behavior and will be lost if +// the code is regenerated. +// +//------------------------------------------------------------------------------ + +namespace DataCore.Adapter.KeyValueStore.Sqlite { + using System; + + + /// + /// A strongly-typed resource class, for looking up localized strings, etc. + /// + // This class was auto-generated by the StronglyTypedResourceBuilder + // class via a tool like ResGen or Visual Studio. + // To add or remove a member, edit your .ResX file then rerun ResGen + // with the /str option, or rebuild your VS project. + [global::System.CodeDom.Compiler.GeneratedCodeAttribute("System.Resources.Tools.StronglyTypedResourceBuilder", "17.0.0.0")] + [global::System.Diagnostics.DebuggerNonUserCodeAttribute()] + [global::System.Runtime.CompilerServices.CompilerGeneratedAttribute()] + internal class Resources { + + private static global::System.Resources.ResourceManager resourceMan; + + private static global::System.Globalization.CultureInfo resourceCulture; + + [global::System.Diagnostics.CodeAnalysis.SuppressMessageAttribute("Microsoft.Performance", "CA1811:AvoidUncalledPrivateCode")] + internal Resources() { + } + + /// + /// Returns the cached ResourceManager instance used by this class. + /// + [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] + internal static global::System.Resources.ResourceManager ResourceManager { + get { + if (object.ReferenceEquals(resourceMan, null)) { + global::System.Resources.ResourceManager temp = new global::System.Resources.ResourceManager("DataCore.Adapter.KeyValueStore.Sqlite.Resources", typeof(Resources).Assembly); + resourceMan = temp; + } + return resourceMan; + } + } + + /// + /// Overrides the current thread's CurrentUICulture property for all + /// resource lookups using this strongly typed resource class. + /// + [global::System.ComponentModel.EditorBrowsableAttribute(global::System.ComponentModel.EditorBrowsableState.Advanced)] + internal static global::System.Globalization.CultureInfo Culture { + get { + return resourceCulture; + } + set { + resourceCulture = value; + } + } + + /// + /// Looks up a localized string similar to Error while deleting key '{Key}'.. + /// + internal static string Log_ErrorDeletingValue { + get { + return ResourceManager.GetString("Log_ErrorDeletingValue", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Error while reading from key '{Key}'.. + /// + internal static string Log_ErrorReadingValue { + get { + return ResourceManager.GetString("Log_ErrorReadingValue", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to Error while writing to key '{Key}'.. + /// + internal static string Log_ErrorWritingValue { + get { + return ResourceManager.GetString("Log_ErrorWritingValue", resourceCulture); + } + } + } +} diff --git a/src/DataCore.Adapter.KeyValueStore.Sqlite/Resources.resx b/src/DataCore.Adapter.KeyValueStore.Sqlite/Resources.resx new file mode 100644 index 00000000..47f4d83a --- /dev/null +++ b/src/DataCore.Adapter.KeyValueStore.Sqlite/Resources.resx @@ -0,0 +1,129 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + Error while deleting key '{Key}'. + + + Error while reading from key '{Key}'. + + + Error while writing to key '{Key}'. + + \ No newline at end of file diff --git a/src/DataCore.Adapter.KeyValueStore.Sqlite/SqliteKeyValueStore.cs b/src/DataCore.Adapter.KeyValueStore.Sqlite/SqliteKeyValueStore.cs new file mode 100644 index 00000000..dd4f027c --- /dev/null +++ b/src/DataCore.Adapter.KeyValueStore.Sqlite/SqliteKeyValueStore.cs @@ -0,0 +1,194 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Threading.Tasks; + +using DataCore.Adapter.Services; + +using Microsoft.Data.Sqlite; +using Microsoft.Extensions.Logging; + +namespace DataCore.Adapter.KeyValueStore.Sqlite { + + /// + /// that uses a Sqlite database to store values. + /// + public class SqliteKeyValueStore : Services.KeyValueStore { + + /// + /// The Sqlite connection string. + /// + private readonly string _connectionString; + + /// + /// The logger for the store. + /// + private readonly ILogger _logger; + + + /// + /// Creates a new object. + /// + /// + /// The for the store. + /// + /// + /// The logger for the store. + /// + /// + /// is . + /// + public SqliteKeyValueStore(SqliteKeyValueStoreOptions options, ILogger? logger = null) { + if (options == null) { + throw new ArgumentNullException(nameof(options)); + } + + _logger = logger ?? (ILogger) Microsoft.Extensions.Logging.Abstractions.NullLogger.Instance; + + _connectionString = string.IsNullOrWhiteSpace(options.ConnectionString) + ? SqliteKeyValueStoreOptions.DefaultConnectionString + : options.ConnectionString; + + CreateKVTable(); + } + + + /// + /// Creates the key-value table in the SQlite database. + /// + private void CreateKVTable() { + using (var connection = new SqliteConnection(_connectionString)) { + connection.Open(); + + using (var command = connection.CreateCommand()) { + command.CommandText = @"CREATE TABLE IF NOT EXISTS kvstore (key TEXT PRIMARY KEY, value BLOB)"; + command.ExecuteNonQuery(); + } + } + } + + + /// + protected override ValueTask WriteAsync(KVKey key, byte[] value) { + var hexKey = ConvertBytesToHexString(key); + + try { + using (var connection = new SqliteConnection(_connectionString)) { + connection.Open(); + + using (var transaction = connection.BeginTransaction()) + using (var command = connection.CreateCommand()) { + command.Transaction = transaction; + + command.CommandText = "INSERT INTO kvstore (key, value) VALUES ($key, $value) ON CONFLICT (key) DO UPDATE SET value = $value"; + command.Parameters.AddWithValue("$key", hexKey); + command.Parameters.AddWithValue("$value", value); + + command.ExecuteNonQuery(); + transaction.Commit(); + + return new ValueTask(KeyValueStoreOperationStatus.OK); + } + } + } + catch (Exception e) { + _logger.LogError(e, Resources.Log_ErrorWritingValue, hexKey); + return new ValueTask(KeyValueStoreOperationStatus.Error); + } + } + + + /// + protected override async ValueTask ReadAsync(KVKey key) { + var hexKey = ConvertBytesToHexString(key); + + try { + using (var connection = new SqliteConnection(_connectionString)) { + connection.Open(); + + using (var command = connection.CreateCommand()) { + command.CommandText = "SELECT value FROM kvstore WHERE key = $key LIMIT 1"; + command.Parameters.AddWithValue("$key", hexKey); + + using (var reader = command.ExecuteReader()) { + if (!reader.Read()) { + return new KeyValueStoreReadResult(KeyValueStoreOperationStatus.NotFound, default); + } + + using (var stream = reader.GetStream(0)) + using (var ms = new MemoryStream()){ + await stream.CopyToAsync(ms).ConfigureAwait(false); + return new KeyValueStoreReadResult(KeyValueStoreOperationStatus.OK, ms.ToArray()); + } + } + } + } + } + catch (Exception e) { + _logger.LogError(e, Resources.Log_ErrorReadingValue, hexKey); + return new KeyValueStoreReadResult(KeyValueStoreOperationStatus.Error, default); + } + } + + + /// + protected override ValueTask DeleteAsync(KVKey key) { + var hexKey = ConvertBytesToHexString(key); + + try { + using (var connection = new SqliteConnection(_connectionString)) { + connection.Open(); + + using (var transaction = connection.BeginTransaction()) + using (var command = connection.CreateCommand()) { + command.Transaction = transaction; + command.CommandText = @"DELETE FROM kvstore WHERE key = $key"; + command.Parameters.AddWithValue("$key", hexKey); + + var count = command.ExecuteNonQuery(); + transaction.Commit(); + + return new ValueTask(count == 0 + ? KeyValueStoreOperationStatus.NotFound + : KeyValueStoreOperationStatus.OK + ); + } + } + } + catch (Exception e) { + _logger.LogError(e, Resources.Log_ErrorDeletingValue, hexKey); + return new ValueTask(KeyValueStoreOperationStatus.Error); + } + } + + + /// + protected override IEnumerable GetKeys(KVKey? prefix) { + var hexPrefix = prefix == null || prefix.Value.Length == 0 + ? null + : ConvertBytesToHexString(prefix); + + using (var connection = new SqliteConnection(_connectionString)) { + connection.Open(); + + using (var command = connection.CreateCommand()) { + if (hexPrefix == null) { + command.CommandText = "SELECT key FROM kvstore"; + } + else { + command.CommandText = "SELECT key from kvstore WHERE key LIKE $filter"; + command.Parameters.AddWithValue("$filter", string.Concat(hexPrefix, "%")); + } + + using (var reader = command.ExecuteReader()) { + while (reader.Read()) { + yield return ConvertHexStringToBytes(reader.GetString(0)); + } + } + } + } + } + + } +} diff --git a/src/DataCore.Adapter.KeyValueStore.Sqlite/SqliteKeyValueStoreOptions.cs b/src/DataCore.Adapter.KeyValueStore.Sqlite/SqliteKeyValueStoreOptions.cs new file mode 100644 index 00000000..5dd0e8aa --- /dev/null +++ b/src/DataCore.Adapter.KeyValueStore.Sqlite/SqliteKeyValueStoreOptions.cs @@ -0,0 +1,20 @@ +namespace DataCore.Adapter.KeyValueStore.Sqlite { + + /// + /// Options for . + /// + public class SqliteKeyValueStoreOptions { + + /// + /// Default Sqlite connection string. + /// + public const string DefaultConnectionString = "Data Source=adapter-kvstore.db"; + + /// + /// The Sqlite connection string to use. + /// + public string ConnectionString { get; set; } = DefaultConnectionString; + + } + +} diff --git a/src/DataCore.Adapter/AssetModel/AssetModelManager.cs b/src/DataCore.Adapter/AssetModel/AssetModelManager.cs index b1d38ccd..82ac9c6a 100644 --- a/src/DataCore.Adapter/AssetModel/AssetModelManager.cs +++ b/src/DataCore.Adapter/AssetModel/AssetModelManager.cs @@ -3,10 +3,12 @@ using System.Collections.Generic; using System.Linq; using System.Runtime.CompilerServices; +using System.Text.Json; using System.Threading; using System.Threading.Tasks; using DataCore.Adapter.Diagnostics; +using DataCore.Adapter.Json; using DataCore.Adapter.Services; using IntelligentPlant.BackgroundTasks; @@ -38,6 +40,11 @@ public class AssetModelManager : IAssetModelBrowse, IAssetModelSearch, IDisposab /// private readonly IKeyValueStore _keyValueStore; + /// + /// Options for serializing/deserializing nodes. + /// + private readonly JsonSerializerOptions _jsonOptions = new JsonSerializerOptions(); + /// /// Flags if the class has been initialised. /// @@ -92,6 +99,8 @@ public AssetModelManager( _onConfigurationChange = onConfigurationChange; _keyValueStore = keyValueStore.CreateScopedStore("asset-model-manager:"); + _jsonOptions.AddDataCoreAdapterConverters(); + _initTask = new Lazy(() => InitAsyncCore(_disposedTokenSource.Token), LazyThreadSafetyMode.ExecutionAndPublication); } @@ -168,7 +177,7 @@ private async Task InitAsyncCore(CancellationToken cancellationToken) { _nodesById.Clear(); // "nodes" key contains an array of the defined node IDs. - var readResult = await _keyValueStore.ReadAsync("nodes").ConfigureAwait(false); + var readResult = await _keyValueStore.ReadJsonAsync("nodes", _jsonOptions).ConfigureAwait(false); if (cancellationToken.IsCancellationRequested) { return; } @@ -186,7 +195,7 @@ private async Task InitAsyncCore(CancellationToken cancellationToken) { } // "nodes:{id}" key contains the the definition with ID {id}. - var nodeReadResult = await _keyValueStore.ReadAsync(string.Concat("nodes:", nodeId)).ConfigureAwait(false); + var nodeReadResult = await _keyValueStore.ReadJsonAsync(string.Concat("nodes:", nodeId), _jsonOptions).ConfigureAwait(false); if (nodeReadResult.Value == null) { continue; } @@ -340,7 +349,7 @@ private async ValueTask AddOrUpdateNodeCoreAsync(AssetModelNode node, bool requi try { // "nodes:{id}" key contains the definition with ID {id}. - var result = await _keyValueStore.WriteAsync(string.Concat("nodes:", node.Id), node).ConfigureAwait(false); + var result = await _keyValueStore.WriteJsonAsync(string.Concat("nodes:", node.Id), node, _jsonOptions).ConfigureAwait(false); if (result == KeyValueStoreOperationStatus.OK) { // Flags if the keys in _nodesById have been modified by this operation. We will // assume that they have by default, and then set to false if we are doing an @@ -357,7 +366,7 @@ private async ValueTask AddOrUpdateNodeCoreAsync(AssetModelNode node, bool requi if (indexHasChanged) { // "nodes" key contains an array of the defined node IDs. - await _keyValueStore.WriteAsync("nodes", _nodesById.Keys.ToArray()).ConfigureAwait(false); + await _keyValueStore.WriteJsonAsync("nodes", _nodesById.Keys.ToArray(), _jsonOptions).ConfigureAwait(false); await OnConfigurationChangeAsync(node, ConfigurationChangeType.Created, cancellationToken).ConfigureAwait(false); } else { @@ -470,7 +479,7 @@ private async ValueTask DeleteNodeCoreAsync(AssetModelNode node, bool chec _nodesById.TryRemove(node.Id, out _); // "nodes" key contains an array of the defined node IDs. - await _keyValueStore.WriteAsync("nodes", _nodesById.Keys.ToArray()).ConfigureAwait(false); + await _keyValueStore.WriteJsonAsync("nodes", _nodesById.Keys.ToArray(), _jsonOptions).ConfigureAwait(false); await OnConfigurationChangeAsync(node, ConfigurationChangeType.Deleted, cancellationToken).ConfigureAwait(false); diff --git a/src/DataCore.Adapter/RealTimeData/SnapshotTagValueManager.cs b/src/DataCore.Adapter/RealTimeData/SnapshotTagValueManager.cs index 38b5d303..6b95a6e0 100644 --- a/src/DataCore.Adapter/RealTimeData/SnapshotTagValueManager.cs +++ b/src/DataCore.Adapter/RealTimeData/SnapshotTagValueManager.cs @@ -2,9 +2,11 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Runtime.CompilerServices; +using System.Text.Json; using System.Threading; using System.Threading.Tasks; +using DataCore.Adapter.Json; using DataCore.Adapter.Services; using DataCore.Adapter.Tags; @@ -31,6 +33,11 @@ public class SnapshotTagValueManager : SnapshotTagValuePush, IReadSnapshotTagVal /// private readonly IKeyValueStore _keyValueStore; + /// + /// Options for serializing/deserializing tag values. + /// + private readonly JsonSerializerOptions _jsonOptions = new JsonSerializerOptions(); + /// /// Cached tag values, indexed by tag ID. /// @@ -68,6 +75,7 @@ public SnapshotTagValueManager( } _keyValueStore = keyValueStore.CreateScopedStore("snapshot-tag-value-manager:"); + _jsonOptions.AddDataCoreAdapterConverters(); } @@ -80,7 +88,7 @@ public SnapshotTagValueManager( } async ValueTask GetTagId() { - var nameToIdResult = await _keyValueStore.ReadAsync($"name-to-id:{tagNameOrId}").ConfigureAwait(false); + var nameToIdResult = await _keyValueStore.ReadJsonAsync($"name-to-id:{tagNameOrId}", _jsonOptions).ConfigureAwait(false); if (nameToIdResult.Status != KeyValueStoreOperationStatus.OK || string.IsNullOrEmpty(nameToIdResult.Value)) { // Assume that tagNameOrId is the tag ID. return tagNameOrId; @@ -90,7 +98,7 @@ async ValueTask GetTagId() { } var tagId = await GetTagId().ConfigureAwait(false); - var valueResult = await _keyValueStore.ReadAsync($"value:{tagId}").ConfigureAwait(false); + var valueResult = await _keyValueStore.ReadJsonAsync($"value:{tagId}", _jsonOptions).ConfigureAwait(false); if (valueResult.Status == KeyValueStoreOperationStatus.OK && valueResult.Value != null) { // Update lookups. @@ -138,8 +146,8 @@ private async ValueTask ValueReceivedCore(TagValueQueryResult sample, bool _valuesByName[sample.TagId] = sample; _valuesByName[sample.TagName] = sample; - await _keyValueStore.WriteAsync($"value:{sample.TagId}", sample).ConfigureAwait(false); - await _keyValueStore.WriteAsync($"name-to-id:{sample.TagName}", sample.TagId).ConfigureAwait(false); + await _keyValueStore.WriteJsonAsync($"value:{sample.TagId}", sample, _jsonOptions).ConfigureAwait(false); + await _keyValueStore.WriteJsonAsync($"name-to-id:{sample.TagName}", sample.TagId, _jsonOptions).ConfigureAwait(false); } return await base.ValueReceived(sample, cancellationToken).ConfigureAwait(false); diff --git a/src/DataCore.Adapter/Services/KVStoreExtensions.cs b/src/DataCore.Adapter/Services/KVStoreExtensions.cs new file mode 100644 index 00000000..eec4a01c --- /dev/null +++ b/src/DataCore.Adapter/Services/KVStoreExtensions.cs @@ -0,0 +1,78 @@ +using System; +using System.Text.Json; +using System.Threading.Tasks; + +namespace DataCore.Adapter.Services { + + /// + /// Extensions for . + /// + public static class KVStoreExtensions { + + /// + /// Serializes the specified value to JSON and writes it to the store. + /// + /// + /// The value type. + /// + /// + /// The store. + /// + /// + /// The key to write to. + /// + /// + /// The value to write. + /// + /// + /// The to use. + /// + /// + /// A that will return the operation result. + /// + /// + /// is . + /// + public static async ValueTask WriteJsonAsync(this IKeyValueStore store, KVKey key, TValue value, JsonSerializerOptions? options = null) { + if (store == null) { + throw new ArgumentNullException(nameof(store)); + } + + return await store.WriteAsync(key, JsonSerializer.SerializeToUtf8Bytes(value, options)).ConfigureAwait(false); + } + + + /// + /// Reads a JSON byte array from the store and deserializes it to the specified type. + /// + /// + /// The type to deserialize the JSON bytes to. + /// + /// + /// The store. + /// + /// + /// The key to read from. + /// + /// + /// The to use. + /// + /// + /// A that will return the operation result. + /// + /// + /// is . + /// + public static async ValueTask> ReadJsonAsync(this IKeyValueStore store, KVKey key, JsonSerializerOptions? options = null) { + if (store == null) { + throw new ArgumentNullException(nameof(store)); + } + + var result = await store.ReadAsync(key).ConfigureAwait(false); + return result.Status == KeyValueStoreOperationStatus.OK + ? new KeyValueStoreReadResult(result.Status, JsonSerializer.Deserialize(result.Value, options)) + : new KeyValueStoreReadResult(result.Status, default); + } + + } +} diff --git a/src/DataCore.Adapter/Tags/TagManager.cs b/src/DataCore.Adapter/Tags/TagManager.cs index d2151cc2..9b2b2126 100644 --- a/src/DataCore.Adapter/Tags/TagManager.cs +++ b/src/DataCore.Adapter/Tags/TagManager.cs @@ -1,9 +1,9 @@ using System; using System.Collections.Concurrent; using System.Collections.Generic; -using System.Globalization; using System.Linq; using System.Runtime.CompilerServices; +using System.Text.Json; using System.Threading; using System.Threading.Tasks; @@ -50,6 +50,11 @@ public class TagManager : ITagSearch, IDisposable { /// private readonly IKeyValueStore _keyValueStore; + /// + /// Options for serializing/deserializing tags. + /// + private readonly JsonSerializerOptions _jsonOptions = new JsonSerializerOptions(); + /// /// Flags if the class has been initialised. /// @@ -182,7 +187,7 @@ private async Task InitAsyncCore(CancellationToken cancellationToken) { _tagsByName.Clear(); // "tags" key contains an array of the defined tag IDs. - var readResult = await _keyValueStore.ReadAsync("tags").ConfigureAwait(false); + var readResult = await _keyValueStore.ReadJsonAsync("tags", _jsonOptions).ConfigureAwait(false); if (cancellationToken.IsCancellationRequested) { return; } @@ -200,7 +205,7 @@ private async Task InitAsyncCore(CancellationToken cancellationToken) { } // "tags:{id}" key contains the the definition with ID {id}. - var tagReadResult = await _keyValueStore.ReadAsync(string.Concat("tags:", tagId)).ConfigureAwait(false); + var tagReadResult = await _keyValueStore.ReadJsonAsync(string.Concat("tags:", tagId), _jsonOptions).ConfigureAwait(false); if (tagReadResult.Value == null) { continue; } @@ -332,7 +337,7 @@ public async ValueTask AddOrUpdateTagAsync(TagDefinition tag, CancellationToken await _initTask.Value.WithCancellation(cancellationToken).ConfigureAwait(false); // "tags:{id}" key contains the the definition with ID {id}. - var result = await _keyValueStore.WriteAsync(string.Concat("tags:", tag.Id), tag).ConfigureAwait(false); + var result = await _keyValueStore.WriteJsonAsync(string.Concat("tags:", tag.Id), tag, _jsonOptions).ConfigureAwait(false); if (result == KeyValueStoreOperationStatus.OK) { // Check if we are renaming the tag. if (_tagsById.TryGetValue(tag.Id, out var oldTag) && !string.Equals(oldTag.Name, tag.Name, StringComparison.OrdinalIgnoreCase)) { @@ -358,7 +363,7 @@ public async ValueTask AddOrUpdateTagAsync(TagDefinition tag, CancellationToken if (indexHasChanged) { // "tags" key contains an array of the defined tag IDs. - await _keyValueStore.WriteAsync("tags", _tagsById.Keys.ToArray()).ConfigureAwait(false); + await _keyValueStore.WriteJsonAsync("tags", _tagsById.Keys.ToArray(), _jsonOptions).ConfigureAwait(false); await OnConfigurationChangeAsync(tag, ConfigurationChangeType.Created, cancellationToken).ConfigureAwait(false); } else { @@ -412,7 +417,7 @@ public async ValueTask DeleteTagAsync(string tagNameOrId, CancellationToke await OnConfigurationChangeAsync(tag, ConfigurationChangeType.Deleted, cancellationToken).ConfigureAwait(false); // "tags" key contains an array of the defined tag IDs. - await _keyValueStore.WriteAsync("tags", _tagsById.Keys.ToArray()).ConfigureAwait(false); + await _keyValueStore.WriteJsonAsync("tags", _tagsById.Keys.ToArray(), _jsonOptions).ConfigureAwait(false); } return result == KeyValueStoreOperationStatus.OK; diff --git a/test/DataCore.Adapter.Tests/DataCore.Adapter.Tests.csproj b/test/DataCore.Adapter.Tests/DataCore.Adapter.Tests.csproj index b0c73135..9a4798b3 100644 --- a/test/DataCore.Adapter.Tests/DataCore.Adapter.Tests.csproj +++ b/test/DataCore.Adapter.Tests/DataCore.Adapter.Tests.csproj @@ -22,6 +22,8 @@ + + @@ -63,6 +65,10 @@ + + diff --git a/test/DataCore.Adapter.Tests/FasterKeyValueStoreTests.cs b/test/DataCore.Adapter.Tests/FasterKeyValueStoreTests.cs index fbb87bf1..1af0329b 100644 --- a/test/DataCore.Adapter.Tests/FasterKeyValueStoreTests.cs +++ b/test/DataCore.Adapter.Tests/FasterKeyValueStoreTests.cs @@ -32,7 +32,7 @@ public async Task ShouldPersistStateBetweenRestarts() { CheckpointManagerFactory = () => FasterKeyValueStore.CreateLocalStorageCheckpointManager(tmpPath.FullName) })) { - var writeResult = await store1.WriteAsync(TestContext.TestName, now); + var writeResult = await store1.WriteJsonAsync(TestContext.TestName, now); Assert.AreEqual(KeyValueStoreOperationStatus.OK, writeResult); // Checkpoint should be created when we dispose because we have specified a @@ -42,7 +42,7 @@ public async Task ShouldPersistStateBetweenRestarts() { using (var store2 = new FasterKeyValueStore(new FasterKeyValueStoreOptions() { CheckpointManagerFactory = () => FasterKeyValueStore.CreateLocalStorageCheckpointManager(tmpPath.FullName) })) { - var readResult = await store2.ReadAsync(TestContext.TestName); + var readResult = await store2.ReadJsonAsync(TestContext.TestName); Assert.AreEqual(KeyValueStoreOperationStatus.OK, readResult.Status); Assert.AreEqual(now, readResult.Value); } @@ -62,7 +62,7 @@ public async Task ShouldNotCreateCheckpointUnlessDirty() { CheckpointManagerFactory = () => FasterKeyValueStore.CreateLocalStorageCheckpointManager(tmpPath.FullName) })) { - var writeResult = await store.WriteAsync(TestContext.TestName, DateTime.UtcNow); + var writeResult = await store.WriteJsonAsync(TestContext.TestName, DateTime.UtcNow); Assert.AreEqual(KeyValueStoreOperationStatus.OK, writeResult); // Create checkpoint - should succeed @@ -73,7 +73,7 @@ public async Task ShouldNotCreateCheckpointUnlessDirty() { var cp2 = await store.TakeFullCheckpointAsync(); Assert.IsFalse(cp2); - writeResult = await store.WriteAsync(TestContext.TestName, DateTime.UtcNow); + writeResult = await store.WriteJsonAsync(TestContext.TestName, DateTime.UtcNow); Assert.AreEqual(KeyValueStoreOperationStatus.OK, writeResult); // Create a final checkpoint - should succeed diff --git a/test/DataCore.Adapter.Tests/KeyValueFileStoreTests.cs b/test/DataCore.Adapter.Tests/KeyValueFileStoreTests.cs new file mode 100644 index 00000000..9a99a244 --- /dev/null +++ b/test/DataCore.Adapter.Tests/KeyValueFileStoreTests.cs @@ -0,0 +1,67 @@ +using System; +using System.IO; +using System.Threading.Tasks; + +using DataCore.Adapter.KeyValueStore.FileSystem; +using DataCore.Adapter.Services; + +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace DataCore.Adapter.Tests { + + [TestClass] + public class KeyValueFileStoreTests : KeyValueStoreTests { + + private static DirectoryInfo s_baseDirectory; + + + [ClassInitialize] + public static void ClassInit(TestContext context) { + s_baseDirectory = new DirectoryInfo(Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString())); + s_baseDirectory.Create(); + } + + + [ClassCleanup] + public static void ClassCleanup() { + if (s_baseDirectory != null) { + s_baseDirectory.Refresh(); + s_baseDirectory.Delete(true); + } + } + + + private static KeyValueFileStore CreateStore(string baseDirectory) { + return new KeyValueFileStore(new KeyValueFileStoreOptions() { + Path = baseDirectory + }); + } + + + protected override KeyValueFileStore CreateStore() { + return CreateStore(Path.Combine(s_baseDirectory.FullName, Guid.NewGuid().ToString())); + } + + + [TestMethod] + public async Task ShouldShareDataBetweenStores() { + var now = DateTime.UtcNow; + var baseDir = Path.Combine(s_baseDirectory.FullName, Guid.NewGuid().ToString()); + + var store1 = CreateStore(baseDir); + var writeResult = await store1.WriteJsonAsync(TestContext.TestName, now); + + Assert.AreEqual(KeyValueStoreOperationStatus.OK, writeResult); + + var store2 = CreateStore(baseDir); + var readResult = await store2.ReadJsonAsync(TestContext.TestName); + + Assert.AreEqual(KeyValueStoreOperationStatus.OK, readResult.Status); + Assert.AreEqual(now, readResult.Value); + + var tmpPath = new DirectoryInfo(Path.Combine(Path.GetTempPath(), nameof(FasterKeyValueStoreTests), Guid.NewGuid().ToString())); + } + + } + +} diff --git a/test/DataCore.Adapter.Tests/KeyValueStoreTests.cs b/test/DataCore.Adapter.Tests/KeyValueStoreTests.cs index 542d0084..3585eb4c 100644 --- a/test/DataCore.Adapter.Tests/KeyValueStoreTests.cs +++ b/test/DataCore.Adapter.Tests/KeyValueStoreTests.cs @@ -1,7 +1,5 @@ using System; -using System.Collections.Generic; using System.Linq; -using System.Text; using System.Threading.Tasks; using DataCore.Adapter.Services; @@ -10,7 +8,7 @@ namespace DataCore.Adapter.Tests { - public abstract class KeyValueStoreTests : TestsBase where T : IKeyValueStore, IDisposable { + public abstract class KeyValueStoreTests : TestsBase where T : IKeyValueStore { protected abstract T CreateStore(); @@ -19,10 +17,16 @@ public abstract class KeyValueStoreTests : TestsBase where T : IKeyValueStore public async Task ShouldWriteValueToStore() { var now = DateTime.UtcNow; - using (var store = CreateStore()) { - var result = await store.WriteAsync(TestContext.TestName, now); + var store = CreateStore(); + try { + var result = await store.WriteJsonAsync(TestContext.TestName, now); Assert.AreEqual(KeyValueStoreOperationStatus.OK, result); } + finally { + if (store is IDisposable disposable) { + disposable.Dispose(); + } + } } @@ -30,14 +34,20 @@ public async Task ShouldWriteValueToStore() { public async Task ShouldReadValueFromStore() { var now = DateTime.UtcNow; - using (var store = CreateStore()) { - var result = await store.WriteAsync(TestContext.TestName, now); + var store = CreateStore(); + try { + var result = await store.WriteJsonAsync(TestContext.TestName, now); Assert.AreEqual(KeyValueStoreOperationStatus.OK, result); - var value = await store.ReadAsync(TestContext.TestName); + var value = await store.ReadJsonAsync(TestContext.TestName); Assert.AreEqual(KeyValueStoreOperationStatus.OK, value.Status); Assert.AreEqual(now, value.Value); } + finally { + if (store is IDisposable disposable) { + disposable.Dispose(); + } + } } @@ -45,30 +55,37 @@ public async Task ShouldReadValueFromStore() { public async Task ShouldDeleteValueFromStore() { var now = DateTime.UtcNow; - using (var store = CreateStore()) { - var result = await store.WriteAsync(TestContext.TestName, now); + var store = CreateStore(); + try { + var result = await store.WriteJsonAsync(TestContext.TestName, now); Assert.AreEqual(KeyValueStoreOperationStatus.OK, result); - var value = await store.ReadAsync(TestContext.TestName); + var value = await store.ReadJsonAsync(TestContext.TestName); Assert.AreEqual(KeyValueStoreOperationStatus.OK, value.Status); Assert.AreEqual(now, value.Value); var delete = await store.DeleteAsync(TestContext.TestName); Assert.AreEqual(KeyValueStoreOperationStatus.OK, delete); - var value2 = await store.ReadAsync(TestContext.TestName); + var value2 = await store.ReadJsonAsync(TestContext.TestName); Assert.AreEqual(KeyValueStoreOperationStatus.NotFound, value2.Status); Assert.AreEqual(default(DateTime), value2.Value); } + finally { + if (store is IDisposable disposable) { + disposable.Dispose(); + } + } } [TestMethod] public async Task ShouldListKeys() { - using (var store = CreateStore()) { + var store = CreateStore(); + try { var keys = Enumerable.Range(1, 10).Select(x => $"key:{x}").ToArray(); foreach (var key in keys) { - var result = await store.WriteAsync(key, 0); + var result = await store.WriteJsonAsync(key, 0); Assert.AreEqual(KeyValueStoreOperationStatus.OK, result); } @@ -79,6 +96,11 @@ public async Task ShouldListKeys() { Assert.IsTrue(keysActual.Contains(key), $"Missing key: {key}"); } } + finally { + if (store is IDisposable disposable) { + disposable.Dispose(); + } + } } @@ -86,20 +108,28 @@ public async Task ShouldListKeys() { public async Task ScopedStoreShouldAddKeyPrefix() { var now = DateTime.UtcNow; - using (var store = CreateStore()) { - var scopedStore = store.CreateScopedStore("INNER:"); + var store = CreateStore(); + try { + var scopedStore = store + .CreateScopedStore("INNER 1:") + .CreateScopedStore("INNER 2:"); - var result = await scopedStore.WriteAsync(TestContext.TestName, now); + var result = await scopedStore.WriteJsonAsync(TestContext.TestName, now); Assert.AreEqual(KeyValueStoreOperationStatus.OK, result); - var value = await scopedStore.ReadAsync(TestContext.TestName); + var value = await scopedStore.ReadJsonAsync(TestContext.TestName); Assert.AreEqual(KeyValueStoreOperationStatus.OK, value.Status); Assert.AreEqual(now, value.Value); - var value2 = await store.ReadAsync("INNER:" + TestContext.TestName); + var value2 = await store.ReadJsonAsync("INNER 1:INNER 2:" + TestContext.TestName); Assert.AreEqual(KeyValueStoreOperationStatus.OK, value2.Status); Assert.AreEqual(now, value2.Value); } + finally { + if (store is IDisposable disposable) { + disposable.Dispose(); + } + } } @@ -107,15 +137,21 @@ public async Task ScopedStoreShouldAddKeyPrefix() { public async Task ScopedStoreShouldRemoveKeyPrefix() { var now = DateTime.UtcNow; - using (var store = CreateStore()) { + var store = CreateStore(); + try { var scopedStore = store.CreateScopedStore("INNER:"); - var result = await scopedStore.WriteAsync(TestContext.TestName, now); + var result = await scopedStore.WriteJsonAsync(TestContext.TestName, now); Assert.AreEqual(KeyValueStoreOperationStatus.OK, result); var keys = scopedStore.GetKeysAsStrings().ToArray(); Assert.IsTrue(keys.Contains(TestContext.TestName)); } + finally { + if (store is IDisposable disposable) { + disposable.Dispose(); + } + } } } diff --git a/test/DataCore.Adapter.Tests/SqliteKeyValueStoreTests.cs b/test/DataCore.Adapter.Tests/SqliteKeyValueStoreTests.cs new file mode 100644 index 00000000..73cc0087 --- /dev/null +++ b/test/DataCore.Adapter.Tests/SqliteKeyValueStoreTests.cs @@ -0,0 +1,72 @@ +using System; +using System.IO; +using System.Threading.Tasks; + +using DataCore.Adapter.KeyValueStore.Sqlite; +using DataCore.Adapter.Services; + +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace DataCore.Adapter.Tests { + + [TestClass] + public class SqliteKeyValueStoreTests : KeyValueStoreTests { + + private static DirectoryInfo s_baseDirectory; + + + [ClassInitialize] + public static void ClassInit(TestContext context) { + s_baseDirectory = new DirectoryInfo(Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString())); + s_baseDirectory.Create(); + } + + + [ClassCleanup] + public static void ClassCleanup() { + if (s_baseDirectory != null) { + s_baseDirectory.Refresh(); + s_baseDirectory.Delete(true); + } + } + + + private static SqliteKeyValueStore CreateStore(string fileName) { + return new SqliteKeyValueStore(new SqliteKeyValueStoreOptions() { + ConnectionString = $"Data Source={fileName};Cache=Shared" + }); + } + + + private static string GetDatabaseFileName() { + return Path.Combine(s_baseDirectory.FullName, Guid.NewGuid().ToString()); + } + + + protected override SqliteKeyValueStore CreateStore() { + return CreateStore(GetDatabaseFileName()); + } + + + [TestMethod] + public async Task ShouldShareDataBetweenStores() { + var now = DateTime.UtcNow; + var path = GetDatabaseFileName(); + + var store1 = CreateStore(path); + var writeResult = await store1.WriteJsonAsync(TestContext.TestName, now); + + Assert.AreEqual(KeyValueStoreOperationStatus.OK, writeResult); + + var store2 = CreateStore(path); + var readResult = await store2.ReadJsonAsync(TestContext.TestName); + + Assert.AreEqual(KeyValueStoreOperationStatus.OK, readResult.Status); + Assert.AreEqual(now, readResult.Value); + + var tmpPath = new DirectoryInfo(Path.Combine(Path.GetTempPath(), nameof(FasterKeyValueStoreTests), Guid.NewGuid().ToString())); + } + + } + +} From 2229a8cd1609eb426b2d88758647fc49c6930281 Mon Sep 17 00:00:00 2001 From: Graham Watts Date: Fri, 7 Jan 2022 14:17:37 +0200 Subject: [PATCH 2/6] Rename file system store --- ...KeyValueFileStore.cs => FileSystemKeyValueStore.cs} | 10 +++++----- ...oreOptions.cs => FileSystemKeyValueStoreOptions.cs} | 4 ++-- ...leStoreTests.cs => FileSystemKeyValueStoreTests.cs} | 8 ++++---- 3 files changed, 11 insertions(+), 11 deletions(-) rename src/DataCore.Adapter.KeyValueStore.FileSystem/{KeyValueFileStore.cs => FileSystemKeyValueStore.cs} (96%) rename src/DataCore.Adapter.KeyValueStore.FileSystem/{KeyValueFileStoreOptions.cs => FileSystemKeyValueStoreOptions.cs} (84%) rename test/DataCore.Adapter.Tests/{KeyValueFileStoreTests.cs => FileSystemKeyValueStoreTests.cs} (84%) diff --git a/src/DataCore.Adapter.KeyValueStore.FileSystem/KeyValueFileStore.cs b/src/DataCore.Adapter.KeyValueStore.FileSystem/FileSystemKeyValueStore.cs similarity index 96% rename from src/DataCore.Adapter.KeyValueStore.FileSystem/KeyValueFileStore.cs rename to src/DataCore.Adapter.KeyValueStore.FileSystem/FileSystemKeyValueStore.cs index 8a84440a..f71807b3 100644 --- a/src/DataCore.Adapter.KeyValueStore.FileSystem/KeyValueFileStore.cs +++ b/src/DataCore.Adapter.KeyValueStore.FileSystem/FileSystemKeyValueStore.cs @@ -16,7 +16,7 @@ namespace DataCore.Adapter.KeyValueStore.FileSystem { /// /// implementation that persists files to disk. /// - public class KeyValueFileStore : Services.KeyValueStore { + public class FileSystemKeyValueStore : Services.KeyValueStore { /// @@ -36,10 +36,10 @@ public class KeyValueFileStore : Services.KeyValueStore { /// - /// Creates a new object. + /// Creates a new object. /// /// - /// The for the store. + /// The for the store. /// /// /// The logger for the store. @@ -47,7 +47,7 @@ public class KeyValueFileStore : Services.KeyValueStore { /// /// is . /// - public KeyValueFileStore(KeyValueFileStoreOptions options, ILogger? logger = null) : base() { + public FileSystemKeyValueStore(FileSystemKeyValueStoreOptions options, ILogger? logger = null) : base() { if (options == null) { throw new ArgumentNullException(nameof(options)); } @@ -55,7 +55,7 @@ public KeyValueFileStore(KeyValueFileStoreOptions options, ILogger - /// Options for . + /// Options for . /// - public class KeyValueFileStoreOptions { + public class FileSystemKeyValueStoreOptions { /// /// Default path to save files to. diff --git a/test/DataCore.Adapter.Tests/KeyValueFileStoreTests.cs b/test/DataCore.Adapter.Tests/FileSystemKeyValueStoreTests.cs similarity index 84% rename from test/DataCore.Adapter.Tests/KeyValueFileStoreTests.cs rename to test/DataCore.Adapter.Tests/FileSystemKeyValueStoreTests.cs index 9a99a244..e5a477a8 100644 --- a/test/DataCore.Adapter.Tests/KeyValueFileStoreTests.cs +++ b/test/DataCore.Adapter.Tests/FileSystemKeyValueStoreTests.cs @@ -10,7 +10,7 @@ namespace DataCore.Adapter.Tests { [TestClass] - public class KeyValueFileStoreTests : KeyValueStoreTests { + public class FileSystemKeyValueStoreTests : KeyValueStoreTests { private static DirectoryInfo s_baseDirectory; @@ -31,14 +31,14 @@ public static void ClassCleanup() { } - private static KeyValueFileStore CreateStore(string baseDirectory) { - return new KeyValueFileStore(new KeyValueFileStoreOptions() { + private static FileSystemKeyValueStore CreateStore(string baseDirectory) { + return new FileSystemKeyValueStore(new FileSystemKeyValueStoreOptions() { Path = baseDirectory }); } - protected override KeyValueFileStore CreateStore() { + protected override FileSystemKeyValueStore CreateStore() { return CreateStore(Path.Combine(s_baseDirectory.FullName, Guid.NewGuid().ToString())); } From 9db9ca20d5e1f849767121c7fa653fb402014d81 Mon Sep 17 00:00:00 2001 From: Graham Watts Date: Fri, 7 Jan 2022 14:18:01 +0200 Subject: [PATCH 3/6] Enable write-ahead logging --- .../SqliteKeyValueStore.cs | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/src/DataCore.Adapter.KeyValueStore.Sqlite/SqliteKeyValueStore.cs b/src/DataCore.Adapter.KeyValueStore.Sqlite/SqliteKeyValueStore.cs index dd4f027c..44a208ba 100644 --- a/src/DataCore.Adapter.KeyValueStore.Sqlite/SqliteKeyValueStore.cs +++ b/src/DataCore.Adapter.KeyValueStore.Sqlite/SqliteKeyValueStore.cs @@ -54,6 +54,21 @@ public SqliteKeyValueStore(SqliteKeyValueStoreOptions options, ILogger + /// Enables write-ahead logging on a Sqlite connection. + /// + /// + /// A command that will execute against the connection to modify. + /// + /// + /// See here for further details. + /// + private void EnableWriteAheadLogging(SqliteCommand command) { + command.CommandText = "PRAGMA journal_mode='wal'"; + command.ExecuteNonQuery(); + } + + /// /// Creates the key-value table in the SQlite database. /// @@ -61,9 +76,15 @@ private void CreateKVTable() { using (var connection = new SqliteConnection(_connectionString)) { connection.Open(); + using (var transaction = connection.BeginTransaction()) using (var command = connection.CreateCommand()) { - command.CommandText = @"CREATE TABLE IF NOT EXISTS kvstore (key TEXT PRIMARY KEY, value BLOB)"; + command.Transaction = transaction; + + EnableWriteAheadLogging(command); + + command.CommandText = "CREATE TABLE IF NOT EXISTS kvstore (key TEXT PRIMARY KEY, value BLOB)"; command.ExecuteNonQuery(); + transaction.Commit(); } } } @@ -81,6 +102,7 @@ protected override ValueTask WriteAsync(KVKey key, using (var command = connection.CreateCommand()) { command.Transaction = transaction; + // TODO: Consider if BLOB I/O is more appropriate here: https://docs.microsoft.com/en-us/dotnet/standard/data/sqlite/blob-io command.CommandText = "INSERT INTO kvstore (key, value) VALUES ($key, $value) ON CONFLICT (key) DO UPDATE SET value = $value"; command.Parameters.AddWithValue("$key", hexKey); command.Parameters.AddWithValue("$value", value); From a5f17d34e0b21db64c534a1610f2126c41d67d08 Mon Sep 17 00:00:00 2001 From: Graham Watts Date: Fri, 7 Jan 2022 14:18:11 +0200 Subject: [PATCH 4/6] Create README.md --- .../README.md | 70 ++++++++++++++++++ .../README.md | 71 +++++++++++++++++++ 2 files changed, 141 insertions(+) create mode 100644 src/DataCore.Adapter.KeyValueStore.FileSystem/README.md create mode 100644 src/DataCore.Adapter.KeyValueStore.Sqlite/README.md diff --git a/src/DataCore.Adapter.KeyValueStore.FileSystem/README.md b/src/DataCore.Adapter.KeyValueStore.FileSystem/README.md new file mode 100644 index 00000000..f57eeae7 --- /dev/null +++ b/src/DataCore.Adapter.KeyValueStore.FileSystem/README.md @@ -0,0 +1,70 @@ +# DataCore.Adapter.KeyValueStore.FileSystem + +Implementation of [IKeyValueStore](/src/DataCore.Adapter.Abstractions/Services/IKeyValueStore.cs) that uses the file system to persist data. + +Each key in the store is saved to a different file. The name of the fix is the key, converted to hexadecimal and suffixed with a file extension e.g. `[0x48, 0x65,0x6C, 0x6C]` becomes `48656C6C.data`. + + +# Example Usage + +Register `FileSystemKeyValueStore` with the `IAdapterConfigurationBuilder` when configuring adapter services in the dependency injection container: + +```csharp +public void ConfigureAdapters(IServiceCollection services) { + services + .AddDataCoreAdapterAspNetCoreServices() + .AddHostInfo(HostInfo.Create( + "My Host", + "A brief description of the hosting application", + "0.9.0-alpha", // SemVer v2 + VendorInfo.Create("Intelligent Plant", "https://appstore.intelligentplant.com"), + AdapterProperty.Create("Project URL", "https://github.com/intelligentplant/AppStoreConnect.Adapters") + )) + .AddKeyValueStore(sp => { + var options = new FileSystemKeyValueStoreOptions() { + Path = Path.Combine(AppContext.BaseDirectory, "Data", "KVStore") + }; + + return ActivatorUtilities.CreateInstance(sp, options); + }) + .AddAdapter(sp => { + const string adapterId = "my-adapter-1"; + // Assume that MyAdapter's constructor expects an IKeyValueStore instance. + return ActivatorUtilities.CreateInstance( + sp, + adapterId, + new MyAdapterOptions() + ); + }); +} +``` + +The `IKeyValueStore` can then be injected into an adapter in the same way as any other service. If you are hosting multiple adapters in the same application, or the `IKeyValueStore` will be used by other parts of your application, it is recommended to pass a scoped `IKeyValueStore` when creating the adapter, to ensure that all items in the store associated with the adapter use a common prefix: + +```csharp +public void ConfigureAdapters(IServiceCollection services) { + services + .AddDataCoreAdapterAspNetCoreServices() + .AddHostInfo(HostInfo.Create( + "My Host", + "A brief description of the hosting application", + "0.9.0-alpha", // SemVer v2 + VendorInfo.Create("Intelligent Plant", "https://appstore.intelligentplant.com"), + AdapterProperty.Create("Project URL", "https://github.com/intelligentplant/AppStoreConnect.Adapters") + )) + .AddKeyValueStore(sp => { + // Configuration removed for brevity + }) + .AddAdapter(sp => { + const string adapterId = "my-adapter-1"; + var kvStore = sp.GetRequiredService(); + + return ActivatorUtilities.CreateInstance( + sp, + adapterId, + new MyAdapterOptions(), + kvStore.CreateScopedStore(adapterId) + ); + }); +} +``` diff --git a/src/DataCore.Adapter.KeyValueStore.Sqlite/README.md b/src/DataCore.Adapter.KeyValueStore.Sqlite/README.md new file mode 100644 index 00000000..73acdb26 --- /dev/null +++ b/src/DataCore.Adapter.KeyValueStore.Sqlite/README.md @@ -0,0 +1,71 @@ +# DataCore.Adapter.KeyValueStore.Sqlite + +Implementation of [IKeyValueStore](/src/DataCore.Adapter.Abstractions/Services/IKeyValueStore.cs) that uses a Sqlite database to persist data. + +All key-value pairs are saved to a table in the database named `kvstore`. The table is created if it does not already exist. + + +# Example Usage + +Register `SqliteKeyValueStore` with the `IAdapterConfigurationBuilder` when configuring adapter services in the dependency injection container: + +```csharp +public void ConfigureAdapters(IServiceCollection services) { + services + .AddDataCoreAdapterAspNetCoreServices() + .AddHostInfo(HostInfo.Create( + "My Host", + "A brief description of the hosting application", + "0.9.0-alpha", // SemVer v2 + VendorInfo.Create("Intelligent Plant", "https://appstore.intelligentplant.com"), + AdapterProperty.Create("Project URL", "https://github.com/intelligentplant/AppStoreConnect.Adapters") + )) + .AddKeyValueStore(sp => { + var path = Path.Combine(AppContext.BaseDirectory, "Data", "kvstore.db"); + var options = new SqliteKeyValueStoreOptions() { + ConnectionString = $"Data Source={path};Cache=Shared" + }; + + return ActivatorUtilities.CreateInstance(sp, options); + }) + .AddAdapter(sp => { + const string adapterId = "my-adapter-1"; + // Assume that MyAdapter's constructor expects an IKeyValueStore instance. + return ActivatorUtilities.CreateInstance( + sp, + adapterId, + new MyAdapterOptions() + ); + }); +} +``` + +The `IKeyValueStore` can then be injected into an adapter in the same way as any other service. If you are hosting multiple adapters in the same application, or the `IKeyValueStore` will be used by other parts of your application, it is recommended to pass a scoped `IKeyValueStore` when creating the adapter, to ensure that all items in the store associated with the adapter use a common prefix: + +```csharp +public void ConfigureAdapters(IServiceCollection services) { + services + .AddDataCoreAdapterAspNetCoreServices() + .AddHostInfo(HostInfo.Create( + "My Host", + "A brief description of the hosting application", + "0.9.0-alpha", // SemVer v2 + VendorInfo.Create("Intelligent Plant", "https://appstore.intelligentplant.com"), + AdapterProperty.Create("Project URL", "https://github.com/intelligentplant/AppStoreConnect.Adapters") + )) + .AddKeyValueStore(sp => { + // Configuration removed for brevity + }) + .AddAdapter(sp => { + const string adapterId = "my-adapter-1"; + var kvStore = sp.GetRequiredService(); + + return ActivatorUtilities.CreateInstance( + sp, + adapterId, + new MyAdapterOptions(), + kvStore.CreateScopedStore(adapterId) + ); + }); +} +``` From 3010e381311eb6b484de5add77f36b17f2e51058 Mon Sep 17 00:00:00 2001 From: Graham Watts Date: Fri, 7 Jan 2022 14:34:00 +0200 Subject: [PATCH 5/6] Enable write-ahead logging if database is created --- .../SqliteKeyValueStore.cs | 50 +++++++++++++++++-- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/src/DataCore.Adapter.KeyValueStore.Sqlite/SqliteKeyValueStore.cs b/src/DataCore.Adapter.KeyValueStore.Sqlite/SqliteKeyValueStore.cs index 44a208ba..abc08114 100644 --- a/src/DataCore.Adapter.KeyValueStore.Sqlite/SqliteKeyValueStore.cs +++ b/src/DataCore.Adapter.KeyValueStore.Sqlite/SqliteKeyValueStore.cs @@ -16,6 +16,11 @@ namespace DataCore.Adapter.KeyValueStore.Sqlite { /// public class SqliteKeyValueStore : Services.KeyValueStore { + /// + /// Sqlite error code when the database file is unavailable. + /// + private const int SQLITE_CANTOPEN = 14; + /// /// The Sqlite connection string. /// @@ -69,22 +74,57 @@ private void EnableWriteAheadLogging(SqliteCommand command) { } + /// + /// Tests if the Sqlite database already exists. + /// + /// + /// The connection string. + /// + /// + /// A flag indicating if the database already exists. + /// + private bool DatabaseExists(string connectionString) { + // See here: https://github.com/dotnet/efcore/blob/c918248457a3629736ff50c970ac022917b894b1/src/EFCore.Sqlite.Core/Storage/Internal/SqliteDatabaseCreator.cs#L66 + + var connectionOptions = new SqliteConnectionStringBuilder(connectionString); + if (connectionOptions.DataSource.Equals(":memory:", StringComparison.OrdinalIgnoreCase) || connectionOptions.Mode == SqliteOpenMode.Memory) { + return true; + } + + var connectionStringBuilder = new SqliteConnectionStringBuilder(connectionString) { + Mode = SqliteOpenMode.ReadOnly, + Pooling = false + }; + + using (var readOnlyConnection = new SqliteConnection(connectionStringBuilder.ToString())) { + try { + readOnlyConnection.Open(); + } + catch (SqliteException ex) when (ex.SqliteErrorCode == SQLITE_CANTOPEN) { + return false; + } + } + + return true; + } + + /// /// Creates the key-value table in the SQlite database. /// private void CreateKVTable() { + var createDatabase = !DatabaseExists(_connectionString); + using (var connection = new SqliteConnection(_connectionString)) { connection.Open(); - using (var transaction = connection.BeginTransaction()) using (var command = connection.CreateCommand()) { - command.Transaction = transaction; - - EnableWriteAheadLogging(command); + if (createDatabase) { + EnableWriteAheadLogging(command); + } command.CommandText = "CREATE TABLE IF NOT EXISTS kvstore (key TEXT PRIMARY KEY, value BLOB)"; command.ExecuteNonQuery(); - transaction.Commit(); } } } From 74de8361e1df01a6ebdef0589141a22c676b24d8 Mon Sep 17 00:00:00 2001 From: Graham Watts Date: Fri, 7 Jan 2022 14:34:03 +0200 Subject: [PATCH 6/6] Update THIRD_PARTY_LICENSES.md --- THIRD_PARTY_LICENSES.md | 1 + 1 file changed, 1 insertion(+) diff --git a/THIRD_PARTY_LICENSES.md b/THIRD_PARTY_LICENSES.md index d9243add..82f798c2 100644 --- a/THIRD_PARTY_LICENSES.md +++ b/THIRD_PARTY_LICENSES.md @@ -1,6 +1,7 @@ # .NET Foundation Some code in this repository is based on the [.NET Runtime](https://github.com/dotnet/runtime). +Some code in this repository is based on [Entity Framework Core](https://github.com/dotnet/efcore). The MIT License (MIT)