Skip to content

Commit

Permalink
Merge pull request #174 from intelligentplant/feature/kvstore-refactor
Browse files Browse the repository at this point in the history
Refactor key-value store and add Sqlite and file system implementations
  • Loading branch information
wazzamatazz authored Jan 7, 2022
2 parents 84bc2d3 + 74de836 commit e66428f
Show file tree
Hide file tree
Showing 39 changed files with 2,193 additions and 1,553 deletions.
14 changes: 14 additions & 0 deletions DataCore.Adapter.sln
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ExampleHostedAdapter", "exa
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DataCore.Adapter.KeyValueStore.FASTER", "src\DataCore.Adapter.KeyValueStore.FASTER\DataCore.Adapter.KeyValueStore.FASTER.csproj", "{E2C09B9E-FCE1-4C61-A547-16ADFA76B7F6}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DataCore.Adapter.KeyValueStore.Sqlite", "src\DataCore.Adapter.KeyValueStore.Sqlite\DataCore.Adapter.KeyValueStore.Sqlite.csproj", "{22DD3CC9-034B-4C43-A143-6240E8E833CC}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DataCore.Adapter.KeyValueStore.FileSystem", "src\DataCore.Adapter.KeyValueStore.FileSystem\DataCore.Adapter.KeyValueStore.FileSystem.csproj", "{8C225A69-794C-4451-B5CE-91EAD5269C87}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand Down Expand Up @@ -269,6 +273,14 @@ Global
{E2C09B9E-FCE1-4C61-A547-16ADFA76B7F6}.Debug|Any CPU.Build.0 = Debug|Any CPU
{E2C09B9E-FCE1-4C61-A547-16ADFA76B7F6}.Release|Any CPU.ActiveCfg = Release|Any CPU
{E2C09B9E-FCE1-4C61-A547-16ADFA76B7F6}.Release|Any CPU.Build.0 = Release|Any CPU
{22DD3CC9-034B-4C43-A143-6240E8E833CC}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{22DD3CC9-034B-4C43-A143-6240E8E833CC}.Debug|Any CPU.Build.0 = Debug|Any CPU
{22DD3CC9-034B-4C43-A143-6240E8E833CC}.Release|Any CPU.ActiveCfg = Release|Any CPU
{22DD3CC9-034B-4C43-A143-6240E8E833CC}.Release|Any CPU.Build.0 = Release|Any CPU
{8C225A69-794C-4451-B5CE-91EAD5269C87}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{8C225A69-794C-4451-B5CE-91EAD5269C87}.Debug|Any CPU.Build.0 = Debug|Any CPU
{8C225A69-794C-4451-B5CE-91EAD5269C87}.Release|Any CPU.ActiveCfg = Release|Any CPU
{8C225A69-794C-4451-B5CE-91EAD5269C87}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand Down Expand Up @@ -314,6 +326,8 @@ Global
{B5692AC0-AA11-4062-8F65-8A120C8846FA} = {7701E410-A9F0-4F50-80E4-51576DABB306}
{CA874402-DD93-40C0-893A-BC8D1D961A43} = {92FD0A48-0AE7-44A6-86BC-C4E47E014C17}
{E2C09B9E-FCE1-4C61-A547-16ADFA76B7F6} = {B158078F-D217-46FE-A9B8-7BD1B99922BA}
{22DD3CC9-034B-4C43-A143-6240E8E833CC} = {B158078F-D217-46FE-A9B8-7BD1B99922BA}
{8C225A69-794C-4451-B5CE-91EAD5269C87} = {B158078F-D217-46FE-A9B8-7BD1B99922BA}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {323415F4-D394-4EAA-B259-7C6834BFD819}
Expand Down
1 change: 1 addition & 0 deletions THIRD_PARTY_LICENSES.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
# .NET Foundation

Some code in this repository is based on the [.NET Runtime](https://github.com/dotnet/runtime).
Some code in this repository is based on [Entity Framework Core](https://github.com/dotnet/efcore).

The MIT License (MIT)

Expand Down
2 changes: 2 additions & 0 deletions build/Dependencies.props
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
<MicrosoftAspNetCoreSignalRVersion>1.1.0</MicrosoftAspNetCoreSignalRVersion>
<MicrosoftAspNetWebApiClientVersion>5.2.7</MicrosoftAspNetWebApiClientVersion>
<MicrosoftBclAsyncInterfacesVersion>6.0.0</MicrosoftBclAsyncInterfacesVersion>
<MicrosoftDataSqliteVersion>6.0.1</MicrosoftDataSqliteVersion>
<!--
Microsoft.Extensions.Hosting and Microsoft.Extensions.Http are referenced by the
DataCore.Adapter.Tests.Helpers project for .NET Standard 2.0 targeting.
Expand Down Expand Up @@ -52,6 +53,7 @@
<IntelligentPlantBackgroundTasksVersion>8.0.1</IntelligentPlantBackgroundTasksVersion>
<JaahasHttpRequestTransformerVersion>2.2.0</JaahasHttpRequestTransformerVersion>
<NewtonsoftJsonVersion>12.0.3</NewtonsoftJsonVersion>
<NitoAsyncExCoordinationVersion>5.1.2</NitoAsyncExCoordinationVersion>
<NSwagAspNetCoreVersion>13.15.3</NSwagAspNetCoreVersion>
<NuGetVersioningVersion>5.8.1</NuGetVersioningVersion>
<OpenTelemetryApiVersion>1.1.0</OpenTelemetryApiVersion>
Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 3 additions & 0 deletions src/DataCore.Adapter.Abstractions/AbstractionsResources.resx
Original file line number Diff line number Diff line change
Expand Up @@ -333,6 +333,9 @@
<data name="DisplayName_WriteTagValueAnnotations" xml:space="preserve">
<value>Write Annotations</value>
</data>
<data name="Error_KeyValueStore_InvalidKey" xml:space="preserve">
<value>Zero byte keys are not allowed.</value>
</data>
<data name="Error_MissingAdapterFeature" xml:space="preserve">
<value>The {0} feature is not implemented by the adapter.</value>
<comment>{0} - IAdapterFeature name</comment>
Expand Down
29 changes: 18 additions & 11 deletions src/DataCore.Adapter.Abstractions/Services/IKeyValueStore.cs
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,20 @@ namespace DataCore.Adapter.Services {
/// A service that can be used to store arbitrary key-value pairs.
/// </summary>
/// <remarks>
///
/// <para>
/// <see cref="IKeyValueStore"/> is intended to allow an adapter to store arbitrary
/// key-value data that can be persisted between restarts of the adapter or its host
/// application.
/// </para>
///
/// <para>
/// Implementations should extend <see cref="KeyValueStore"/> rather than implementing
/// <see cref="IKeyValueStore"/> directly.
/// </para>
///
/// </remarks>
/// <seealso cref="KeyValueStore"/>
/// <seealso cref="InMemoryKeyValueStore"/>
/// <seealso cref="ScopedKeyValueStore"/>
/// <seealso cref="KeyValueStoreExtensions"/>
Expand All @@ -19,9 +29,6 @@ public interface IKeyValueStore {
/// <summary>
/// Writes a value to the store.
/// </summary>
/// <typeparam name="TValue">
/// The value type.
/// </typeparam>
/// <param name="key">
/// The key for the value.
/// </param>
Expand All @@ -31,23 +38,20 @@ public interface IKeyValueStore {
/// <returns>
/// A <see cref="ValueTask{TResult}"/> that will return the status of the operation.
/// </returns>
ValueTask<KeyValueStoreOperationStatus> WriteAsync<TValue>(byte[] key, TValue? value);
ValueTask<KeyValueStoreOperationStatus> WriteAsync(KVKey key, byte[] value);


/// <summary>
/// Reads a value from the store.
/// </summary>
/// <typeparam name="TValue">
/// The value type.
/// </typeparam>
/// <param name="key">
/// The key for the value.
/// </param>
/// <returns>
/// A <see cref="ValueTask{TResult}"/> that will return a <see cref="KeyValueStoreReadResult{T}"/>
/// A <see cref="ValueTask{TResult}"/> that will return a <see cref="KeyValueStoreReadResult"/>
/// containing the operation status and value.
/// </returns>
ValueTask<KeyValueStoreReadResult<TValue>> ReadAsync<TValue>(byte[] key);
ValueTask<KeyValueStoreReadResult> ReadAsync(KVKey key);


/// <summary>
Expand All @@ -59,16 +63,19 @@ public interface IKeyValueStore {
/// <returns>
/// A <see cref="ValueTask{TResult}"/> that will return the status of the operation.
/// </returns>
ValueTask<KeyValueStoreOperationStatus> DeleteAsync(byte[] key);
ValueTask<KeyValueStoreOperationStatus> DeleteAsync(KVKey key);


/// <summary>
/// Gets the keys that are defined in the store.
/// </summary>
/// <param name="prefix">
/// Only keys beginning with this prefix will be returned.
/// </param>
/// <returns>
/// The keys.
/// </returns>
IEnumerable<byte[]> GetKeys();
IEnumerable<KVKey> GetKeys(KVKey? prefix);

}
}
47 changes: 23 additions & 24 deletions src/DataCore.Adapter.Abstractions/Services/InMemoryKeyValueStore.cs
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Text;
using System.Threading.Tasks;

namespace DataCore.Adapter.Services {
Expand All @@ -13,47 +12,42 @@ namespace DataCore.Adapter.Services {
/// <remarks>
/// This implementation does not provide any persistence of data.
/// </remarks>
public sealed class InMemoryKeyValueStore : IKeyValueStore, IDisposable {
public sealed class InMemoryKeyValueStore : KeyValueStore, IDisposable {

/// <summary>
/// The value store.
/// </summary>
private readonly ConcurrentDictionary<string, object?> _values = new ConcurrentDictionary<string, object?>(StringComparer.Ordinal);
private readonly ConcurrentDictionary<string, byte[]> _values = new ConcurrentDictionary<string, byte[]>(StringComparer.Ordinal);


/// <summary>
/// Creats a new <see cref="InMemoryKeyValueStore"/> object.
/// </summary>
public InMemoryKeyValueStore() : base(default) { }


/// <inheritdoc/>
public ValueTask<KeyValueStoreOperationStatus> WriteAsync<TValue>(byte[] key, TValue? value) {
if (key == null) {
throw new ArgumentNullException(nameof(key));
}
var keyAsString = Encoding.UTF8.GetString(key);
protected override ValueTask<KeyValueStoreOperationStatus> WriteAsync(KVKey key, byte[] value) {
var keyAsString = ConvertBytesToHexString(key);
_values[keyAsString] = value;
return new ValueTask<KeyValueStoreOperationStatus>(KeyValueStoreOperationStatus.OK);
}


/// <inheritdoc/>
public ValueTask<KeyValueStoreReadResult<TValue>> ReadAsync<TValue>(byte[] key) {
if (key == null) {
throw new ArgumentNullException(nameof(key));
}

var keyAsString = Encoding.UTF8.GetString(key);
protected override ValueTask<KeyValueStoreReadResult> ReadAsync(KVKey key) {
var keyAsString = ConvertBytesToHexString(key);
if (!_values.TryGetValue(keyAsString, out var value)) {
return new ValueTask<KeyValueStoreReadResult<TValue>>(new KeyValueStoreReadResult<TValue>(KeyValueStoreOperationStatus.NotFound, default));
return new ValueTask<KeyValueStoreReadResult>(new KeyValueStoreReadResult(KeyValueStoreOperationStatus.NotFound, default));
}

return new ValueTask<KeyValueStoreReadResult<TValue>>(new KeyValueStoreReadResult<TValue>(KeyValueStoreOperationStatus.OK, (TValue?) value));
return new ValueTask<KeyValueStoreReadResult>(new KeyValueStoreReadResult(KeyValueStoreOperationStatus.OK, value));
}


/// <inheritdoc/>
public ValueTask<KeyValueStoreOperationStatus> DeleteAsync(byte[] key) {
if (key == null) {
throw new ArgumentNullException(nameof(key));
}

var keyAsString = Encoding.UTF8.GetString(key);
protected override ValueTask<KeyValueStoreOperationStatus> DeleteAsync(KVKey key) {
var keyAsString = ConvertBytesToHexString(key);
if (!_values.TryRemove(keyAsString, out _)) {
return new ValueTask<KeyValueStoreOperationStatus>(KeyValueStoreOperationStatus.NotFound);
}
Expand All @@ -63,9 +57,14 @@ public ValueTask<KeyValueStoreOperationStatus> DeleteAsync(byte[] key) {


/// <inheritdoc/>
public IEnumerable<byte[]> GetKeys() {
protected override IEnumerable<KVKey> GetKeys(KVKey? prefix) {
foreach (var item in _values.Keys) {
yield return Encoding.UTF8.GetBytes(item);
var bytes = ConvertHexStringToBytes(item);
if (prefix != null && prefix.Value.Length > 0 && !StartsWithPrefix(prefix.Value.Value, bytes)) {
continue;
}

yield return bytes;
};
}

Expand Down
109 changes: 109 additions & 0 deletions src/DataCore.Adapter.Abstractions/Services/KVKey.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
using System;
using System.Globalization;
using System.Text;

namespace DataCore.Adapter.Services {

/// <summary>
/// Describes a key or a prefix in a <see cref="IKeyValueStore"/>.
/// </summary>
public struct KVKey : IEquatable<KVKey> {

/// <summary>
/// A <see cref="KVKey"/> that represents an empty prefix.
/// </summary>
public static KVKey Empty => default;

/// <summary>
/// The value of the key.
/// </summary>
public byte[] Value { get; }

/// <summary>
/// The length of the key, in bytes.
/// </summary>
public int Length => Value?.Length ?? 0;


/// <summary>
/// Creates a new <see cref="KVKey"/>.
/// </summary>
/// <param name="value">
/// The value of the key.
/// </param>
public KVKey(byte[]? value) {
Value = value!;
}


/// <inheritdoc/>
public override int GetHashCode() {
#if NETSTANDARD2_0 || NET461
return HashGenerator.Combine(Value);
#else
return HashCode.Combine(Value);
#endif
}


/// <inheritdoc/>
public override bool Equals(object obj) {
return obj is KVKey other
? Equals(other)
: false;
}


/// <inheritdoc/>
public bool Equals(KVKey other) {
if (Value == null && other.Value == null) {
return true;
}

if (Value == null && other.Value != null) {
return false;
}

if (Value != null && other.Value == null) {
return false;
}

if (Value!.Length != other.Value!.Length) {
return false;
}

for (var i = 0; i < Value.Length; i++) {
if (Value[i] != other.Value[i]) {
return false;
}
}

return true;
}

#pragma warning disable CS1591 // Missing XML comment for publicly visible type or member

public static bool operator ==(KVKey left, KVKey right) => left.Equals(right);
public static bool operator !=(KVKey left, KVKey right) => !left.Equals(right);

public static implicit operator KVKey(byte[] value) => new KVKey(value);
public static implicit operator KVKey(string? value) => new KVKey(Encoding.UTF8.GetBytes(value ?? string.Empty));
public static implicit operator KVKey(DateTime value) => new KVKey(Encoding.UTF8.GetBytes(value.ToString("O", CultureInfo.InvariantCulture)));
public static implicit operator KVKey(TimeSpan value) => new KVKey(Encoding.UTF8.GetBytes(value.ToString("C", CultureInfo.InvariantCulture)));
public static implicit operator KVKey(byte value) => new KVKey(new[] { value });
public static implicit operator KVKey(bool value) => new KVKey(BitConverter.GetBytes(value));
public static implicit operator KVKey(char value) => new KVKey(BitConverter.GetBytes(value));
public static implicit operator KVKey(double value) => new KVKey(BitConverter.GetBytes(value));
public static implicit operator KVKey(float value) => new KVKey(BitConverter.GetBytes(value));
public static implicit operator KVKey(int value) => new KVKey(BitConverter.GetBytes(value));
public static implicit operator KVKey(long value) => new KVKey(BitConverter.GetBytes(value));
public static implicit operator KVKey(short value) => new KVKey(BitConverter.GetBytes(value));
public static implicit operator KVKey(ushort value) => new KVKey(BitConverter.GetBytes(value));
public static implicit operator KVKey(uint value) => new KVKey(BitConverter.GetBytes(value));
public static implicit operator KVKey(ulong value) => new KVKey(BitConverter.GetBytes(value));
public static implicit operator byte[](KVKey value) => value.Value;

#pragma warning restore CS1591 // Missing XML comment for publicly visible type or member

}
}
Loading

0 comments on commit e66428f

Please sign in to comment.