Skip to content

Commit

Permalink
Merge branch 'develop/5.0.0'
Browse files Browse the repository at this point in the history
  • Loading branch information
wazzamatazz committed Sep 26, 2024
2 parents 9fbdfbd + ac096b8 commit e4f6e04
Show file tree
Hide file tree
Showing 54 changed files with 1,319 additions and 992 deletions.
2 changes: 1 addition & 1 deletion Directory.Build.props
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
building via build.ps1 or build.sh. It is defined here to allow Visual Studio to build with
the solution with the correct version number.
-->
<Version>4.0.0</Version>
<Version>5.0.0</Version>
</PropertyGroup>

<Choose>
Expand Down
53 changes: 26 additions & 27 deletions Directory.Packages.props
Original file line number Diff line number Diff line change
Expand Up @@ -6,59 +6,58 @@
</PropertyGroup>
<ItemGroup>
<PackageVersion Include="Castle.Core" Version="5.1.1" />
<PackageVersion Include="CsvHelper" Version="32.0.2" />
<PackageVersion Include="Google.Protobuf" Version="3.26.1" />
<PackageVersion Include="Grpc.AspNetCore.Server" Version="2.62.0" />
<PackageVersion Include="Grpc.Net.Client" Version="2.62.0" />
<PackageVersion Include="Grpc.Tools" Version="2.63.0" />
<PackageVersion Include="CsvHelper" Version="33.0.1" />
<PackageVersion Include="Google.Protobuf" Version="3.28.2" />
<PackageVersion Include="Grpc.AspNetCore.Server" Version="2.66.0" />
<PackageVersion Include="Grpc.Net.Client" Version="2.66.0" />
<PackageVersion Include="Grpc.Tools" Version="2.66.0" />
<PackageVersion Include="IntelligentPlant.BackgroundTasks" Version="12.0.1" />
<PackageVersion Include="IntelligentPlant.BackgroundTasks.AspNetCore" Version="12.0.1" />
<PackageVersion Include="IntelligentPlant.BackgroundTasks.DependencyInjection" Version="12.0.1" />
<PackageVersion Include="Jaahas.HttpRequestTransformer" Version="2.2.0" />
<PackageVersion Include="JsonSchema.Net" Version="7.0.1" />
<PackageVersion Include="JsonSchema.Net.Generation" Version="4.3.0.2" />
<PackageVersion Include="Jaahas.OpenTelemetry.Extensions" Version="2.0.2" />
<PackageVersion Include="JsonSchema.Net" Version="7.2.3" />
<PackageVersion Include="JsonSchema.Net.Generation" Version="4.5.0" />
<PackageVersion Include="Microsoft.AspNetCore.Mvc" Version="2.2.0" />
<PackageVersion Include="Microsoft.AspNetCore.SignalR" Version="1.1.0" />
<PackageVersion Include="Microsoft.AspNetCore.SignalR.Client" Version="8.0.4" />
<PackageVersion Include="Microsoft.AspNetCore.SignalR.Client" Version="8.0.8" />
<PackageVersion Include="Microsoft.Bcl.AsyncInterfaces" Version="8.0.0" />
<PackageVersion Include="Microsoft.Bcl.HashCode" Version="1.1.1" />
<PackageVersion Include="Microsoft.CodeAnalysis.PublicApiAnalyzers" Version="3.3.4" />
<PackageVersion Include="Microsoft.Data.Sqlite" Version="8.0.4" />
<PackageVersion Include="Microsoft.Data.Sqlite" Version="8.0.8" />
<PackageVersion Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
<PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.1" />
<PackageVersion Include="Microsoft.Extensions.Options" Version="8.0.2" />
<PackageVersion Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="8.0.0" />
<PackageVersion Include="Microsoft.FASTER.Core" Version="2.6.5" />
<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.9.0" />
<PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
<PackageVersion Include="Microsoft.NETFramework.ReferenceAssemblies" Version="1.0.3" />
<PackageVersion Include="Microsoft.SourceLink.GitHub" Version="8.0.0" />
<PackageVersion Include="Microsoft.Web.LibraryManager.Build" Version="2.1.175" />
<PackageVersion Include="MiniValidation" Version="0.9.1" />
<PackageVersion Include="MSTest.TestAdapter" Version="3.3.1" />
<PackageVersion Include="MSTest.TestFramework" Version="3.3.1" />
<PackageVersion Include="MSTest.TestAdapter" Version="3.6.0" />
<PackageVersion Include="MSTest.TestFramework" Version="3.6.0" />
<PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />
<PackageVersion Include="Nito.AsyncEx.Coordination" Version="5.1.2" />
<PackageVersion Include="NSwag.AspNetCore" Version="13.20.0" />
<PackageVersion Include="OpenTelemetry" Version="1.8.1" />
<PackageVersion Include="OpenTelemetry.Api" Version="1.8.1" />
<PackageVersion Include="OpenTelemetry.Exporter.Console" Version="1.8.1" />
<PackageVersion Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.8.1" />
<PackageVersion Include="OpenTelemetry.Exporter.Prometheus.AspNetCore" Version="1.8.0-rc.1" />
<PackageVersion Include="OpenTelemetry.Extensions.Hosting" Version="1.8.1" />
<PackageVersion Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.8.1" />
<PackageVersion Include="OpenTelemetry.Instrumentation.GrpcNetClient" Version="1.8.0-beta.1" />
<PackageVersion Include="OpenTelemetry.Instrumentation.Http" Version="1.8.1" />
<PackageVersion Include="OpenTelemetry.Instrumentation.Runtime" Version="1.8.0" />
<PackageVersion Include="OpenTelemetry.Instrumentation.SqlClient" Version="1.8.0-beta.1" />
<PackageVersion Include="OpenTelemetry" Version="1.9.0" />
<PackageVersion Include="OpenTelemetry.Api" Version="1.9.0" />
<PackageVersion Include="OpenTelemetry.Exporter.Console" Version="1.9.0" />
<PackageVersion Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.9.0" />
<PackageVersion Include="OpenTelemetry.Exporter.Prometheus.AspNetCore" Version="1.9.0-beta.2" />
<PackageVersion Include="OpenTelemetry.Extensions.Hosting" Version="1.9.0" />
<PackageVersion Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.9.0" />
<PackageVersion Include="OpenTelemetry.Instrumentation.GrpcNetClient" Version="1.9.0-beta.1" />
<PackageVersion Include="OpenTelemetry.Instrumentation.Http" Version="1.9.0" />
<PackageVersion Include="OpenTelemetry.Instrumentation.Runtime" Version="1.9.0" />
<PackageVersion Include="OpenTelemetry.Instrumentation.SqlClient" Version="1.9.0-beta.1" />
<PackageVersion Include="Semver" Version="2.3.0" />
<PackageVersion Include="Serilog.AspNetCore" Version="8.0.1" />
<PackageVersion Include="Serilog.Expressions" Version="4.0.0" />
<PackageVersion Include="SQLitePCLRaw.bundle_green" Version="2.1.8" />
<PackageVersion Include="SQLitePCLRaw.bundle_green" Version="2.1.10" />
<PackageVersion Include="System.ComponentModel.Annotations" Version="5.0.0" />
<PackageVersion Include="System.Diagnostics.DiagnosticSource" Version="8.0.1" />
<PackageVersion Include="System.Linq.Async" Version="6.0.1" />
<PackageVersion Include="System.Net.Http.Json" Version="8.0.0" />
<PackageVersion Include="System.Net.Http.WinHttpHandler" Version="8.0.0" />
<PackageVersion Include="System.Net.Http.WinHttpHandler" Version="8.0.2" />
<PackageVersion Include="System.Text.Json" Version="8.0.4" />
<PackageVersion Include="System.Threading.Channels" Version="8.0.0" />
<PackageVersion Include="System.ValueTuple" Version="4.5.0" />
Expand Down
2 changes: 1 addition & 1 deletion build.cake
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ const string VersionFile = "./build/version.json";
//
///////////////////////////////////////////////////////////////////////////////////////////////////

#load nuget:?package=Jaahas.Cake.Extensions&version=2.0.2
#load nuget:?package=Jaahas.Cake.Extensions&version=2.2.1

// Bootstrap build context and tasks.
Bootstrap(DefaultSolutionFile, VersionFile);
Expand Down
6 changes: 3 additions & 3 deletions build/version.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"Major": 4,
"Minor": 1,
"Patch": 1,
"Major": 5,
"Minor": 0,
"Patch": 0,
"PreRelease": ""
}
54 changes: 5 additions & 49 deletions docs/adapter-host-logging.md
Original file line number Diff line number Diff line change
@@ -1,57 +1,13 @@
# Adapter Host Logging
# Adapter Host Telemetry and Observability

> Note: this document assumes that you have used the project template for Visual Studio and `dotnet new` to create an adapter host. See [here](../src/DataCore.Adapter.Templates) for more information.
By default, adapter hosts created using the project template for Visual Studio and `dotnet new` use [Serilog](https://serilog.net/) to write log messages to the console.
The adapter host uses OpenTelemetry to observe traces, metrics and logs generated by the application. An OpenTelemetry Protocol (OTLP) exporter that exports traces is enabled by default. You can disable the exporter, modify the signals to be exported, or configure the OTLP collector endpoint using the `appsettings.json` file.

The section below describes how to use Serilog to write logs to files in addition to the console. Please refer to the Serilog documentation for full details of how to write logs to additional destinations.
See [here](https://github.com/wazzamatazz/opentelemetry-extensions) for more information about configuring the OTLP exporter.

Additionally, metrics can be scraped from the application's `/metrics` endpoint in Prometheus format. The Prometheus endpoint can be disabled or removed from the `Program.cs` file if not required.

# Configuring Serilog to write to files via `appsettings.json`
The adapter host uses the Microsoft.Extensions.Logging library for logging. If required, you can add additional logging providers to the application by modifying the `Program.cs` file.

Serilog can be configured to write to log files with a simple configuration file modification:

```json
{
"Serilog": {
"MinimumLevel": {
"Default": "Information",
"Override": {
"Microsoft": "Warning",
"Microsoft.Hosting.Lifetime": "Information"
}
},
"WriteTo": [
{
"Name": "File",
"Args": {
"path": "./logs/log-.txt",
"rollingInterval": "Day",
"formatter": {
"type": "Serilog.Formatting.Compact.CompactJsonFormatter, Serilog.Formatting.Compact"
}
}
},
{
"Name": "Console",
"Args": {
"formatter": {
"type": "Serilog.Templates.ExpressionTemplate, Serilog.Expressions",
"template": "{@l:w4}: {SourceContext}[{Coalesce(EventId.Id, '0')}] \n {@m}\n{@x}",
"theme": "Serilog.Templates.Themes.TemplateTheme::Code, Serilog.Expressions"
}
}
}
]
}
}
```

The above example performs the following actions:

- Writes log messages to the console using the same default format used by Microsoft's console logger.
- Writes log messages to daily log files in the `logs/` folder under the adapter host's installation directory using the [Compact Log Event Format](https://clef-json.org/) (CLEF).

By default, log files for the last 31 days are kept.

Refer to the Serilog documentation for full details about configuration of logging destinations!
45 changes: 45 additions & 0 deletions docs/release-notes/v5.0.0.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
# Release Notes - v5.0.0

Welcome to version 5.0.0 of the adapter toolkit for App Store Connect!


# Breaking Changes

This release contains the following breaking changes:


## Changes to Entity Builder Types

Types such as `TagValueBuilder` that are used to construct adapter entities using a fluent interface have been refactored to derive from a common base type, `AdapterEntityBuilder<T>`. The base type defines some common behaviour such as management of custom `AdapterProperty` properties added to the builder.

Duplicate properties can no longer be defined on any builder derived from `AdapterEntityBuilder<T>`. Instead, callers can choose whether existing properties with the same name should be retained or dropped when adding properties to the builder.

Note that `AdapterEntityBuilder<T>` does not define any public methods for adding properties to the builder; these are instead provided by extension methods defined on the `AdapterEntityBuilderExtensions` class.


## Hash Code Computation in `TagIdentifierComparer`

The hash code computation in `TagIdentifierComparer` has been modified to remove unnecessary allocations. A side effect of this change is that the hash code computed for a given `TagIdentifier` is now different.


# Non-Breaking Changes

This release includes the following changes and features:


## String Caching

The new `StringCache` class is now used to cache frequently-used string instances instead of using `string.Intern` directly. `StringCache` uses a `ConcurrentDictionary<string, string>` to cache string instances; benchmarks show that this usually gives superior performance compared to `string.Intern`. `StringCache` can be configured to use `string.Intern` instead of its own cache by setting the `DataCore.Adapter.StringCache.UseNativeIntern` flag on `AppContext` to `false`.

Additionally, new `string` extension methods have been added in the `System` namespace to allow strings to be interned easily using `StringCache`:

* `InternToStringCache` will intern the string if it has not already been interned and returns the interned reference to the string e.g. `var str = "my string".InternToStringCache();`
* `GetFromStringCache` returns the interned reference to the string or `null` if the string has not been interned e.g. `var str = "my other string".GetFromStringCache();`


## Template Changes

The `dotnet new` template has been modified:

* The template no longer uses Serilog for logging and instead reverts to the standard Microsoft.Extensions.Logging library. Developers are free to add additional logging providers to the application as required.
* An OpenTelemetry Protocol (OTLP) exporter is now enabled by default for exporting traces. The exporter endpoint and the signals to be exported (i.e. traces, logs, metrics) can be configured using the `appsettings.json` file.
4 changes: 3 additions & 1 deletion docs/tutorials/mqtt-adapter/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -413,10 +413,12 @@ When an MQTT message is processed, a tag definition will be created if one does

The adapter constructor accepts an `IOptionsMonitor<MyAdapterOptions>` parameter. Since `IOptionsMonitor<T>` supports change notifications, this allows the adapter's settings in `adaptersettings.json` to be modified at runtime and the adapter will automatically reconfigure itself to use the updated settings. Runtime changes are handled by overriding the `OnOptionsChange` method from the `AdapterBase<TAdapterOptions>` base class.

### Telemetry
### Telemetry and Observability

The `MyAdapter` class creates a static metric counter that is incremented every time a message is received from the MQTT broker. The counter can be observed via the [dotnet-counters](https://learn.microsoft.com/en-us/dotnet/core/diagnostics/dotnet-counters) tool (by observing the `IntelligentPlant.AppStoreConnect.Adapter:mqtt.messages_received` counter) or via the adapter host's Prometheus scraping endpoint at `/metrics`.

Additionally, you can export any combination of traces, logs and/or metrics to a compatible OTLP collector. See [here](../../adapter-host-logging.md) for more information.


# Update the Adapter Options Editor Form

Expand Down
3 changes: 1 addition & 2 deletions examples/DataCore.Adapter.ExampleAdapter/ExampleAdapter.cs
Original file line number Diff line number Diff line change
Expand Up @@ -72,8 +72,7 @@ protected override async Task OnStartedAsync(CancellationToken cancellationToken
while (!cancellationToken.IsCancellationRequested) {
var evtManager = (InMemoryEventMessageStore) Features.Get<IWriteEventMessages>().Unwrap();
await evtManager.WriteEventMessages(
EventMessageBuilder
.Create()
new EventMessageBuilder()
.WithPriority(EventPriority.Low)
.WithCategory("System Messages")
.WithMessage($"Uptime: {(DateTime.UtcNow - startup)}")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,7 @@ internal async Task Init(string adapterId, Tags.ITagSearch tagSearch, Cancellati
var tags = await dataReferencesChannel.ToEnumerable(cancellationToken: cancellationToken).ConfigureAwait(false);

_nodes = nodeDefinitions.Select(x =>
AssetModelNodeBuilder
.Create()
new AssetModelNodeBuilder()
.WithId(x.Id)
.WithName(x.Name)
.WithNodeType(x.NodeType)
Expand Down
3 changes: 1 addition & 2 deletions examples/ExampleHostedAdapter/ExampleHostedAdapter.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
<ProjectReference Include="..\..\src\DataCore.Adapter.AspNetCore.SignalR\DataCore.Adapter.AspNetCore.SignalR.csproj" />
<ProjectReference Include="..\..\src\DataCore.Adapter.KeyValueStore.Sqlite\DataCore.Adapter.KeyValueStore.Sqlite.csproj" />
<ProjectReference Include="..\..\src\DataCore.Adapter.OpenTelemetry\DataCore.Adapter.OpenTelemetry.csproj" />
<PackageReference Include="Jaahas.OpenTelemetry.Extensions" />
<PackageReference Include="Microsoft.Web.LibraryManager.Build" />
<PackageReference Include="OpenTelemetry" />
<PackageReference Include="OpenTelemetry.Api" />
Expand All @@ -26,8 +27,6 @@
<PackageReference Include="OpenTelemetry.Instrumentation.Http" />
<PackageReference Include="OpenTelemetry.Instrumentation.Runtime" />
<PackageReference Include="OpenTelemetry.Instrumentation.SqlClient" />
<PackageReference Include="Serilog.AspNetCore" />
<PackageReference Include="Serilog.Expressions" />
</ItemGroup>

</Project>
21 changes: 6 additions & 15 deletions examples/ExampleHostedAdapter/Program.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,11 @@

using ExampleHostedAdapter;

using OpenTelemetry;
using OpenTelemetry.Metrics;
using OpenTelemetry.Resources;
using OpenTelemetry.Trace;

using Serilog;

// The [VendorInfo] attribute is used to add vendor information to the adapters in this assembly,
// as well as the host information for the application.
[assembly: DataCore.Adapter.VendorInfo("My Company", "https://my-company.com")]
Expand All @@ -18,16 +17,6 @@
builder.Configuration
.AddJsonFile(Constants.AdapterSettingsFilePath, false, true);

// Configure logging using Serilog. Additional logging destinations such as files can be added
// using appsettings.json. See https://github.com/serilog/serilog-settings-configuration for more
// information.
builder.Host.UseSerilog((context, services, configuration) => {
configuration
.ReadFrom.Configuration(context.Configuration)
.ReadFrom.Services(services)
.Enrich.FromLogContext();
});

// Parent PID. If specified, we will gracefully shut down if the parent process exits.
var pid = builder.Configuration.GetValue<int>("AppStoreConnect:Adapter:Host:ParentPid");
if (pid > 0) {
Expand Down Expand Up @@ -101,6 +90,10 @@
builder.Services
.AddOpenTelemetry()
.ConfigureResource(resourceBuilder => resourceBuilder.AddDataCoreAdapterApiService(instanceId))
// Export telemetry signals using OTLP (OpenTelemetry Protocol). Exporter settings can be
// configured in appsettings.json. See https://github.com/wazzamatazz/opentelemetry-extensions#configuring-a-multi-signal-opentelemetry-protocol-otlp-exporter
// for more information on configuring the OTLP exporter.
.AddOtlpExporter(builder.Configuration)
.WithTracing(otel => otel
// Records incoming HTTP requests made to the adapter host.
.AddAspNetCoreInstrumentation()
Expand All @@ -111,9 +104,7 @@
// Records queries made by System.Data.SqlClient and Microsoft.Data.SqlClient.
.AddSqlClientInstrumentation()
// Records activities created by adapters and adapter hosting packages.
.AddDataCoreAdapterInstrumentation()
// Exports traces in OTLP format using default settings (i.e. http://localhost:4317 using OTLP/gRPC format).
.AddOtlpExporter())
.AddDataCoreAdapterInstrumentation())
.WithMetrics(otel => otel
// Observe instrumentation for the .NET runtime.
.AddRuntimeInstrumentation()
Expand Down
10 changes: 4 additions & 6 deletions examples/ExampleHostedAdapter/appsettings.Development.json
Original file line number Diff line number Diff line change
@@ -1,11 +1,9 @@
{
"Serilog": {
"MinimumLevel": {
"Logging": {
"LogLevel": {
"Default": "Information",
"Override": {
"Microsoft": "Warning",
"Microsoft.Hosting.Lifetime": "Information"
}
"Microsoft": "Warning",
"Microsoft.Hosting.Lifetime": "Information"
}
}
}
Loading

0 comments on commit e4f6e04

Please sign in to comment.