diff --git a/Cognite.Simulator.Tests/UtilsTests/ConnectorBaseTest.cs b/Cognite.Simulator.Tests/UtilsTests/ConnectorBaseTest.cs index bbf58926..d4528fd3 100644 --- a/Cognite.Simulator.Tests/UtilsTests/ConnectorBaseTest.cs +++ b/Cognite.Simulator.Tests/UtilsTests/ConnectorBaseTest.cs @@ -1,4 +1,5 @@ using Cognite.Extractor.Common; +using Cognite.Extractor.Logging; using Cognite.Extractor.Utils; using Cognite.Simulator.Extensions; using Cognite.Simulator.Utils; @@ -23,6 +24,9 @@ public async Task TestConnectorBase() { var services = new ServiceCollection(); services.AddCogniteTestClient(); + services.AddLogger(); + services.AddSingleton>(provider => null!); + services.AddSingleton(); services.AddTransient(); services.AddSingleton(); var simConfig = new SimulatorConfig @@ -110,20 +114,24 @@ await cdf.ExtPipes } } + /// /// Implements a simple mock connector that only reports /// status back to CDF (Heartbeat) /// - internal class TestConnector : ConnectorBase + internal class TestConnector : ConnectorBase { private readonly ExtractionPipeline _pipeline; + private readonly RemoteConfigManager _remoteConfigManager; private readonly SimulatorConfig _config; - + + public TestConnector( CogniteDestination cdf, ExtractionPipeline pipeline, SimulatorConfig config, - ILogger logger) : + ILogger logger, + RemoteConfigManager remoteConfigManager) : base( cdf, new ConnectorConfig @@ -135,10 +143,12 @@ public TestConnector( { config }, - logger) + logger, + remoteConfigManager) { _pipeline = pipeline; _config = config; + _remoteConfigManager = remoteConfigManager; } public override string GetConnectorVersion() diff --git a/Cognite.Simulator.Utils/ConnectorBase.cs b/Cognite.Simulator.Utils/ConnectorBase.cs index a6cb8827..5351e70a 100644 --- a/Cognite.Simulator.Utils/ConnectorBase.cs +++ b/Cognite.Simulator.Utils/ConnectorBase.cs @@ -8,6 +8,7 @@ using System.Text; using System.Threading; using System.Threading.Tasks; +using Cognite.Extractor.Configuration; namespace Cognite.Simulator.Utils { @@ -16,7 +17,7 @@ namespace Cognite.Simulator.Utils /// The connector information is saved as a CDF sequence, where the rows /// are key/value pairs (see ) /// - public abstract class ConnectorBase + public abstract class ConnectorBase where T : BaseConfig { /// /// CDF client wrapper @@ -30,11 +31,14 @@ public abstract class ConnectorBase private ConnectorConfig Config { get; } private readonly Dictionary _simulatorSequenceIds; - private readonly ILogger _logger; + private readonly ILogger> _logger; private readonly ConnectorConfig _config; private long LastLicenseCheckTimestamp { get; set; } private string LastLicenseCheckResult { get; set; } + private const int FIFTEEN_MIN = 9000; + + private readonly RemoteConfigManager _remoteConfigManager; /// /// Initialize the connector with the given parameters @@ -43,11 +47,13 @@ public abstract class ConnectorBase /// Connector configuration /// List of simulator configurations /// Logger + /// public ConnectorBase( CogniteDestination cdf, ConnectorConfig config, IList simulators, - ILogger logger) + ILogger> logger, + RemoteConfigManager remoteConfigManager) { Cdf = cdf; Simulators = simulators; @@ -55,6 +61,7 @@ public ConnectorBase( _simulatorSequenceIds = new Dictionary(); _logger = logger; _config = config; + _remoteConfigManager = remoteConfigManager; } /// @@ -284,6 +291,33 @@ await UpdateIntegrationRows(false, token) .ConfigureAwait(false); } } + + /// + /// Task that runs in a loop, checking for new config in extraction pipelines + /// + public async Task CheckRemoteConfig(CancellationToken token) + { + while (!token.IsCancellationRequested) + { + await Task + .Delay(FIFTEEN_MIN, token) // Run every 15 minutes + .ConfigureAwait(false); + _logger.LogDebug("Checking remote config updates"); + if (_remoteConfigManager == null) return; + var newConfig = await _remoteConfigManager.FetchLatest(token).ConfigureAwait(false); + if (newConfig != null) + { + throw new NewConfigDetected(); + } + } + } + } + + /// + /// Exception used to restart connector + /// + public class NewConfigDetected : Exception + { } /// diff --git a/Cognite.Simulator.Utils/ExtractionPipeline.cs b/Cognite.Simulator.Utils/ExtractionPipeline.cs index 22a9cd57..9e768b3f 100644 --- a/Cognite.Simulator.Utils/ExtractionPipeline.cs +++ b/Cognite.Simulator.Utils/ExtractionPipeline.cs @@ -8,6 +8,8 @@ using System.Text; using System.Threading; using System.Threading.Tasks; +using Cognite.Extractor.Configuration; +using System.IO; namespace Cognite.Simulator.Utils { @@ -237,5 +239,54 @@ public static void AddExtractionPipeline(this IServiceCollection services, Conne services.AddSingleton(config.PipelineNotification); services.AddScoped(); } + /// + /// Use `type: remote` to fetch the config from Fusion, or use `type: local` to use the local file instead + /// Example from config.yml using the remote config from Fusion + /// type: remote # this is required + /// cognite: + /// project: ... + /// host: ... + /// extraction-pipeline: + /// pipeline-id: ... # as well as this + /// idp-authentication: + /// ... + /// + /// The complete config object to be parsed + /// Service collection + /// Path to config file + /// Types to use for deserialization + /// App ID for measuring network data + /// Cancellation token + /// Config version + /// Accepted config versions + public static async Task AddConfiguration( + this IServiceCollection services, + string path, + Type[] types, + string appId, + CancellationToken token, + int version = 1, + int[] acceptedConfigVersions = null) where T : BaseConfig + { + var localConfig = services.AddConfig(path, version); + var remoteConfig = new RemoteConfig + { + Type = localConfig.Type, + Cognite = localConfig.Cognite + }; + + return await services.AddRemoteConfig( + logger: null, + path: path, + types: types, + appId: appId, + userAgent: null, + setDestination: true, + bufferConfigFile: false, + remoteConfig: remoteConfig, + token: token, + acceptedConfigVersions: acceptedConfigVersions + ).ConfigureAwait(false); + } } } diff --git a/version b/version index 505f2413..4bd73bcd 100644 --- a/version +++ b/version @@ -1 +1 @@ -1.0.0-alpha-013 +1.0.0-alpha-014