From 37e69dc1d656248d5c331e27401449b6bf56fbb8 Mon Sep 17 00:00:00 2001 From: Ivan Polomani Date: Thu, 14 Dec 2023 15:43:38 +0100 Subject: [PATCH] Fix more tests --- .../SequencesExtensions.cs | 99 ----------- Cognite.Simulator.Extensions/Types.cs | 54 ------ Cognite.Simulator.Tests/TestHelpers.cs | 51 ++++++ .../UtilsTests/SimulationRunnerTest.cs | 41 +---- .../UtilsTests/SimulationSchedulerTest.cs | 159 +----------------- 5 files changed, 59 insertions(+), 345 deletions(-) create mode 100644 Cognite.Simulator.Tests/TestHelpers.cs diff --git a/Cognite.Simulator.Extensions/SequencesExtensions.cs b/Cognite.Simulator.Extensions/SequencesExtensions.cs index 2717bd9d..abd7f672 100644 --- a/Cognite.Simulator.Extensions/SequencesExtensions.cs +++ b/Cognite.Simulator.Extensions/SequencesExtensions.cs @@ -415,105 +415,6 @@ public static string[] GetStringValues(this SequenceRow row) } return result.ToArray(); } - - // /// - // /// Simply upserts a key value pair into a key value pair based sequence - // /// - // /// CDF Sequence resource - // /// External id of the sequence. - // /// Key in sequence to update - // /// Value to update the Key value pair to - // /// The cancellation token - // /// Nothing - // public static async - // Task UpsertItemInKVPSequence(this SequencesResource sequences, string sequenceExternalId, string updateKey, string updateValue, CancellationToken token) { - // if (sequenceExternalId == "") { - // throw new Exception("External Id is required to upsert a sequence"); - // } - // SequenceRowQuery query = new SequenceRowQuery(); - // query.ExternalId = sequenceExternalId; - // var sequenceList = await sequences.ListRowsAsync(query, token).ConfigureAwait(false);; - // var rows = sequenceList.Rows; - // var columns = sequenceList.Columns; - // var newRows = rows.Select(r => new SequenceRow - // { - // RowNumber = r.RowNumber, - // Values = r.Values - // }).ToList(); - // // Checking to see if the key alread exists - // var foundIndex = newRows.FindIndex(0, newRows.Count, (item => item.Values.FirstOrDefault().ToString().Contains(updateKey) )); - // if (foundIndex == -1){ - // newRows.Insert(newRows.Count , new SequenceRow{ - // RowNumber = newRows.Count, - // Values = new List { - // MultiValue.Create(updateKey), - // MultiValue.Create(updateValue) - // } - // }); - // } else { - // newRows[foundIndex].Values = new List { - // MultiValue.Create(updateKey), - // MultiValue.Create(updateValue) - // }; - // } - - // SequenceDataCreate sequenceData = new SequenceDataCreate - // { - // ExternalId = sequenceExternalId, - // Columns = new List { - // KeyValuePairSequenceColumns.Key, - // KeyValuePairSequenceColumns.Value - // }, - // Rows = newRows - // }; - // var rowsToCreate = new List(); - // rowsToCreate.Add(sequenceData); - // var result = await sequences.InsertAsync( - // rowsToCreate, - // keyChunkSize: 10, - // valueChunkSize: 100, - // sequencesChunk: 10, - // throttleSize: 1, - // RetryMode.None, - // SanitationMode.None, - // token - // ).ConfigureAwait(false); - - // if (!result.IsAllGood) - // { - // throw new SimulatorIntegrationSequenceException("Could not update simulator integration sequence", result.Errors); - // } - // } - - // /// - // /// Gets the externalId of a SimulatorIntegrations sequence (This function can throw an exception) - // /// - // /// CDF Sequence resource - // /// The simulator name. - // /// The simulator datasetId - // /// The connector Name - // /// The cancellation token - // /// externalId string - // public static async Task GetSequenceExternalId(this SequencesResource sequences, string simulatorName, long simulatorDataSetId, string connectorName, CancellationToken token) { - // var simulatorSequenceIds = new Dictionary(); - // var simulatorsDict = new SimulatorIntegration - // { - // Simulator = simulatorName, - // DataSetId = simulatorDataSetId, - // ConnectorName = connectorName - // }; - - // var integrations = await sequences.GetOrCreateSimulatorIntegrations( - // new List { simulatorsDict }, - // token).ConfigureAwait(false); - // foreach (var integration in integrations) - // { - // simulatorSequenceIds.Add( - // integration.Metadata[BaseMetadata.SimulatorKey], - // integration.ExternalId); - // } - // return simulatorSequenceIds[simulatorName]; - // } } /// diff --git a/Cognite.Simulator.Extensions/Types.cs b/Cognite.Simulator.Extensions/Types.cs index 8c453448..2dc6d32d 100644 --- a/Cognite.Simulator.Extensions/Types.cs +++ b/Cognite.Simulator.Extensions/Types.cs @@ -374,58 +374,4 @@ public void OverwriteTimeSeriesId(string externalId) ExternalIdOverwrite = externalId; } } - - // /// - // /// Represents simulator integration information in CDF. Each connector can register one or more - // /// simulator integration record with CDF - // /// - // public class SimulatorIntegration - // { - // /// - // /// Name of the simulator - // /// - // public string Simulator { get; set; } - - // /// - // /// Name of the connector - // /// - // public string ConnectorName { get; set; } - - // /// - // /// ID of the data set that holds simulator data in CDF - // /// - // public long? DataSetId { get; set; } - // } - - // /// - // /// Represents data used to update the simulator integration information in CDF - // /// - // public class SimulatorIntegrationUpdate : SimulatorIntegration - // { - // /// - // /// Version of the connector - // /// - // public string ConnectorVersion { get; set; } - - // /// - // /// Version of the simulator - // /// - // public string SimulatorVersion { get; set; } - - // /// - // /// Flag indicating if the connector is using Cognite's Simulator Integration API - // /// or just core CDF resources. - // /// - // public bool SimulatorApiEnabled { get; set; } - - // /// - // /// Flag indicating if there connector will check for simulator liceses - // /// - // public bool LicenseCheckEnabled { get; set; } - - // /// - // /// Any extra information that can be registered by the connector - // /// - // public Dictionary ExtraInformation { get; set; } = new Dictionary(); - // } } diff --git a/Cognite.Simulator.Tests/TestHelpers.cs b/Cognite.Simulator.Tests/TestHelpers.cs new file mode 100644 index 00000000..14cb610b --- /dev/null +++ b/Cognite.Simulator.Tests/TestHelpers.cs @@ -0,0 +1,51 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Cognite.Extractor.Common; +using CogniteSdk; +using CogniteSdk.Alpha; + +namespace Cognite.Simulator.Tests { + public class TestHelpers { + + public static async Task SimulateProsperRunningAsync(Client cdf, string connectorName = "scheduler-test-connector" ) { + var integrations = await cdf.Alpha.Simulators.ListSimulatorIntegrationsAsync( + new SimulatorIntegrationQuery + { + } + ).ConfigureAwait(false); + var existing = integrations.Items.FirstOrDefault(i => i.ExternalId == connectorName && i.SimulatorExternalId == "PROSPER"); + if (existing == null) { + await cdf.Alpha.Simulators.CreateSimulatorIntegrationAsync( + new List + { + new SimulatorIntegrationCreate + { + ExternalId = connectorName, + SimulatorExternalId = "PROSPER", + DataSetId = CdfTestClient.TestDataset, + Heartbeat = DateTime.UtcNow.ToUnixTimeMilliseconds(), + ConnectorVersion = "N/A", + SimulatorVersion = "N/A", + RunApiEnabled = true, + } + } + ).ConfigureAwait(false); + } else { + await cdf.Alpha.Simulators.UpdateSimulatorIntegrationAsync( + new List + { + new SimulatorIntegrationUpdateItem(existing.Id) + { + Update = new SimulatorIntegrationUpdate + { + Heartbeat = new Update(DateTime.UtcNow.ToUnixTimeMilliseconds()) + } + } + } + ).ConfigureAwait(false); + } + } + } +} \ No newline at end of file diff --git a/Cognite.Simulator.Tests/UtilsTests/SimulationRunnerTest.cs b/Cognite.Simulator.Tests/UtilsTests/SimulationRunnerTest.cs index 8b70d201..b1f1b515 100644 --- a/Cognite.Simulator.Tests/UtilsTests/SimulationRunnerTest.cs +++ b/Cognite.Simulator.Tests/UtilsTests/SimulationRunnerTest.cs @@ -103,7 +103,7 @@ public async Task TestSimulationRunnerBase(String calculationName, String calcTy inTsIds.AddRange(inConstTsIds); } - await SimulateProsperRunningAsync(cdf, "integration-tests-connector").ConfigureAwait(true); + await TestHelpers.SimulateProsperRunningAsync(cdf, "integration-tests-connector").ConfigureAwait(true); var runs = await cdf.Alpha.Simulators.CreateSimulationRunsAsync( new List @@ -292,44 +292,7 @@ await cdf.TimeSeries.DeleteAsync(new TimeSeriesDelete } - public static async Task SimulateProsperRunningAsync(Client cdf, string connectorName = "scheduler-test-connector" ) { - var integrations = await cdf.Alpha.Simulators.ListSimulatorIntegrationsAsync( - new SimulatorIntegrationQuery - { - } - ).ConfigureAwait(false); - var existing = integrations.Items.FirstOrDefault(i => i.ExternalId == connectorName && i.SimulatorExternalId == "PROSPER"); - if (existing == null) { - await cdf.Alpha.Simulators.CreateSimulatorIntegrationAsync( - new List - { - new SimulatorIntegrationCreate - { - ExternalId = connectorName, - SimulatorExternalId = "PROSPER", - DataSetId = CdfTestClient.TestDataset, - Heartbeat = DateTime.UtcNow.ToUnixTimeMilliseconds(), - ConnectorVersion = "N/A", - SimulatorVersion = "N/A", - RunApiEnabled = true, - } - } - ).ConfigureAwait(false); - } else { - await cdf.Alpha.Simulators.UpdateSimulatorIntegrationAsync( - new List - { - new SimulatorIntegrationUpdateItem(existing.Id) - { - Update = new SimulatorIntegrationUpdate - { - Heartbeat = new Update(DateTime.UtcNow.ToUnixTimeMilliseconds()) - } - } - } - ).ConfigureAwait(false); - } - } + private static Dictionary ToRowDictionary(SequenceData data) { Dictionary result = new(); diff --git a/Cognite.Simulator.Tests/UtilsTests/SimulationSchedulerTest.cs b/Cognite.Simulator.Tests/UtilsTests/SimulationSchedulerTest.cs index 4bef2f5b..1fc7b6e2 100644 --- a/Cognite.Simulator.Tests/UtilsTests/SimulationSchedulerTest.cs +++ b/Cognite.Simulator.Tests/UtilsTests/SimulationSchedulerTest.cs @@ -22,132 +22,6 @@ public class SimulationSchedulerTest { [Fact] public async Task TestSimulationSchedulerBase() - { - var services = new ServiceCollection(); - services.AddCogniteTestClient(); - services.AddHttpClient(); - services.AddSingleton(); - services.AddSingleton(new ConnectorConfig - { - NamePrefix = "scheduler-test-connector", - AddMachineNameSuffix = false, - SchedulerUpdateInterval = 2 - }); - services.AddSingleton(); - - StateStoreConfig stateConfig = null; - - List eventIds = new List(); - - using var source = new CancellationTokenSource(); - using var provider = services.BuildServiceProvider(); - var cdf = provider.GetRequiredService(); - try - { - // var simint = new Extensions.SimulatorIntegration () { - // Simulator = "PROSPER", - // DataSetId = CdfTestClient.TestDataset, - // ConnectorName = "scheduler-test-connector", - // }; - // var simulators = new List { simint }; - - // var integrations = await cdf.Sequences.GetOrCreateSimulatorIntegrations( - // simulators, - // CancellationToken.None).ConfigureAwait(false); - - // var sequenceExternalId = integrations.First().ExternalId; - - - // var updateObj = new CogniteSdk.Alpha.SimulatorIntegrationCreate - // { - // ExternalId = sequenceExternalId, - // SimulatorExternalId = simint.Simulator, - // DataSetId = simint.DataSetId ?? 0L, - // RunApiEnabled = true, - // ConnectorVersion = "n/a", - // SimulatorVersion = "n/a", - // }; - // await cdf.Alpha.Simulators.CreateSimulatorIntegrationAsync( - // new List { updateObj }, source.Token - // ).ConfigureAwait(false); - - // await cdf.Sequences.UpdateSimulatorIntegrationsData( - // sequenceExternalId, - // true, - // new Extensions.SimulatorIntegrationUpdate - // { - // Simulator = simint.Simulator, - // DataSetId = simint.DataSetId, - // ConnectorName = simint.ConnectorName, - // SimulatorApiEnabled = true, - // }, - // CancellationToken.None, - // lastLicenseCheckTimestamp: 0, - // lastLicenseCheckResult: "Available").ConfigureAwait(false); - - - stateConfig = provider.GetRequiredService(); - var configLib = provider.GetRequiredService(); - var scheduler = provider.GetRequiredService(); - - await configLib.Init(source.Token).ConfigureAwait(false); - - using var linkedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(source.Token); - var linkedToken = linkedTokenSource.Token; - linkedTokenSource.CancelAfter(TimeSpan.FromSeconds(10)); - var taskList = new List { scheduler.Run(linkedToken) }; - taskList.AddRange(configLib.GetRunTasks(linkedToken)); - await taskList.RunAll(linkedTokenSource).ConfigureAwait(false); - - Assert.NotEmpty(configLib.State); - var configState = Assert.Contains( - "PROSPER-SC-UserDefined-SST-Connector_Test_Model", // This simulator configuration should exist in CDF - (IReadOnlyDictionary)configLib.State); - var configObj = configLib.GetSimulationConfiguration( - "PROSPER", "Connector Test Model", "UserDefined", "SST"); - Assert.NotNull(configObj); - - // Should have created at least one simulation event ready to run - var events = await cdf.Alpha.Simulators.ListSimulationRunsAsync( - new SimulationRunQuery { - Filter = new SimulationRunFilter { - ModelName = configObj.ModelName, - RoutineName = configObj.CalculationName, - SimulatorName = configObj.Simulator, - Status = SimulationRunStatus.ready - } - }, source.Token).ConfigureAwait(false); - - Assert.NotEmpty(events.Items); - // order events.Items by created time in descending order and filter only items created in the last 10 seconds - var latestEvents = events.Items.OrderByDescending(e => e.CreatedTime).ToList(); - var latestEventsFiltered = latestEvents.Where(e => e.CreatedTime > DateTimeOffset.UtcNow.AddSeconds(-10).ToUnixTimeMilliseconds()).ToList(); - Assert.Contains(latestEventsFiltered, e => e.RunType == SimulationRunType.scheduled); - } - finally - { - if (eventIds.Any()) - { - await cdf.Events.DeleteAsync(eventIds, source.Token) - .ConfigureAwait(false); - - // TODO Delete simulation runs created on the API - } - provider.Dispose(); // Dispose provider to also dispose managed services - if (Directory.Exists("./configurations")) - { - Directory.Delete("./configurations", true); - } - if (stateConfig != null) - { - StateUtils.DeleteLocalFile(stateConfig.Location); - } - } - } - - [Fact] - [Trait("Category", "API")] - public async Task TestSimulationSchedulerBaseWithApi() { var services = new ServiceCollection(); services.AddCogniteTestClient(); @@ -170,34 +44,10 @@ public async Task TestSimulationSchedulerBaseWithApi() using var source = new CancellationTokenSource(); using var provider = services.BuildServiceProvider(); var cdf = provider.GetRequiredService(); - // var simint = new Extensions.SimulatorIntegration () { - // Simulator = "PROSPER", - // DataSetId = CdfTestClient.TestDataset, - // ConnectorName = "scheduler-test-connector", - // }; - // var simulators = new List { simint }; + try { - // var integrations = await cdf.Sequences.GetOrCreateSimulatorIntegrations( - // simulators, - // CancellationToken.None).ConfigureAwait(false); - - // var sequenceExternalId = integrations.First().ExternalId; - - // // Update the sequence with connector heartbeat - // await cdf.Sequences.UpdateSimulatorIntegrationsData( - // sequenceExternalId, - // true, - // new Extensions.SimulatorIntegrationUpdate - // { - // Simulator = simint.Simulator, - // DataSetId = simint.DataSetId, - // ConnectorName = simint.ConnectorName, - // SimulatorApiEnabled = true, - // }, - // CancellationToken.None, - // lastLicenseCheckTimestamp: testStartTimeMillis, - // lastLicenseCheckResult: "Available").ConfigureAwait(false); + await TestHelpers.SimulateProsperRunningAsync(cdf, "scheduler-test-connector").ConfigureAwait(false); stateConfig = provider.GetRequiredService(); var configLib = provider.GetRequiredService(); @@ -235,7 +85,10 @@ public async Task TestSimulationSchedulerBaseWithApi() Assert.NotEmpty(simRuns.Items); // check if there are any simulation runs in the time span of the test - Assert.Contains(simRuns.Items, r => r.CreatedTime > testStartTimeMillis && r.CreatedTime < DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()); + // with the run type set to scheduled + var latestEventsFiltered = simRuns.Items.Where(r => r.CreatedTime >= testStartTimeMillis && r.CreatedTime < DateTimeOffset.UtcNow.ToUnixTimeMilliseconds()); + Assert.NotEmpty(latestEventsFiltered); + Assert.Contains(latestEventsFiltered, e => e.RunType == SimulationRunType.scheduled); } finally {