Merge remote-tracking branch 'origin/main' into bug/fhattinf/O2B-1139…

…/Refreshtagslistwhenatagisupdated/created

CHANGELOG.md

@@ -2,6 +2,25 @@
 
 All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines.
 
+## [1.4.1](https://github.com/AliceO2Group/Bookkeeping/releases/tag/%40aliceo2%2Fbookkeeping%401.4.1)
+* Notable changes for developers:
+  * Fixed the runs start/stop extraction from AliECS kafka messages
+  * Added more logs to external service synchronization (CCDB, MonALISA)
+
+## [1.4.0](https://github.com/AliceO2Group/Bookkeeping/releases/tag/%40aliceo2%2Fbookkeeping%401.4.0)
+* Notable changes for users:
+  * Fixed tag color not being updated when switching to run details after updating a tag
+  * Environment variable ENABLE_HOUSEKEEPING must be set to true (case-insensitive) to actually enable housekeeping
+  * Use time range picker for runs start & stop filtering
+  * Run details page has been modernized
+* Notable changes for developers:
+  * Runs and QC flags timestamps now store milliseconds
+  * Fixed users that start/stop runs not being extracted from kafka message
+  * Fixed TF timestamps being ignored when creating QC flags
+  * Fixed randomly failing test in FLP frontend tests
+  * Use coalesced run time in raw SQL querries
+  * Removed the max number of retries for kafka connection
+
 ## [1.3.0](https://github.com/AliceO2Group/Bookkeeping/releases/tag/%40aliceo2%2Fbookkeeping%401.3.0)
 * Notable changes for users:
   * Fixed physical constants values which resulted in wrong AVG center of mass energy

Dockerfile

@@ -22,7 +22,7 @@ RUN apk add --no-cache \
     freetype=2.13.0-r5 \
     freetype-dev=2.13.0-r5 \
     harfbuzz=7.3.0-r0 \
-    ca-certificates=20240226-r0
+    ca-certificates=20241121-r0
 
 # Tell Puppeteer to skip installing Chrome. We'll be using the installed package.
 ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true

database/CHANGELOG.md

@@ -1,3 +1,24 @@
+## [1.4.0](https://github.com/AliceO2Group/Bookkeeping/releases/tag/%40aliceo2%2Fbookkeeping%401.4.0)
+* Changes made to the database
+  * Following columns in runs table has been changed to Datetime(3):
+    * time_o2_start
+    * time_trg_start
+    * first_tf_timestamp
+    * last_tf_timestamp
+    * time_trg_end
+    * time_o2_end
+  * Following columns in quality_control_flags table has been changed to Datetime(3):
+    * from
+    * to
+    * created_at
+    * updated_at
+  * Following columns in quality_control_flag_effective_periods table has been changed to Datetime(3):
+    * from
+    * to
+    * created_at
+    * updated_at
+  * Added two more virtual columns to runs, rct_time_start and rct_time_end that coalesce first/last TF timestamps with run start/stop
+
 ## [1.3.0](https://github.com/AliceO2Group/Bookkeeping/releases/tag/%40aliceo2%2Fbookkeeping%401.3.0)
 * Changes made to the database
   * Fixed physical constants values in database

docker-compose.yml

@@ -24,7 +24,8 @@ services:
       FLP_INFOLOGGER_URL: "${ALI_ECS_GUI_URL:-http://localhost:8081}"
       QC_GUI_URL: "${ALI_ECS_GUI_URL:-http://localhost:8082}"
       ALI_FLP_INDEX_URL: "${ALI_ECS_GUI_URL:-http://localhost:80}"
-      CCDB_ENABLE_SYNCHRONIZATION: false
+      CCDB_ENABLE_SYNCHRONIZATION: "${CCDB_ENABLE_SYNCHRONIZATION:-false}"
+      CCDB_RUN_INFO_URL: "${CCDB_RUN_INFO_URL:-}"
     links:
       - database
     restart: unless-stopped

lib/application.js

@@ -48,7 +48,6 @@ class BookkeepingApplication {
             const kafkaClient = new Kafka({
                 clientId: 'bookkeeping',
                 brokers: KafkaConfig.brokers,
-                retry: { retries: 3 },
                 logLevel: logLevel.NOTHING,
             });
 
@@ -94,15 +93,9 @@ class BookkeepingApplication {
                 const ccdbSynchronizer = new CcdbSynchronizer(ccdbConfig.runInfoUrl);
                 this.scheduledProcessesManager.schedule(
                     // Sync runs a few sync period ago in case some synchronization failed
-                    async () => {
-                        try {
-                            await ccdbSynchronizer.syncFirstAndLastTf(new Date(Date.now() - ccdbConfig.synchronizationPeriod * 5));
-                        } catch (e) {
-                            this._logger.errorMessage(`Failed to synchronize runs first and last TF:\n ${e.stack}`);
-                        }
-                    },
+                    () => ccdbSynchronizer.syncFirstAndLastTf(new Date(Date.now() - ccdbConfig.synchronizationPeriod * 5)),
                     {
-                        wait: 3 * 1000,
+                        wait: 10 * 1000,
                         every: ccdbConfig.synchronizationPeriod,
                     },
                 );

lib/config/services.js

@@ -26,7 +26,7 @@ const {
 } = process.env ?? {};
 
 exports.services = {
-    enableHousekeeping: process.env?.ENABLE_HOUSEKEEPING ?? false,
+    enableHousekeeping: process.env?.ENABLE_HOUSEKEEPING?.toLowerCase() === 'true',
     aliEcsGui: {
         url: process.env?.ALI_ECS_GUI_URL || null,
         token: process.env?.ALI_ECS_GUI_TOKEN || null,
@@ -64,7 +64,7 @@ exports.services = {
 
     ccdb: {
         enableSynchronization: CCDB_ENABLE_SYNCHRONIZATION?.toLowerCase() === 'true',
-        synchronizationPeriod: Number(CCDB_SYNCHRONIZATION_PERIOD) || 24 * 60 * 60 * 1000, // 1h in milliseconds
+        synchronizationPeriod: Number(CCDB_SYNCHRONIZATION_PERIOD) || 24 * 60 * 60 * 1000, // 1d in milliseconds
         runInfoUrl: CCDB_RUN_INFO_URL,
     },
 };

lib/database/adapters/RunAdapter.js

@@ -140,6 +140,7 @@ class RunAdapter {
             tfFileSize,
             otherFileCount,
             otherFileSize,
+            nTfOrbits,
             lhcFill,
             flpRoles,
             logs,
@@ -214,6 +215,7 @@ class RunAdapter {
             tfFileSize: tfFileSize !== null ? String(tfFileSize) : null,
             otherFileCount,
             otherFileSize: otherFileSize !== null ? String(otherFileSize) : null,
+            nTfOrbits: nTfOrbits !== null ? String(nTfOrbits) : null,
             lhcFill,
             crossSection,
             triggerEfficiency,
@@ -322,6 +324,7 @@ class RunAdapter {
             tfFileSize: entityObject.tfFileSize,
             otherFileCount: entityObject.otherFileCount,
             otherFileSize: entityObject.otherFileSize,
+            nTfOrbits: entityObject.nTfOrbits,
             concatenatedDetectors: entityObject.detectors,
             definition: entityObject.definition,
             calibrationStatus: entityObject.calibrationStatus,

lib/database/migrations/20241212074001-create-rct-run-start-and-stop.js

@@ -0,0 +1,17 @@
+'use strict';
+
+const ADD_RUN_RCT_START_AND_STOP = `
+    ALTER TABLE runs
+        ADD COLUMN qc_time_start DATETIME(3) AS (COALESCE(first_tf_timestamp, time_trg_start, time_o2_start)) VIRTUAL AFTER time_start,
+        ADD COLUMN qc_time_end   DATETIME(3) AS (COALESCE(last_tf_timestamp, time_trg_end, time_o2_end)) VIRTUAL AFTER qc_time_start;
+`;
+
+/** @type {import('sequelize-cli').Migration} */
+module.exports = {
+    up: async (queryInterface) => queryInterface.sequelize.query(ADD_RUN_RCT_START_AND_STOP),
+
+    down: async (queryInterface) => queryInterface.sequelize.transaction(async (transaction) => {
+        await queryInterface.removeColumn('runs', 'qc_time_start', { transaction });
+        await queryInterface.removeColumn('runs', 'qc_time_end', { transaction });
+    }),
+};

lib/database/migrations/20241218095948-add-n-tf-orbits.js

@@ -0,0 +1,17 @@
+'use strict';
+
+/** @type {import('sequelize-cli').Migration} */
+module.exports = {
+    up: async (queryInterface, Sequelize) => await queryInterface.addColumn(
+        'runs',
+        'n_tf_orbits',
+        {
+            type: Sequelize.DataTypes.BIGINT,
+            allowNull: true,
+            default: null,
+            after: 'other_file_size',
+        },
+    ),
+
+    down: async (queryInterface) => await queryInterface.removeColumn('runs', 'n_tf_orbits'),
+};

lib/database/models/run.js

@@ -203,6 +203,10 @@ module.exports = (sequelize) => {
             type: Sequelize.BIGINT,
             default: null,
         },
+        nTfOrbits: {
+            type: Sequelize.BIGINT,
+            default: null,
+        },
         definition: {
             type: Sequelize.ENUM(...RUN_DEFINITIONS),
         },

lib/database/models/typedefs/SequelizeRun.js

@@ -67,6 +67,7 @@
  * @property {string|number|null} tfFileSize
  * @property {string|null} otherFileCount
  * @property {string|number|null} otherFileSize
+ * @property {string|number|null} nTfOrbits
  * @property {number|null} inelasticInteractionRateAvg
  * @property {number|null} inelasticInteractionRateAtStart
  * @property {number|null} inelasticInteractionRateAtMid

lib/database/repositories/QcFlagRepository.js

@@ -203,32 +203,17 @@ class QcFlagRepository extends Repository {
                 GROUP_CONCAT(effectivePeriods.flagsList) AS flagsList,
 
                 IF(
-                    (
-                        COALESCE(run.time_trg_end,   run.time_o2_end  ) IS NULL
-                     OR COALESCE(run.time_trg_start, run.time_o2_start) IS NULL
-                    ),
+                    run.time_start IS NULL OR run.time_end IS NULL,
                     IF(
-                        SUM(
-                            COALESCE(effectivePeriods.\`to\`  , 0)
-                          + COALESCE(effectivePeriods.\`from\`, 0)
-                        ) = 0,
+                        effectivePeriods.\`from\` IS NULL AND effectivePeriods.\`to\`  IS NULL,
                         1,
                         null
                     ),
                     SUM(
-                        COALESCE(
-                            effectivePeriods.\`to\`,
-                            UNIX_TIMESTAMP(run.time_trg_end),
-                            UNIX_TIMESTAMP(run.time_o2_end)
-                        )
-                      - COALESCE(
-                            effectivePeriods.\`from\`,
-                            UNIX_TIMESTAMP(run.time_trg_start),
-                            UNIX_TIMESTAMP(run.time_o2_start)
-                        )
+                        COALESCE(effectivePeriods.\`to\`, UNIX_TIMESTAMP(run.time_end))
+                      - COALESCE(effectivePeriods.\`from\`, UNIX_TIMESTAMP(run.time_start))
                     ) / (
-                        UNIX_TIMESTAMP(COALESCE(run.time_trg_end,   run.time_o2_end))
-                      - UNIX_TIMESTAMP(COALESCE(run.time_trg_start, run.time_o2_start))
+                        UNIX_TIMESTAMP(run.time_end) - UNIX_TIMESTAMP(run.time_start)
                     )
                 ) AS effectiveRunCoverage
 

lib/database/seeders/20200713103855-runs.js

@@ -53,6 +53,7 @@ module.exports = {
                     tf_file_size: '214920239535280',
                     other_file_count: 50,
                     other_file_size: '9999999999999999',
+                    n_tf_orbits: '9999999999999999',
                     definition: RunDefinition.PHYSICS,
                     cross_section: 1.23,
                     trigger_efficiency: 2.34,

lib/domain/dtos/UpdateRunByRunNumberDto.js

@@ -37,6 +37,7 @@ const BodyDto = Joi.object({
     tfFileSize: Joi.string().optional(),
     otherFileCount: Joi.number().optional(),
     otherFileSize: Joi.string().optional(),
+    nTfOrbits: Joi.string().optional(),
     crossSection: Joi.number().optional(),
     triggerEfficiency: Joi.number().optional(),
     triggerAcceptance: Joi.number().optional(),

lib/domain/entities/Run.js

@@ -65,6 +65,7 @@
  * @property {string|null} tfFileSize
  * @property {string|null} otherFileCount
  * @property {string|null} otherFileSize
+ * @property {string|null} nTfOrbits
  * @property {number|null} muInelasticInteractionRate
  * @property {number|null} inelasticInteractionRateAvg
  * @property {number|null} inelasticInteractionRateAtStart

lib/public/views/Runs/Details/editTagsPanel.js → lib/public/components/Filters/RunsFilter/o2StartFilter.js

@@ -11,17 +11,12 @@
  * or submit itself to any jurisdiction.
  */
 
-import { h } from '/js/src/index.js';
-import { tagPicker } from '../../../components/tag/tagPicker.js';
+import { timeRangeFilter } from '../common/filters/timeRangeFilter.js';
 
 /**
- * A panel containing:
- * a list of potential tags to add to a RUN
- * a button to update the tag selection
- * @param {RunDetailsModel} runDetailsModel the details model
- * @return {vnode} virtual node with representation of the panel
+ * Returns a filter to be applied on run start
+ *
+ * @param {RunsOverviewModel} runsOverviewModel the run overview model object
+ * @return {Component} the filter component
  */
-export const editTagsPanel = (runDetailsModel) => h(
-    '#tags-selection.flex-column.w-30.p2.g2',
-    tagPicker(runDetailsModel.editionTagPickerModel),
-);
+export const o2StartFilter = (runsOverviewModel) => timeRangeFilter(runsOverviewModel.o2StartFilterModel);

lib/public/components/Filters/RunsFilter/o2StopFilter.js

@@ -0,0 +1,21 @@
+/**
+ *  @license
+ *  Copyright CERN and copyright holders of ALICE O2. This software is
+ *  distributed under the terms of the GNU General Public License v3 (GPL
+ *  Version 3), copied verbatim in the file "COPYING".
+ *
+ *  See http://alice-o2.web.cern.ch/license for full licensing information.
+ *
+ *  In applying this license CERN does not waive the privileges and immunities
+ *  granted to it by virtue of its status as an Intergovernmental Organization
+ *  or submit itself to any jurisdiction.
+ */
+import { timeRangeFilter } from '../common/filters/timeRangeFilter.js';
+
+/**
+ * Returns a filter to be applied on run stop
+ *
+ * @param {RunsOverviewModel} runsOverviewModel the run overview model object
+ * @return {Component} the filter component
+ */
+export const o2StopFilter = (runsOverviewModel) => timeRangeFilter(runsOverviewModel.o2stopFilterModel);