diff --git a/lib/database/migrations/v1/20250518123000-create-gaq-views.js b/lib/database/migrations/v1/20250518123000-create-gaq-views.js new file mode 100644 index 0000000000..65e4bb3532 --- /dev/null +++ b/lib/database/migrations/v1/20250518123000-create-gaq-views.js @@ -0,0 +1,168 @@ +'use strict'; + +const SELECT_RUNS_START_TIMESTAMPS_FOR_GAQ_PERIODS = ` + SELECT + gaqd.data_pass_id, + gaqd.run_number, + r.qc_time_start AS timestamp, + COALESCE(r.qc_time_start, '0001-01-01 00:00:00.000') AS ordering_timestamp, + r.qc_time_start AS qc_run_start, + r.qc_time_end AS qc_run_end + FROM global_aggregated_quality_detectors AS gaqd + INNER JOIN runs as r ON gaqd.run_number = r.run_number + INNER JOIN quality_control_flags AS qcf ON qcf.run_number = r.run_number + INNER JOIN data_pass_quality_control_flag AS dpqcf + ON dpqcf.quality_control_flag_id = qcf.id AND dpqcf.data_pass_id = gaqd.data_pass_id +`; + +const SELECT_RUNS_END_TIMESTAMPS_FOR_GAQ_PERIODS = ` + SELECT + gaqd.data_pass_id, + gaqd.run_number, + r.qc_time_end AS timestamp, + COALESCE(r.qc_time_end, NOW(3)) AS ordering_timestamp, + r.qc_time_start AS qc_run_start, + r.qc_time_end AS qc_run_end + FROM global_aggregated_quality_detectors AS gaqd + INNER JOIN runs as r ON gaqd.run_number = r.run_number + INNER JOIN quality_control_flags AS qcf ON qcf.run_number = r.run_number + INNER JOIN data_pass_quality_control_flag AS dpqcf + ON dpqcf.quality_control_flag_id = qcf.id AND dpqcf.data_pass_id = gaqd.data_pass_id +`; + +const SELECT_QCF_EFFECTIVE_PERIODS_START_TIMESTAMPS_FOR_GAQ_PERIODS = ` + SELECT gaqd.data_pass_id, + gaqd.run_number, + COALESCE(qcfep.\`from\`, r.qc_time_start) AS timestamp, + COALESCE(qcfep.\`from\`, r.qc_time_start, '0001-01-01 00:00:00.000') AS ordering_timestamp, + r.qc_time_start AS qc_run_start, + r.qc_time_end AS qc_run_end + FROM quality_control_flag_effective_periods AS qcfep + INNER JOIN quality_control_flags AS qcf ON qcf.id = qcfep.flag_id + INNER JOIN runs AS r ON qcf.run_number = r.run_number + INNER JOIN data_pass_quality_control_flag AS dpqcf ON dpqcf.quality_control_flag_id = qcf.id + -- Only flags of detectors which are defined in global_aggregated_quality_detectors + -- should be taken into account for calculation of gaq_effective_periods + INNER JOIN global_aggregated_quality_detectors AS gaqd + ON gaqd.data_pass_id = dpqcf.data_pass_id + AND gaqd.run_number = qcf.run_number + AND gaqd.detector_id = qcf.detector_id +`; + +const SELECT_QCF_EFFECTIVE_PERIODS_END_TIMESTAMPS_FOR_GAQ_PERIODS = ` + SELECT gaqd.data_pass_id, + gaqd.run_number, + COALESCE(qcfep.\`to\`, r.qc_time_end) AS timestamp, + COALESCE(qcfep.\`to\`, r.qc_time_end, NOW(3)) AS ordering_timestamp, + r.qc_time_start AS qc_run_start, + r.qc_time_end AS qc_run_end + FROM quality_control_flag_effective_periods AS qcfep + INNER JOIN quality_control_flags AS qcf ON qcf.id = qcfep.flag_id + INNER JOIN runs AS r ON qcf.run_number = r.run_number + INNER JOIN data_pass_quality_control_flag AS dpqcf ON dpqcf.quality_control_flag_id = qcf.id + -- Only flags of detectors which are defined in global_aggregated_quality_detectors + -- should be taken into account for calculation of gaq_effective_periods + INNER JOIN global_aggregated_quality_detectors AS gaqd + ON gaqd.data_pass_id = dpqcf.data_pass_id + AND gaqd.run_number = qcf.run_number + AND gaqd.detector_id = qcf.detector_id +`; + +const CREATE_GAQ_PERIODS_VIEW = ` +CREATE OR REPLACE VIEW gaq_periods AS + SELECT + data_pass_id, + run_number, + \`from\`, + \`to\`, + from_ordering_timestamp, + (UNIX_TIMESTAMP(\`to\`) - UNIX_TIMESTAMP(\`from\`)) / (UNIX_TIMESTAMP(qc_run_end) - UNIX_TIMESTAMP(qc_run_start)) AS coverage_ratio + FROM ( + SELECT + data_pass_id, + run_number, + LAG(timestamp) OVER w AS \`from\`, + timestamp AS \`to\`, + LAG(ordering_timestamp) OVER w AS from_ordering_timestamp, + qc_run_start, + qc_run_end + FROM ( + -- Two selects for runs' timestamps (in case QC flag's eff. period doesn't start at run's start or end at run's end ) + ( ${SELECT_RUNS_START_TIMESTAMPS_FOR_GAQ_PERIODS} ) + UNION + ( ${SELECT_RUNS_END_TIMESTAMPS_FOR_GAQ_PERIODS} ) + UNION + -- Two selects for timestamps of QC flags' effective periods + ( ${SELECT_QCF_EFFECTIVE_PERIODS_START_TIMESTAMPS_FOR_GAQ_PERIODS} ) + UNION + ( ${SELECT_QCF_EFFECTIVE_PERIODS_END_TIMESTAMPS_FOR_GAQ_PERIODS} ) + + ORDER BY ordering_timestamp + ) AS ap + WINDOW w AS ( + PARTITION BY data_pass_id, + run_number + ORDER BY ap.ordering_timestamp + ) + ) as gaq_periods_with_last_nullish_row + WHERE gaq_periods_with_last_nullish_row.from_ordering_timestamp IS NOT NULL +`; + +const DROP_GAQ_PERIODS_VIEW = 'DROP VIEW gaq_periods'; + +const CREATE_QC_FLAG_BLOCK_SIGNIFCANCE_AGGREGATE_FUNCTION = ` + CREATE OR REPLACE AGGREGATE FUNCTION qc_flag_block_significance( + row_bad TINYINT(1), + row_mc_reproducible TINYINT(1) + ) RETURNS ENUM ('bad', 'mcr', 'good') + BEGIN + DECLARE mc_reproducible TINYINT(1) DEFAULT 0; + DECLARE bad TINYINT(1) DEFAULT 0; + DECLARE CONTINUE HANDLER FOR NOT FOUND RETURN IF(bad, 'bad', IF(mc_reproducible, 'mcr', 'good')); + LOOP + FETCH group NEXT ROW; + IF row_mc_reproducible THEN + SET mc_reproducible = 1; + ELSEIF row_bad THEN + SET bad = 1; + END IF; + END LOOP; + END +`; + +const DROP_QC_FLAG_BLOCK_SIGNIFCANCE_AGGREGATE_FUNCTION = 'DROP FUNCTION qc_flag_block_significance'; + +const CREATE_QC_FLAG_BLOCK_SIGNIFCANCE_COVERAGE_AGGREGATE_FUNCTION = ` + CREATE OR REPLACE AGGREGATE FUNCTION qc_flag_block_significance_coverage( + row_significance ENUM ('bad', 'mcr', 'good'), -- The significance of the row + coverage_ratio FLOAT, -- The coverage ratio of the row + significance ENUM ('bad', 'mcr', 'good') -- The significance to aggregate over + ) RETURNS FLOAT + BEGIN + DECLARE coverage FLOAT DEFAULT 0; + DECLARE CONTINUE HANDLER FOR NOT FOUND RETURN coverage; + LOOP + FETCH group NEXT ROW; + IF row_significance = significance THEN + SET coverage = coverage + coverage_ratio; + END IF; + END LOOP; + END +`; + +const DROP_QC_FLAG_BLOCK_SIGNIFCANCE_COVERAGE_AGGREGATE_FUNCTION = 'DROP FUNCTION qc_flag_block_significance_coverage'; + +/** @type {import('sequelize-cli').Migration} */ +module.exports = { + up: async (queryInterface) => queryInterface.sequelize.transaction(async (transaction) => { + await queryInterface.sequelize.query(CREATE_GAQ_PERIODS_VIEW, { transaction }); + await queryInterface.sequelize.query(CREATE_QC_FLAG_BLOCK_SIGNIFCANCE_AGGREGATE_FUNCTION, { transaction }); + await queryInterface.sequelize.query(CREATE_QC_FLAG_BLOCK_SIGNIFCANCE_COVERAGE_AGGREGATE_FUNCTION, { transaction }); + }), + + down: async (queryInterface) => queryInterface.sequelize.transaction(async (transaction) => { + await queryInterface.sequelize.query(DROP_GAQ_PERIODS_VIEW, { transaction }); + await queryInterface.sequelize.query(DROP_QC_FLAG_BLOCK_SIGNIFCANCE_AGGREGATE_FUNCTION, { transaction }); + await queryInterface.sequelize.query(DROP_QC_FLAG_BLOCK_SIGNIFCANCE_COVERAGE_AGGREGATE_FUNCTION, { transaction }); + }), +}; diff --git a/lib/database/repositories/QcFlagRepository.js b/lib/database/repositories/QcFlagRepository.js index d5d4d3d1be..91f310efc4 100644 --- a/lib/database/repositories/QcFlagRepository.js +++ b/lib/database/repositories/QcFlagRepository.js @@ -15,59 +15,6 @@ const { Op } = require('sequelize'); const { models: { QcFlag } } = require('..'); const Repository = require('./Repository'); -const GAQ_PERIODS_VIEW = ` - SELECT * FROM ( - SELECT - data_pass_id, - run_number, - LAG(timestamp) OVER w AS \`from\`, - timestamp AS \`to\`, - LAG(ordering_timestamp) OVER w AS from_ordering_timestamp - FROM ( - ( - SELECT gaqd.data_pass_id, - gaqd.run_number, - COALESCE(qcfep.\`from\`, r.qc_time_start) AS timestamp, - COALESCE(qcfep.\`from\`, r.qc_time_start, '0001-01-01 00:00:00.000') AS ordering_timestamp - FROM quality_control_flag_effective_periods AS qcfep - INNER JOIN quality_control_flags AS qcf ON qcf.id = qcfep.flag_id - INNER JOIN runs AS r ON qcf.run_number = r.run_number - INNER JOIN data_pass_quality_control_flag AS dpqcf ON dpqcf.quality_control_flag_id = qcf.id - -- Only flags of detectors which are defined in global_aggregated_quality_detectors - -- should be taken into account for calculation of gaq_effective_periods - INNER JOIN global_aggregated_quality_detectors AS gaqd - ON gaqd.data_pass_id = dpqcf.data_pass_id - AND gaqd.run_number = qcf.run_number - AND gaqd.detector_id = qcf.detector_id - ) - UNION - ( - SELECT gaqd.data_pass_id, - gaqd.run_number, - COALESCE(qcfep.\`to\`, r.qc_time_end) AS timestamp, - COALESCE(qcfep.\`to\`, r.qc_time_end, NOW()) AS ordering_timestamp - FROM quality_control_flag_effective_periods AS qcfep - INNER JOIN quality_control_flags AS qcf ON qcf.id = qcfep.flag_id - INNER JOIN runs AS r ON qcf.run_number = r.run_number - INNER JOIN data_pass_quality_control_flag AS dpqcf ON dpqcf.quality_control_flag_id = qcf.id - -- Only flags of detectors which are defined in global_aggregated_quality_detectors - -- should be taken into account for calculation of gaq_effective_periods - INNER JOIN global_aggregated_quality_detectors AS gaqd - ON gaqd.data_pass_id = dpqcf.data_pass_id - AND gaqd.run_number = qcf.run_number - AND gaqd.detector_id = qcf.detector_id - ) - ORDER BY ordering_timestamp - ) AS ap - WINDOW w AS ( - PARTITION BY data_pass_id, - run_number - ORDER BY ap.ordering_timestamp - ) - ) as gaq_periods_with_last_nullish_row - WHERE gaq_periods_with_last_nullish_row.from_ordering_timestamp IS NOT NULL - `; - /** * @typedef GaqPeriod * @@ -79,14 +26,14 @@ const GAQ_PERIODS_VIEW = ` */ /** - * @typedef RunGaqSubSummary aggregation of QC flags information by QcFlagType property `bad` + * @typedef RunGaqSubSummary aggregation of QC flags information by QcFlagType property `bad` and `mc_reproducible` * - * @property {number} runNumber - * @property {number} bad - * @property {number} effectiveRunCoverage + * @property {number} badCoverage + * @property {number} mcReproducibleCoverage + * @property {number} goodCoverage + * @property {number} totalCoverage * @property {number[]} flagsIds * @property {number[]} verifiedFlagsIds - * @property {number} mcReproducible */ /** @@ -117,17 +64,16 @@ class QcFlagRepository extends Repository { group_concat(qcf.id) AS contributingFlagIds FROM quality_control_flags AS qcf - INNER JOIN quality_control_flag_effective_periods AS qcfep - ON qcf.id = qcfep.flag_id + INNER JOIN quality_control_flag_effective_periods AS qcfep ON qcf.id = qcfep.flag_id INNER JOIN data_pass_quality_control_flag AS dpqcf ON dpqcf.quality_control_flag_id = qcf.id - INNER JOIN (${GAQ_PERIODS_VIEW}) AS gaq_periods ON gaq_periods.data_pass_id = dpqcf.data_pass_id + INNER JOIN gaq_periods ON gaq_periods.data_pass_id = dpqcf.data_pass_id INNER JOIN global_aggregated_quality_detectors AS gaqd ON gaqd.data_pass_id = gaq_periods.data_pass_id - AND gaqd.run_number = gaq_periods.run_number - AND gaqd.detector_id = qcf.detector_id - AND gaq_periods.run_number = qcf.run_number - AND (qcfep.\`from\` IS NULL OR qcfep.\`from\` <= gaq_periods.\`from\`) - AND (qcfep.\`to\` IS NULL OR gaq_periods.\`to\` <= qcfep.\`to\`) + AND gaqd.run_number = gaq_periods.run_number + AND gaqd.detector_id = qcf.detector_id + AND gaq_periods.run_number = qcf.run_number + AND (qcfep.\`from\` IS NULL OR qcfep.\`from\` <= gaq_periods.\`from\`) + AND (qcfep.\`to\` IS NULL OR gaq_periods.\`to\` <= qcfep.\`to\`) WHERE gaq_periods.data_pass_id = ${dataPassId} ${runNumber ? `AND gaq_periods.run_number = ${runNumber}` : ''} @@ -155,107 +101,78 @@ class QcFlagRepository extends Repository { } /** - * Get GAQ sub-summaries for given data pass + * Return the good, bad and MC reproducible coverage per runs for a given data pass + * and informtion about missing and unverified flags * - * @param {number} dataPassId id of data pass id - * @param {object} [options] additional options - * @param {boolean} [options.mcReproducibleAsNotBad = false] if set to true, - * `Limited Acceptance MC Reproducible` flag type is treated as good one - * @return {Promise} Resolves with the GAQ sub-summaries + * @param {number} dataPassId the id of a data-pass + * @return {Promise>} resolves with the map between run number and the corresponding run GAQ summary */ - async getRunGaqSubSummaries(dataPassId, { mcReproducibleAsNotBad = false } = {}) { - const effectivePeriodsWithTypeSubQuery = ` - SELECT - gaq_periods.data_pass_id AS dataPassId, - gaq_periods.run_number AS runNumber, - gaq_periods.\`from\` AS \`from\`, - gaq_periods.\`to\` AS \`to\`, - SUM(IF(qcft.monte_carlo_reproducible AND :mcReproducibleAsNotBad, false, qcft.bad)) >= 1 AS bad, - SUM(qcft.bad) = SUM(qcft.monte_carlo_reproducible) AND SUM(qcft.monte_carlo_reproducible) AS mcReproducible, - GROUP_CONCAT( DISTINCT qcfv.flag_id ) AS verifiedFlagsList, - GROUP_CONCAT( DISTINCT qcf.id ) AS flagsList - - FROM quality_control_flags AS qcf - INNER JOIN quality_control_flag_types AS qcft - ON qcft.id = qcf.flag_type_id - LEFT JOIN quality_control_flag_verifications AS qcfv - ON qcfv.flag_id = qcf.id - INNER JOIN quality_control_flag_effective_periods AS qcfep - ON qcf.id = qcfep.flag_id - INNER JOIN data_pass_quality_control_flag AS dpqcf - ON dpqcf.quality_control_flag_id = qcf.id - INNER JOIN (${GAQ_PERIODS_VIEW}) AS gaq_periods - ON gaq_periods.data_pass_id = dpqcf.data_pass_id - INNER JOIN global_aggregated_quality_detectors AS gaqd - ON gaqd.data_pass_id = gaq_periods.data_pass_id - AND gaqd.run_number = gaq_periods.run_number - AND gaqd.detector_id = qcf.detector_id - AND gaq_periods.run_number = qcf.run_number - AND (qcfep.\`from\` IS NULL OR qcfep.\`from\` <= gaq_periods.\`from\`) - AND (qcfep.\`to\` IS NULL OR gaq_periods.\`to\` <= qcfep.\`to\`) - - GROUP BY + async getGaqCoverages(dataPassId) { + const blockAggregationQuery = ` + SELECT gaq_periods.data_pass_id, gaq_periods.run_number, - gaq_periods.\`from\`, - gaq_periods.\`to\` - `; + gaq_periods.coverage_ratio, + IF(COUNT(DISTINCT qcf.id) > 0, qc_flag_block_significance(qcft.bad, qcft.monte_carlo_reproducible), NULL) AS significance, + GROUP_CONCAT( DISTINCT qcfv.flag_id ) AS verified_flags_list, + GROUP_CONCAT( DISTINCT qcfep.flag_id ) AS flags_list - const query = ` - SELECT - effectivePeriods.runNumber, - effectivePeriods.dataPassId, - effectivePeriods.bad, - SUM(effectivePeriods.mcReproducible) > 0 AS mcReproducible, - GROUP_CONCAT(effectivePeriods.verifiedFlagsList) AS verifiedFlagsList, - GROUP_CONCAT(effectivePeriods.flagsList) AS flagsList, - - IF( - run.qc_time_start IS NULL OR run.qc_time_end IS NULL, - IF( - effectivePeriods.\`from\` IS NULL AND effectivePeriods.\`to\` IS NULL, - 1, - null - ), - SUM( - UNIX_TIMESTAMP(COALESCE(effectivePeriods.\`to\`,run.qc_time_end)) - - UNIX_TIMESTAMP(COALESCE(effectivePeriods.\`from\`, run.qc_time_start)) - ) / (UNIX_TIMESTAMP(run.qc_time_end) - UNIX_TIMESTAMP(run.qc_time_start)) - ) AS effectiveRunCoverage - - FROM (${effectivePeriodsWithTypeSubQuery}) AS effectivePeriods - INNER JOIN runs AS run ON run.run_number = effectivePeriods.runNumber - - WHERE effectivePeriods.dataPassId = :dataPassId + FROM gaq_periods - GROUP BY - effectivePeriods.dataPassId, - effectivePeriods.runNumber, - effectivePeriods.bad + INNER JOIN global_aggregated_quality_detectors AS gaqd + ON gaqd.data_pass_id = gaq_periods.data_pass_id + AND gaqd.run_number = gaq_periods.run_number + + LEFT JOIN ( + data_pass_quality_control_flag AS dpqcf + INNER JOIN quality_control_flags AS qcf ON dpqcf.quality_control_flag_id = qcf.id + INNER JOIN quality_control_flag_types AS qcft ON qcft.id = qcf.flag_type_id + INNER JOIN quality_control_flag_effective_periods AS qcfep ON qcf.id = qcfep.flag_id + LEFT JOIN quality_control_flag_verifications AS qcfv ON qcfv.flag_id = qcf.id + ) + ON gaq_periods.data_pass_id = dpqcf.data_pass_id + AND qcf.run_number = gaq_periods.run_number + AND gaqd.detector_id = qcf.detector_id + AND gaq_periods.run_number = qcf.run_number + AND (qcfep.from IS NULL OR qcfep.\`from\` < gaq_periods.\`to\`) + AND (qcfep.to IS NULL OR qcfep.\`to\` > gaq_periods.\`from\`) + + WHERE gaq_periods.data_pass_id = :dataPassId + GROUP BY gaq_periods.data_pass_id, gaq_periods.run_number, gaq_periods.\`from\`, gaq_periods.to `; - const [rows] = await this.model.sequelize.query(query, { replacements: { dataPassId, mcReproducibleAsNotBad } }); - return rows.map(({ - runNumber, - bad, - effectiveRunCoverage, - mcReproducible, - flagsList, - verifiedFlagsList, - }) => { - if ((effectiveRunCoverage ?? null) != null) { - effectiveRunCoverage = Math.min(1, Math.max(0, parseFloat(effectiveRunCoverage))); - } - - return { - runNumber, - bad, - effectiveRunCoverage, - mcReproducible: Boolean(mcReproducible), - flagsIds: [...new Set(flagsList.split(','))], - verifiedFlagsIds: verifiedFlagsList ? [...new Set(verifiedFlagsList.split(','))] : [], - }; - }); + const summaryQuery = ` + SELECT + data_pass_id, + run_number, + qc_flag_block_significance_coverage(gaq.significance, coverage_ratio, 'bad') AS bad_coverage, + qc_flag_block_significance_coverage(gaq.significance, coverage_ratio, 'mcr') AS mcr_coverage, + qc_flag_block_significance_coverage(gaq.significance, coverage_ratio, 'good') AS good_coverage, + GROUP_CONCAT(verified_flags_list) AS verified_flags_list, + GROUP_CONCAT(flags_list) AS flags_list + + FROM (${blockAggregationQuery}) AS gaq + GROUP BY gaq.data_pass_id, gaq.run_number; + `; + const [rows] = await this.model.sequelize.query(summaryQuery, { replacements: { dataPassId } }); + const entries = rows.map(({ + run_number, + bad_coverage, + mcr_coverage, + good_coverage, + flags_list, + verifiedd_flags_list, + }) => [ + run_number, + { + badCoverage: parseFloat(bad_coverage ?? '0'), + mcReproducibleCoverage: parseFloat(mcr_coverage ?? '0'), + goodCoverage: parseFloat(good_coverage ?? '0'), + flagsIds: [...new Set(flags_list?.split(','))], + verifiedFlagsIds: [...new Set(verifiedd_flags_list?.split(','))], + }, + ]); + return Object.fromEntries(entries); } /** diff --git a/lib/domain/entities/QcSummary.js b/lib/domain/entities/QcSummary.js new file mode 100644 index 0000000000..c4ba113904 --- /dev/null +++ b/lib/domain/entities/QcSummary.js @@ -0,0 +1,32 @@ +/** + * @typedef RunDetectorQcSummary + * + * @property {number} badEffectiveRunCoverage - fraction of run's data, marked explicitly with bad QC flag + * @property {number} explicitlyNotBadEffectiveRunCoverage - fraction of run's data, marked explicitly with good QC flag + * @property {boolean} mcReproducible - if true states that some Limited Acceptance MC Reproducible flag was assigned + * @property {number} missingVerificationsCount - number of QC flags that are unverified and have not been discarded + */ + +/** + * @typedef {Object.} RunQcSummary + * detectorId mapping to RunDetectorQcSummary + */ + +/** + * @typedef {Object.} QcSummary + * runNumber mapping to RunQcSummary + */ + +/** + * @typedef GaqRunSummary + * + * @property {number} badEffectiveRunCoverage - fraction of run's aggregated quality interpreted as bad + * @property {number} explicitlyNotBadEffectiveRunCoverage - fraction of run's aggregated quality interpreted as not-bad + * @property {boolean} mcReproducible - if true states that some of periods have aggregated quality 'Mc Reproducible' + * @property {number} missingVerificationsCount - number of QC flags that are unverified and have not been discarded + */ + +/** + * @typedef {Object.} GaqSummary + * runNumber mapping to GaqRunSummary + */ diff --git a/lib/domain/enums/QcSummaryProperties.js b/lib/domain/enums/QcSummaryProperties.js new file mode 100644 index 0000000000..027f85d991 --- /dev/null +++ b/lib/domain/enums/QcSummaryProperties.js @@ -0,0 +1,19 @@ +/** + * @license + * Copyright CERN and copyright holders of ALICE O2. This software is + * distributed under the terms of the GNU General Public License v3 (GPL + * Version 3), copied verbatim in the file "COPYING". + * + * See http://alice-o2.web.cern.ch/license for full licensing information. + * + * In applying this license CERN does not waive the privileges and immunities + * granted to it by virtue of its status as an Intergovernmental Organization + * or submit itself to any jurisdiction. + */ + +exports.QcSummarProperties = { + BAD_EFFECTIVE_RUN_COVERAGE: 'badEffectiveRunCoverage', + EXPLICITELY_NOT_BAD_EFFECTIVE_RUN_COVERAGE: 'explicitlyNotBadEffectiveRunCoverage', + MISSING_VERIFICATIONS: 'missingVerificationsCount', + MC_REPRODUCIBLE: 'mcReproducible', +}; diff --git a/lib/server/services/qualityControlFlag/GaqService.js b/lib/server/services/qualityControlFlag/GaqService.js index b9d4551a4a..c9857834f4 100644 --- a/lib/server/services/qualityControlFlag/GaqService.js +++ b/lib/server/services/qualityControlFlag/GaqService.js @@ -29,21 +29,9 @@ const { getOneDataPassOrFail } = require('../dataPasses/getOneDataPassOrFail.js'); const { QcFlagRepository } = require('../../../database/repositories/index.js'); -const { QcFlagSummaryService } = require('./QcFlagSummaryService.js'); const { qcFlagAdapter } = require('../../../database/adapters/index.js'); const { Op } = require('sequelize'); - -/** - * @typedef GaqSummary aggregated global quality summaries for given data pass - * @type {Object} runNumber to RunGaqSummary mapping - */ - -const QC_SUMMARY_PROPERTIES = { - badEffectiveRunCoverage: 'badEffectiveRunCoverage', - explicitlyNotBadEffectiveRunCoverage: 'explicitlyNotBadEffectiveRunCoverage', - missingVerificationsCount: 'missingVerificationsCount', - mcReproducible: 'mcReproducible', -}; +const { QcSummarProperties } = require('../../../domain/enums/QcSummaryProperties.js'); /** * Globally aggregated quality (QC flags aggregated for a predefined list of detectors per runs) service @@ -56,48 +44,32 @@ class GaqService { * @param {object} [options] additional options * @param {boolean} [options.mcReproducibleAsNotBad = false] if set to true, * `Limited Acceptance MC Reproducible` flag type is treated as good one - * @return {Promise} Resolves with the GAQ Summary + * @return {Promise} Resolves with the GAQ Summary */ async getSummary(dataPassId, { mcReproducibleAsNotBad = false } = {}) { await getOneDataPassOrFail({ id: dataPassId }); - const runGaqSubSummaries = await QcFlagRepository.getRunGaqSubSummaries(dataPassId, { mcReproducibleAsNotBad }); - - const summary = {}; - const flagsAndVerifications = {}; - - // Fold list of subSummaries into one summary - for (const subSummary of runGaqSubSummaries) { - const { - runNumber, + const gaqCoverages = await QcFlagRepository.getGaqCoverages(dataPassId); + const gaqSummary = Object.entries(gaqCoverages).map(([ + runNumber, + { + badCoverage, + mcReproducibleCoverage, + goodCoverage, flagsIds, verifiedFlagsIds, - } = subSummary; - - if (!summary[runNumber]) { - summary[runNumber] = { [QC_SUMMARY_PROPERTIES.mcReproducible]: false }; - } - if (!flagsAndVerifications[runNumber]) { - flagsAndVerifications[runNumber] = {}; - } - - const runSummary = summary[runNumber]; - - const distinctRunFlagsIds = flagsAndVerifications[runNumber]?.distinctFlagsIds ?? []; - const distinctRunVerifiedFlagsIds = flagsAndVerifications[runNumber]?.distinctVerifiedFlagsIds ?? []; - - flagsAndVerifications[runNumber] = { - distinctFlagsIds: new Set([...distinctRunFlagsIds, ...flagsIds]), - distinctVerifiedFlagsIds: new Set([...distinctRunVerifiedFlagsIds, ...verifiedFlagsIds]), - }; - - QcFlagSummaryService.mergeIntoSummaryUnit(runSummary, subSummary); - } - - for (const [runNumber, { distinctFlagsIds, distinctVerifiedFlagsIds }] of Object.entries(flagsAndVerifications)) { - summary[runNumber][QC_SUMMARY_PROPERTIES.missingVerificationsCount] = distinctFlagsIds.size - distinctVerifiedFlagsIds.size; - } - - return summary; + }, + ]) => [ + runNumber, + { + [QcSummarProperties.BAD_EFFECTIVE_RUN_COVERAGE]: badCoverage + (mcReproducibleAsNotBad ? 0 : mcReproducibleCoverage), + [QcSummarProperties.EXPLICITELY_NOT_BAD_EFFECTIVE_RUN_COVERAGE]: + goodCoverage + (mcReproducibleAsNotBad ? mcReproducibleCoverage : 0), + [QcSummarProperties.MC_REPRODUCIBLE]: mcReproducibleCoverage > 0, + [QcSummarProperties.MISSING_VERIFICATIONS]: flagsIds.length - verifiedFlagsIds.length, + }, + ]); + + return Object.fromEntries(gaqSummary); } /** diff --git a/lib/server/services/qualityControlFlag/QcFlagSummaryService.js b/lib/server/services/qualityControlFlag/QcFlagSummaryService.js index f68b84eb59..913df599a2 100644 --- a/lib/server/services/qualityControlFlag/QcFlagSummaryService.js +++ b/lib/server/services/qualityControlFlag/QcFlagSummaryService.js @@ -33,30 +33,16 @@ const { BadParameterError } = require('../../errors/BadParameterError.js'); const { dataSource } = require('../../../database/DataSource.js'); const { QcFlagRepository } = require('../../../database/repositories/index.js'); const { Op } = require('sequelize'); - -/** - * @typedef RunDetectorQcSummary - * @property {number} badEffectiveRunCoverage - fraction of run's data, marked explicitly with bad QC flag - * @property {number} explicitlyNotBadEffectiveRunCoverage - fraction of run's data, marked explicitly with good QC flag - * @property {number} missingVerificationsCount - number of not verified QC flags which are not discarded - * @property {boolean} mcReproducible - states whether some Limited Acceptance MC Reproducible flag was assigned - */ - -const QC_SUMMARY_PROPERTIES = { - badEffectiveRunCoverage: 'badEffectiveRunCoverage', - explicitlyNotBadEffectiveRunCoverage: 'explicitlyNotBadEffectiveRunCoverage', - missingVerificationsCount: 'missingVerificationsCount', - mcReproducible: 'mcReproducible', -}; +const { QcSummarProperties } = require('../../../domain/enums/QcSummaryProperties.js'); /** * QC flag summary service */ class QcFlagSummaryService { /** - * Update RunDetectorQcSummary or RunGaqSummary with new information + * Update RunDetectorQcSummary with new information * - * @param {RunDetectorQcSummary|RunGaqSummary} summaryUnit RunDetectorQcSummary or RunGaqSummary + * @param {RunDetectorQcSummary} summaryUnit RunDetectorQcSummary or RunGaqSummary * @param {{ bad: boolean, effectiveRunCoverage: number, mcReproducible: boolean}} partialSummaryUnit new properties * to be applied to the summary object * @return {void} @@ -69,19 +55,19 @@ class QcFlagSummaryService { } = partialSummaryUnit; if (bad) { - summaryUnit[QC_SUMMARY_PROPERTIES.badEffectiveRunCoverage] = effectiveRunCoverage; - summaryUnit[QC_SUMMARY_PROPERTIES.mcReproducible] = - mcReproducible || summaryUnit[QC_SUMMARY_PROPERTIES.mcReproducible]; + summaryUnit[QcSummarProperties.BAD_EFFECTIVE_RUN_COVERAGE] = effectiveRunCoverage; + summaryUnit[QcSummarProperties.MC_REPRODUCIBLE] = + mcReproducible || summaryUnit[QcSummarProperties.MC_REPRODUCIBLE]; } else { - summaryUnit[QC_SUMMARY_PROPERTIES.explicitlyNotBadEffectiveRunCoverage] = effectiveRunCoverage; - summaryUnit[QC_SUMMARY_PROPERTIES.mcReproducible] = - mcReproducible || summaryUnit[QC_SUMMARY_PROPERTIES.mcReproducible]; + summaryUnit[QcSummarProperties.EXPLICITELY_NOT_BAD_EFFECTIVE_RUN_COVERAGE] = effectiveRunCoverage; + summaryUnit[QcSummarProperties.MC_REPRODUCIBLE] = + mcReproducible || summaryUnit[QcSummarProperties.MC_REPRODUCIBLE]; } - if (summaryUnit[QC_SUMMARY_PROPERTIES.badEffectiveRunCoverage] === undefined) { - summaryUnit[QC_SUMMARY_PROPERTIES.badEffectiveRunCoverage] = 0; + if (summaryUnit[QcSummarProperties.BAD_EFFECTIVE_RUN_COVERAGE] === undefined) { + summaryUnit[QcSummarProperties.BAD_EFFECTIVE_RUN_COVERAGE] = 0; } - if (summaryUnit[QC_SUMMARY_PROPERTIES.explicitlyNotBadEffectiveRunCoverage] === undefined) { - summaryUnit[QC_SUMMARY_PROPERTIES.explicitlyNotBadEffectiveRunCoverage] = 0; + if (summaryUnit[QcSummarProperties.EXPLICITELY_NOT_BAD_EFFECTIVE_RUN_COVERAGE] === undefined) { + summaryUnit[QcSummarProperties.EXPLICITELY_NOT_BAD_EFFECTIVE_RUN_COVERAGE] = 0; } } @@ -95,7 +81,7 @@ class QcFlagSummaryService { * @param {object} [options] additional options * @param {boolean} [options.mcReproducibleAsNotBad = false] if set to true, `Limited Acceptance MC Reproducible` flag type is treated as * good one - * @return {Promise} summary + * @return {Promise} summary */ async getSummary({ dataPassId, simulationPassId, lhcPeriodId }, { mcReproducibleAsNotBad = false } = {}) { if (Boolean(dataPassId) + Boolean(simulationPassId) + Boolean(lhcPeriodId) > 1) { @@ -203,13 +189,13 @@ class QcFlagSummaryService { summary[runNumber] = {}; } if (!summary[runNumber][detectorId]) { - summary[runNumber][detectorId] = { [QC_SUMMARY_PROPERTIES.mcReproducible]: false }; + summary[runNumber][detectorId] = { [QcSummarProperties.MC_REPRODUCIBLE]: false }; } const runDetectorSummary = summary[runNumber][detectorId]; - runDetectorSummary[QC_SUMMARY_PROPERTIES.missingVerificationsCount] = - (runDetectorSummary[QC_SUMMARY_PROPERTIES.missingVerificationsCount] ?? 0) + missingVerificationsCount; + runDetectorSummary[QcSummarProperties.MISSING_VERIFICATIONS] = + (runDetectorSummary[QcSummarProperties.MISSING_VERIFICATIONS] ?? 0) + missingVerificationsCount; QcFlagSummaryService.mergeIntoSummaryUnit(runDetectorSummary, runDetectorSummaryForFlagTypesClass); } diff --git a/test/lib/server/services/qualityControlFlag/QcFlagService.test.js b/test/lib/server/services/qualityControlFlag/QcFlagService.test.js index d15e562564..3a4f02b424 100644 --- a/test/lib/server/services/qualityControlFlag/QcFlagService.test.js +++ b/test/lib/server/services/qualityControlFlag/QcFlagService.test.js @@ -43,6 +43,8 @@ const getEffectivePeriodsOfQcFlag = async (flagId) => (await QcFlagEffectivePeri const t = (timeString) => new Date(`2024-07-16 ${timeString}`).getTime(); const goodFlagTypeId = 3; +const badPidFlagTypeId = 12; +const limitedAccMCTypeId = 5; const qcFlagWithId1 = { id: 1, @@ -1574,9 +1576,6 @@ module.exports = () => { const t = (timeString) => new Date(`2024-07-10 ${timeString}`).getTime(); const relations = { user: { roles: ['admin'], externalUserId: 456 } }; - const goodFlagTypeId = 3; - const badPidFlagTypeId = 12; - const limitedAccMCTypeId = 5; it('should successfully get GAQ flags', async () => { const dataPassId = 3; diff --git a/test/public/qcFlags/index.js b/test/public/qcFlags/index.js index 0a1afbb6c7..14ad7351c8 100644 --- a/test/public/qcFlags/index.js +++ b/test/public/qcFlags/index.js @@ -28,5 +28,5 @@ module.exports = () => { describe('For Simulation Pass Creation Page', ForSimulationPassCreationSuite); describe('Details For Data Pass Page', DetailsForDataPassPageSuite); describe('Details For Simulation Pass Page', DetailsForSimulationPassPageSuite); - describe('GAQ Overview page', GaqOverviewPageSuite); + describe('GAQ Overview Page', GaqOverviewPageSuite); };