Add new tests for reduced ProcessHistory using mocked CMSSW release v…

…ersion

The existing test relies on old (4_2_X-time) files.

FWCore/Framework/src/ScheduleItems.cc

@@ -134,8 +134,15 @@ namespace edm {
 
     act_table_ = std::make_unique<ExceptionToActionTable>(parameterSet);
     std::string processName = parameterSet.getParameter<std::string>("@process_name");
-    processConfiguration_ = std::make_shared<ProcessConfiguration>(
-        processName, getReleaseVersion(), getPassID());  // propagate_const<T> has no reset() function
+    std::string releaseVersion;
+    if (parameterSet.existsAs<std::string>("@special_override_release_version_only_for_testing", false)) {
+      releaseVersion =
+          parameterSet.getUntrackedParameter<std::string>("@special_override_release_version_only_for_testing");
+    } else {
+      releaseVersion = getReleaseVersion();
+    }
+    // propagate_const<T> has no reset() function
+    processConfiguration_ = std::make_shared<ProcessConfiguration>(processName, releaseVersion, getPassID());
     auto common = std::make_shared<CommonParams>(
         parameterSet.getUntrackedParameterSet("maxEvents").getUntrackedParameter<int>("input"),
         parameterSet.getUntrackedParameterSet("maxLuminosityBlocks").getUntrackedParameter<int>("input"),

FWCore/ParameterSet/python/Config.py

@@ -149,6 +149,7 @@ def __init__(self,name: str, *Mods):
         self.__isStrict = False
         self.__dict__['_Process__modifiers'] = Mods
         self.__dict__['_Process__accelerators'] = {}
+        self.__dict__['_Process__specialOverrideReleaseVersionOnlyForTesting'] = None
         self.__injectValidValue('options', Process.defaultOptions_())
         self.__injectValidValue('maxEvents', Process.defaultMaxEvents_())
         self.maxLuminosityBlocks = Process.defaultMaxLuminosityBlocks_()
@@ -813,6 +814,10 @@ def extend(self,other,items=()):
 
         self.__dict__['_Process__InExtendCall'] = False
 
+    def _specialOverrideReleaseVersionOnlyForTesting(self, version):
+        "This function is intended only for specific framework tests. Do not use for anything else."
+        self.__specialOverrideReleaseVersionOnlyForTesting = version
+
     def _dumpConfigNamedList(self,items,typeName:str,options:PrintOptions) -> str:
         returnValue = ''
         for name,item in items:
@@ -1458,6 +1463,8 @@ def __extractPSet(self,pset):
 
         self.validate()
         processPSet.addString(True, "@process_name", self.name_())
+        if self.__specialOverrideReleaseVersionOnlyForTesting is not None:
+            processPSet.addString(False, "@special_override_release_version_only_for_testing", self.__specialOverrideReleaseVersionOnlyForTesting)
         self.handleProcessAccelerators(processPSet)
         all_modules = self.producers_().copy()
         all_modules.update(self.filters_())
@@ -4944,4 +4951,16 @@ def testProcessAccelerator(self):
             self.assertEqual((False, "sp@test1"), p.values["sp"][1].values["@chosen_case"])
             self.assertEqual(["cpu", "test1"], p.values["@available_accelerators"][1])
 
+        def testProcessSpecialOverrideReleaseVersion(self):
+            proc = Process("TEST")
+            p = TestMakePSet()
+            proc.fillProcessDesc(p)
+            self.assertFalse("@special_override_release_version_only_for_testing" in p.values)
+
+            proc = Process("TEST")
+            proc._specialOverrideReleaseVersionOnlyForTesting("CMSSW_15_0_0")
+            p = TestMakePSet()
+            proc.fillProcessDesc(p)
+            self.assertEqual((False, "CMSSW_15_0_0"), p.values["@special_override_release_version_only_for_testing"])
+
     unittest.main()

IOPool/Input/test/BuildFile.xml

@@ -38,5 +38,6 @@
     <lib name="SchemaEvolutionTest"/>
   </test>
   <test name="TestIOPoolInputRefProductIDMetadataConsistency" command="testRefProductIDMetadataConsistencyRoot.sh"/>
+  <test name="TestIOPoolInputReducedProcessHistory" command="testReducedProcessHistory.sh"/>
 
 </environment>

IOPool/Input/test/testReducedProcessHistory.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+
+function die { echo $1: status $2 ;  exit $2; }
+function runSuccess {
+    echo "cmsRun $@"
+    cmsRun $@ || die "cmsRun $*" $?
+    echo
+}
+function runFailure {
+    echo "cmsRun $@ (expected to fail)"
+    cmsRun $@ && die "cmsRun $*" 1
+    echo
+}
+
+VERSION_ARR=(${CMSSW_VERSION//_/ })
+VERSION1="${VERSION_ARR[0]}_${VERSION_ARR[1]}_${VERSION_ARR[2]}_0"
+VERSION2="${VERSION_ARR[0]}_${VERSION_ARR[1]}_${VERSION_ARR[2]}_1"
+VERSION3="${VERSION_ARR[0]}_${VERSION_ARR[1]}_$((${VERSION_ARR[2]}+1))_0"
+
+# Check that changing the patch version does not lead to new lumi or run
+runSuccess ${SCRAM_TEST_PATH}/testReducedProcessHistoryCreate_cfg.py --version ${VERSION1} --firstEvent 1 --output version1.root
+runSuccess ${SCRAM_TEST_PATH}/testReducedProcessHistoryCreate_cfg.py --version ${VERSION2} --firstEvent 101 --output version2.root
+
+edmProvDump version1.root | grep -q "PROD.*'${VERSION1}'" || die "Did not find ${VERSION1} from version.root provenance" $?
+edmProvDump version2.root | grep -q "PROD.*'${VERSION2}'" || die "Did not find ${VERSION2} from version.root provenance" $?
+
+runSuccess ${SCRAM_TEST_PATH}/test_merge_two_files.py version1.root version2.root
+
+runSuccess ${SCRAM_TEST_PATH}/testReducedProcessHistory_cfg.py --input merged_files.root
+
+
+# Check that changing the minor version leads to new lumi
+runSuccess ${SCRAM_TEST_PATH}/testReducedProcessHistoryCreate_cfg.py --version ${VERSION3} --firstEvent 201 --output version3_lumi.root
+
+edmProvDump version3_lumi.root | grep -q "PROD.*'${VERSION3}'" || die "Did not find ${VERSION3} from version3_lumi.root provenance" $?
+
+runSuccess ${SCRAM_TEST_PATH}/test_merge_two_files.py version1.root version3_lumi.root --output merged_files3_lumi.root --bypassVersionCheck
+
+runFailure ${SCRAM_TEST_PATH}/testReducedProcessHistory_cfg.py --input merged_files3_lumi.root --bypassVersionCheck
+
+runSuccess ${SCRAM_TEST_PATH}/testReducedProcessHistory_cfg.py --input merged_files3_lumi.root --bypassVersionCheck --expectNewLumi
+
+
+# Check that changing the minor version leads to new run
+runSuccess ${SCRAM_TEST_PATH}/testReducedProcessHistoryCreate_cfg.py --version ${VERSION3} --firstEvent 201 --lumi 2 --output version3_run.root
+
+edmProvDump version3_run.root | grep -q "PROD.*'${VERSION3}'" || die "Did not find ${VERSION3} from version3_lumi.root provenance" $?
+
+runSuccess ${SCRAM_TEST_PATH}/test_merge_two_files.py version1.root version3_run.root --output merged_files3_run.root --bypassVersionCheck
+
+runFailure ${SCRAM_TEST_PATH}/testReducedProcessHistory_cfg.py --input merged_files3_run.root --bypassVersionCheck
+
+runSuccess ${SCRAM_TEST_PATH}/testReducedProcessHistory_cfg.py --input merged_files3_run.root --bypassVersionCheck --expectNewRun
+
+exit 0

IOPool/Input/test/testReducedProcessHistoryCreate_cfg.py

@@ -0,0 +1,39 @@
+import FWCore.ParameterSet.Config as cms
+import argparse
+
+parser = argparse.ArgumentParser(description='Create files for reduced ProcessHistory test')
+parser.add_argument("--version", type=str, help="CMSSW version to be used in the ProcessHistory")
+parser.add_argument("--firstEvent", default=1, type=int, help="Number of first event")
+parser.add_argument("--lumi", default=1, type=int, help="LuminosityBlock number")
+parser.add_argument("--output", type=str, help="Output file name")
+
+args = parser.parse_args()
+
+process = cms.Process("PROD")
+process._specialOverrideReleaseVersionOnlyForTesting(args.version)
+
+process.maxEvents.input = 10
+
+from FWCore.Modules.modules import EmptySource
+process.source = EmptySource(
+    firstEvent = args.firstEvent,
+    firstLuminosityBlock = args.lumi,
+)
+
+from IOPool.Output.modules import PoolOutputModule
+process.out = PoolOutputModule(
+    fileName = args.output
+)
+
+from FWCore.Framework.modules import IntProducer
+process.intProducer = IntProducer(ivalue = 42)
+
+from FWCore.Integration.modules import ThingWithMergeProducer
+process.thingWithMergeProducer = ThingWithMergeProducer()
+
+process.t = cms.Task(
+    process.intProducer,
+    process.thingWithMergeProducer,
+)
+process.p = cms.Path(process.t)
+process.ep = cms.EndPath(process.out)

IOPool/Input/test/testReducedProcessHistory_cfg.py

@@ -0,0 +1,117 @@
+import FWCore.ParameterSet.Config as cms
+import argparse
+
+parser = argparse.ArgumentParser(description='Test reduced ProcessHistory')
+parser.add_argument("--input", type=str, help="Input file")
+parser.add_argument("--bypassVersionCheck", action="store_true", help="Bypass version check")
+parser.add_argument("--expectNewLumi", action="store_true", help="Set this if a new lumi is expected between the original files")
+parser.add_argument("--expectNewRun", action="store_true", help="Set this if a new run is expected between the original files")
+
+args = parser.parse_args()
+
+process = cms.Process("READ")
+
+from IOPool.Input.modules import PoolSource
+process.source = PoolSource(
+    fileNames = [f"file:{args.input}"],
+    bypassVersionCheck = args.bypassVersionCheck,
+)
+
+from FWCore.Framework.modules import TestMergeResults, RunLumiEventAnalyzer
+process.testmerge = TestMergeResults(
+    expectedBeginRunProd = [10001, 20004, 10003],
+    expectedEndRunProd = [100001, 200004, 100003],
+    expectedBeginLumiProd = [101, 204, 103],
+    expectedEndLumiProd = [1001, 2004, 1003],
+
+    expectedBeginRunNew = [10001, 10002, 10003],
+    expectedEndRunNew = [100001, 100002, 100003],
+    expectedBeginLumiNew = [101, 102, 103],
+    expectedEndLumiNew = [1001, 1002, 1003],
+    expectedProcessHistoryInRuns = [
+        'PROD',
+        'MERGETWOFILES',
+        'READ'
+    ]
+)
+def setWithMergeAndCopyEntry(p, value):
+    p[1] = value
+    p[3:5] = p[0:2]
+if args.expectNewLumi or args.expectNewRun:
+    setWithMergeAndCopyEntry(process.testmerge.expectedBeginRunProd, 10002)
+    setWithMergeAndCopyEntry(process.testmerge.expectedEndRunProd, 100002)
+    setWithMergeAndCopyEntry(process.testmerge.expectedBeginLumiProd, 102)
+    setWithMergeAndCopyEntry(process.testmerge.expectedEndLumiProd, 1002)
+
+process.test = RunLumiEventAnalyzer(
+    expectedRunLumiEvents = [
+        1, 0, 0, # beginRun
+        1, 1, 0, # beginLumi
+        1, 1, 1,
+        1, 1, 2,
+        1, 1, 3,
+        1, 1, 4,
+        1, 1, 5,
+        1, 1, 6,
+        1, 1, 7,
+        1, 1, 8,
+        1, 1, 9,
+        1, 1, 10,
+        1, 1, 101,
+        1, 1, 102,
+        1, 1, 103,
+        1, 1, 104,
+        1, 1, 105,
+        1, 1, 106,
+        1, 1, 107,
+        1, 1, 108,
+        1, 1, 109,
+        1, 1, 110,
+        1, 1, 0, # endLumi
+        1, 0, 0, # endRun
+    ]
+)
+endFirstFileIndex = 3*(10+2)
+if args.expectNewLumi:
+    process.test.expectedRunLumiEvents = process.test.expectedRunLumiEvents[:endFirstFileIndex] + [
+        1, 1, 0, # endLumi
+        1, 0, 0, # endRun
+        1, 0, 0, # beginRun
+        1, 1, 0, # beginLumi
+        1, 1, 201,
+        1, 1, 202,
+        1, 1, 203,
+        1, 1, 204,
+        1, 1, 205,
+        1, 1, 206,
+        1, 1, 207,
+        1, 1, 208,
+        1, 1, 209,
+        1, 1, 210,
+        1, 1, 0, # endLumi
+        1, 0, 0, # endRun
+    ]
+elif args.expectNewRun:
+    process.test.expectedRunLumiEvents = process.test.expectedRunLumiEvents[:endFirstFileIndex] + [
+        1, 1, 0, # endLumi
+        1, 0, 0, # endRun
+        1, 0, 0, # beginRun
+        1, 2, 0, # beginLumi
+        1, 2, 201,
+        1, 2, 202,
+        1, 2, 203,
+        1, 2, 204,
+        1, 2, 205,
+        1, 2, 206,
+        1, 2, 207,
+        1, 2, 208,
+        1, 2, 209,
+        1, 2, 210,
+        1, 2, 0, # endLumi
+        1, 0, 0, # endRun
+    ]
+
+process.p = cms.Path(
+    process.testmerge +
+    process.test
+)

IOPool/Input/test/test_merge_two_files.py

@@ -1,18 +1,28 @@
 import FWCore.ParameterSet.Config as cms
-import sys
+import argparse
+
+parser = argparse.ArgumentParser(description='Create files for reduced ProcessHistory test')
+parser.add_argument("file1", type=str, help="First file to merge")
+parser.add_argument("file2", type=str, help="Second file to merge")
+parser.add_argument("--output", default="merged_files.root", help="Output file name")
+parser.add_argument("--bypassVersionCheck", action="store_true", help="Bypass version check")
+
+args = parser.parse_args()
 
 process = cms.Process("MERGETWOFILES")
 
-process.source = cms.Source("PoolSource",
-                            fileNames = cms.untracked.vstring("file:"+sys.argv[1],
-                                                              "file:"+sys.argv[2]),
-                            duplicateCheckMode = cms.untracked.string("noDuplicateCheck")
+from IOPool.Input.modules import PoolSource
+process.source = PoolSource(
+    fileNames = ["file:"+args.file1,"file:"+args.file2],
+    duplicateCheckMode = "noDuplicateCheck",
+    bypassVersionCheck = args.bypassVersionCheck,
 )
 
-process.thingWithMergeProducer = cms.EDProducer("ThingWithMergeProducer")
+from FWCore.Integration.modules import ThingWithMergeProducer
+process.thingWithMergeProducer = ThingWithMergeProducer()
 
-process.out = cms.OutputModule("PoolOutputModule",
-                            fileName = cms.untracked.string("merged_files.root"))
+from IOPool.Output.modules import PoolOutputModule
+process.out = PoolOutputModule(fileName = args.output)
 
 process.p = cms.Path(process.thingWithMergeProducer)