Skip to content

Commit

Permalink
Add support for running with IOSvc
Browse files Browse the repository at this point in the history
  • Loading branch information
@jmcarcell
jmcarcell committed Jan 14, 2025
1 parent 880b57a commit 5967d0b
Show file tree
Hide file tree
Showing 9 changed files with 289 additions and 78 deletions.
64 changes: 42 additions & 22 deletions k4MarlinWrapper/examples/clicRec_e4h_input.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,16 @@

import os

from Gaudi.Configuration import *
from Gaudi.Configuration import DEBUG, WARNING

from Configurables import LcioEvent, MarlinProcessorWrapper
from k4MarlinWrapper.parseConstants import *
from Configurables import MarlinProcessorWrapper
from k4MarlinWrapper.parseConstants import parseConstants

from Configurables import Lcio2EDM4hepTool, EDM4hep2LcioTool
from Configurables import k4DataSvc, PodioInput, PodioOutput, EventDataSvc

from k4FWCore import ApplicationMgr, IOSvc
from k4FWCore.parseArgs import parser

algList = []

Expand All @@ -33,22 +39,33 @@

parseConstants(CONSTANTS)


# For converters
from Configurables import ToolSvc, Lcio2EDM4hepTool, EDM4hep2LcioTool


from Configurables import k4DataSvc, PodioInput

evtsvc = k4DataSvc("EventDataSvc")
evtsvc.input = os.path.join(
"$TEST_DIR/inputFiles/", os.environ.get("INPUTFILE", "ttbar_edm4hep_frame.root")
parser.add_argument(
"--iosvc", action="store_true", default=False, help="Use IOSvc instead of PodioDataSvc"
)


inp = PodioInput("InputReader")
inp.OutputLevel = DEBUG

parser.add_argument("--rec-output", help="Output file name for the REC file")
parser.add_argument("--dst-output", help="Output file name for the DST file")

args = parser.parse_known_args()[0]

if args.iosvc:
evtsvc = EventDataSvc("EventDataSvc")
iosvc = IOSvc()
iosvc.Input = os.path.join(
"$TEST_DIR/inputFiles/", os.environ.get("INPUTFILE", "ttbar_edm4hep_frame.root")
)
iosvc.Output = "my_output.root"
iosvc.outputDstName = ["keep *, drop RefinedVertexJets_PID_RefinedVertex"]
else:
evtsvc = k4DataSvc("EventDataSvc")
evtsvc.input = os.path.join(
"$TEST_DIR/inputFiles/", os.environ.get("INPUTFILE", "ttbar_edm4hep_frame.root")
)

inp = PodioInput("InputReader")
inp.OutputLevel = DEBUG

out = PodioOutput("PodioOutput", filename="my_output.root")
out.outputCommands = ["keep *, drop RefinedVertexJets_PID_RefinedVertex"]

MyAIDAProcessor = MarlinProcessorWrapper("MyAIDAProcessor")
MyAIDAProcessor.OutputLevel = WARNING
Expand Down Expand Up @@ -1167,7 +1184,7 @@
"DropCollectionTypes": [],
"FullSubsetCollections": ["EfficientMCParticles", "InefficientMCParticles"],
"KeepCollectionNames": [],
"LCIOOutputFile": ["Output_REC_e4h_input.slcio"],
"LCIOOutputFile": [args.rec_output],
"LCIOWriteMode": ["WRITE_NEW"],
}

Expand Down Expand Up @@ -1233,7 +1250,7 @@
"RefinedVertices",
"RefinedVertices_RP",
],
"LCIOOutputFile": ["Output_DST_e4h_input.slcio"],
"LCIOOutputFile": [args.dst_output],
"LCIOWriteMode": ["WRITE_NEW"],
}

Expand Down Expand Up @@ -2429,6 +2446,7 @@
}


<<<<<<< HEAD
# Write output to EDM4hep
from Configurables import PodioOutput

Expand All @@ -2437,6 +2455,8 @@


algList.append(inp)
=======
>>>>>>> cb3c7a3 (Add support for running with IOSvc)
algList.append(MyAIDAProcessor)
algList.append(EventNumber)
algList.append(InitDD4hep)
Expand Down Expand Up @@ -2486,8 +2506,8 @@
# # algList.append(VertexFinderUnconstrained) # Config.VertexUnconstrainedON
algList.append(Output_REC)
algList.append(Output_DST)
algList.append(out)

from Configurables import ApplicationMgr
if not args.iosvc:
algList = [inp] + algList + [out]

ApplicationMgr(TopAlg=algList, EvtSel="NONE", EvtMax=3, ExtSvc=[evtsvc], OutputLevel=WARNING)
4 changes: 4 additions & 0 deletions k4MarlinWrapper/k4MarlinWrapper/converters/EDM4hep2Lcio.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,8 @@ class EDM4hep2LcioTool : public AlgTool, virtual public IEDMConverter {

PodioDataSvc* m_podioDataSvc;
ServiceHandle<IDataProviderSvc> m_eventDataSvc;
std::vector<std::string> m_collectionNames;
std::map<uint32_t, std::string> m_idToName;

void convertTracks(TrackMap& tracks_vec, const std::string& e4h_coll_name, const std::string& lcio_coll_name,
lcio::LCEventImpl* lcio_event);
Expand Down Expand Up @@ -110,6 +112,8 @@ class EDM4hep2LcioTool : public AlgTool, virtual public IEDMConverter {
std::vector<EDM4hep2LCIOConv::ParticleIDConvData>& pidCollections,
std::vector<EDM4hep2LCIOConv::TrackDqdxConvData>& dQdxCollections);

StatusCode getAvailableCollectionsFromStore();

/// Get an EDM4hep collection by name, consulting either the podio based data
/// svc or the IOSvc
podio::CollectionBase* getEDM4hepCollection(const std::string& name) const;
Expand Down
2 changes: 1 addition & 1 deletion k4MarlinWrapper/k4MarlinWrapper/converters/Lcio2EDM4hep.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ class Lcio2EDM4hepTool : public AlgTool, virtual public IEDMConverter {
Gaudi::Property<std::map<std::string, std::string>> m_collNames{this, "collNameMapping", {}};
Gaudi::Property<bool> m_convertAll{this, "convertAll", true};

ServiceHandle<IDataProviderSvc> m_eds;
ServiceHandle<IDataProviderSvc> m_eventDataSvc;
PodioDataSvc* m_podioDataSvc;

// **********************************
Expand Down
146 changes: 129 additions & 17 deletions k4MarlinWrapper/src/components/EDM4hep2Lcio.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,16 +17,24 @@
* limitations under the License.
*/
#include "k4MarlinWrapper/converters/EDM4hep2Lcio.h"
#include <GaudiKernel/StatusCode.h>
#include "GlobalConvertedObjectsMap.h"

#include "UTIL/PIDHandler.h"

#include "edm4hep/Constants.h"

#include "k4FWCore/DataHandle.h"
#include "k4FWCore/FunctionalUtils.h"
#include "k4FWCore/MetaDataHandle.h"
#include "k4FWCore/PodioDataSvc.h"

#include "GaudiKernel/AnyDataWrapper.h"
#include "GaudiKernel/IDataManagerSvc.h"
#include "GaudiKernel/IDataProviderSvc.h"
#include "GaudiKernel/SmartDataPtr.h"

#include <functional>
#include <memory>

DECLARE_COMPONENT(EDM4hep2LcioTool);
Expand Down Expand Up @@ -55,14 +63,14 @@ EDM4hep2LcioTool::EDM4hep2LcioTool(const std::string& type, const std::string& n
}

StatusCode EDM4hep2LcioTool::initialize() {
StatusCode sc = m_eventDataSvc.retrieve();
m_podioDataSvc = dynamic_cast<PodioDataSvc*>(m_eventDataSvc.get());

StatusCode sc = m_eventDataSvc.retrieve();
if (sc == StatusCode::FAILURE) {
error() << "Error retrieving Event Data Service" << endmsg;
error() << "Could not retrieve the EventDataSvc;" << endmsg;
return StatusCode::FAILURE;
}

m_podioDataSvc = dynamic_cast<PodioDataSvc*>(m_eventDataSvc.get());

return AlgTool::initialize();
}

Expand Down Expand Up @@ -277,7 +285,7 @@ void EDM4hep2LcioTool::convertEventHeader(const std::string& e4h_coll_name, lcio

podio::CollectionBase* EDM4hep2LcioTool::getEDM4hepCollection(const std::string& collName) const {
DataObject* p;
auto sc = m_podioDataSvc->retrieveObject(collName, p);
auto sc = m_eventDataSvc->retrieveObject(collName, p);
if (sc.isFailure()) {
throw GaudiException("Collection not found", name(), StatusCode::FAILURE);
}
Expand All @@ -302,14 +310,59 @@ podio::CollectionBase* EDM4hep2LcioTool::getEDM4hepCollection(const std::string&
throw GaudiException("Collection could not be casted to the expected type", name(), StatusCode::FAILURE);
}

StatusCode EDM4hep2LcioTool::getAvailableCollectionsFromStore() {
SmartIF<IDataManagerSvc> mgr;
mgr = evtSvc();

SmartDataPtr<DataObject> root(evtSvc(), "/Event");
if (!root) {
error() << "Failed to retrieve root object /Event" << endmsg;
return StatusCode::FAILURE;
}

auto pObj = root->registry();
if (!pObj) {
error() << "Failed to retrieve the root registry object" << endmsg;
return StatusCode::FAILURE;
}
std::vector<IRegistry*> leaves;
StatusCode sc = mgr->objectLeaves(pObj, leaves);
if (!sc.isSuccess()) {
error() << "Failed to retrieve object leaves" << endmsg;
return StatusCode::FAILURE;
}
for (const auto& pReg : leaves) {
if (pReg->name() == k4FWCore::frameLocation) {
continue;
}
DataObject* p;
sc = m_eventDataSvc->retrieveObject("/Event" + pReg->name(), p);
if (sc.isFailure()) {
error() << "Could not retrieve object " << pReg->name() << " from the EventStore" << endmsg;
return sc;
}
auto wrapper = dynamic_cast<AnyDataWrapper<std::unique_ptr<podio::CollectionBase>>*>(p);
if (!wrapper) {
continue;
}
// Remove the leading /
m_collectionNames.push_back(pReg->name().substr(1, pReg->name().size() - 1));
m_idToName.emplace(wrapper->getData()->getID(), pReg->name());
}
return StatusCode::SUCCESS;
}

// Select the appropriate method to convert a collection given its type
void EDM4hep2LcioTool::convertAdd(const std::string& e4h_coll_name, const std::string& lcio_coll_name,
lcio::LCEventImpl* lcio_event, CollectionPairMappings& collection_pairs,
std::vector<EDM4hep2LCIOConv::ParticleIDConvData>& pidCollections,
std::vector<EDM4hep2LCIOConv::TrackDqdxConvData>& dQdxCollections) {
const auto& metadata = m_podioDataSvc->getMetaDataFrame();
const auto collPtr = getEDM4hepCollection(e4h_coll_name);
const auto fulltype = collPtr->getValueTypeName();
std::optional<std::reference_wrapper<podio::Frame>> metadata;
if (m_podioDataSvc) {
metadata = m_podioDataSvc->getMetaDataFrame();
}
const auto collPtr = getEDM4hepCollection(e4h_coll_name);
const auto fulltype = collPtr->getValueTypeName();

debug() << "Converting type " << fulltype << " from input " << e4h_coll_name << endmsg;

Expand Down Expand Up @@ -340,8 +393,32 @@ void EDM4hep2LcioTool::convertAdd(const std::string& e4h_coll_name, const std::s
} else if (fulltype == "edm4hep::EventHeader") {
convertEventHeader(e4h_coll_name, lcio_event);
} else if (fulltype == "edm4hep::ParticleID") {
pidCollections.emplace_back(e4h_coll_name, static_cast<const edm4hep::ParticleIDCollection*>(collPtr),
edm4hep::utils::PIDHandler::getAlgoInfo(metadata, e4h_coll_name));
std::optional<std::string> maybeAlgoName;
std::optional<int> maybeAlgoType;
std::optional<std::vector<std::string>> maybeParamNames;

if (m_podioDataSvc) {
const auto& frame = metadata.value().get();
maybeAlgoName =
frame.getParameter<std::string>(podio::collMetadataParamName(e4h_coll_name, edm4hep::labels::PIDAlgoName));
maybeAlgoType =
frame.getParameter<int>(podio::collMetadataParamName(e4h_coll_name, edm4hep::labels::PIDAlgoType));
maybeParamNames = frame.getParameter<std::vector<std::string>>(
podio::collMetadataParamName(e4h_coll_name, edm4hep::labels::PIDParameterNames));

} else {
maybeAlgoName = k4FWCore::getParameter<std::string>(
podio::collMetadataParamName(e4h_coll_name, edm4hep::labels::PIDAlgoName));
maybeAlgoType =
k4FWCore::getParameter<int>(podio::collMetadataParamName(e4h_coll_name, edm4hep::labels::PIDAlgoType));
maybeParamNames = k4FWCore::getParameter<std::vector<std::string>>(
podio::collMetadataParamName(e4h_coll_name, edm4hep::labels::PIDParameterNames));
}

edm4hep::utils::ParticleIDMeta pidInfo{std::move(maybeAlgoName.value()), maybeAlgoType.value(),
maybeParamNames.value()};

pidCollections.emplace_back(e4h_coll_name, static_cast<const edm4hep::ParticleIDCollection*>(collPtr), pidInfo);
} else if (fulltype == "edm4hep::RecDqDx") {
dQdxCollections.emplace_back(e4h_coll_name, static_cast<const edm4hep::RecDqdxCollection*>(collPtr));
}
Expand All @@ -362,8 +439,17 @@ void EDM4hep2LcioTool::convertAdd(const std::string& e4h_coll_name, const std::s
// Parse property parameters and convert the indicated collections.
// Use the collection names in the parameters to read and write them
StatusCode EDM4hep2LcioTool::convertCollections(lcio::LCEventImpl* lcio_event) {
const auto& edmEvent = m_podioDataSvc->getEventFrame();
const auto collections = edmEvent.getAvailableCollections();
std::optional<std::reference_wrapper<const podio::Frame>> edmEvent;
if (m_collectionNames.empty() && m_podioDataSvc) {
edmEvent = m_podioDataSvc->getEventFrame();
m_collectionNames = edmEvent.value().get().getAvailableCollections();
} else if (m_collectionNames.empty()) {
auto sc = getAvailableCollectionsFromStore();
if (sc.isFailure()) {
warning() << "Could not retrieve available collections from the EventStore" << endmsg;
return sc;
}
}
// Start off with the pre-defined collection name mappings
auto collsToConvert{m_collNames.value()};
// We *always* want to convert the EventHeader
Expand All @@ -372,7 +458,7 @@ StatusCode EDM4hep2LcioTool::convertCollections(lcio::LCEventImpl* lcio_event) {
info() << "Converting all collections from EDM4hep to LCIO" << endmsg;
// And simply add the rest, exploiting the fact that emplace will not
// replace existing entries with the same key
for (const auto& name : collections) {
for (const auto& name : m_collectionNames) {
collsToConvert.emplace(name, name);
}
}
Expand All @@ -399,13 +485,39 @@ StatusCode EDM4hep2LcioTool::convertCollections(lcio::LCEventImpl* lcio_event) {
}
debug() << "Event: " << lcio_event->getEventNumber() << " Run: " << lcio_event->getRunNumber() << endmsg;

// Deal with
EDM4hep2LCIOConv::sortParticleIDs(pidCollections);
if (!m_podioDataSvc) {
DataObject* p;
StatusCode code = m_eventDataSvc->retrieveObject("/Event" + k4FWCore::frameLocation, p);
if (code.isSuccess()) {
auto* frame = dynamic_cast<AnyDataWrapper<podio::Frame>*>(p);
edmEvent = std::cref(frame->getData());
} else {
auto frame = podio::Frame{};
edmEvent = frame;
}
}
for (const auto& pidCollMeta : pidCollections) {
const auto algoId = attachParticleIDMetaData(lcio_event, edmEvent, pidCollMeta);
auto algoId = attachParticleIDMetaData(lcio_event, edmEvent.value(), pidCollMeta);
if (!algoId.has_value()) {
warning() << "Could not determine algorithm type for ParticleID collection " << pidCollMeta.name
<< " for setting consistent metadata" << endmsg;
// Now go over the collections that have been produced in a functional algorithm (if any)
bool found = false;
if (!m_podioDataSvc) {
const auto id = (*pidCollMeta.coll)[0].getParticle().id().collectionID;
if (auto it = m_idToName.find(id); it != m_idToName.end()) {
auto name = it->second;
if (pidCollMeta.metadata.has_value()) {
UTIL::PIDHandler pidHandler(lcio_event->getCollection(name));
algoId =
pidHandler.addAlgorithm(pidCollMeta.metadata.value().algoName, pidCollMeta.metadata.value().paramNames);
found = true;
}
}
}
if (!found) {
warning() << "Could not determine algorithm type for ParticleID collection " << pidCollMeta.name
<< " for setting consistent metadata" << endmsg;
}
}
convertParticleIDs(collection_pairs.particleIDs, pidCollMeta.name, algoId.value_or(-1));
}
Expand Down
Loading

0 comments on commit 5967d0b

Please sign in to comment.