Merge pull request #46702 from patinkaew/nanoaod_dev_collection_14_2_…

…0_pre3

Add support for adding variable-size attributes (std::vector) within objects for NanoAOD

CommonTools/Utils/interface/TypedStringObjectMethodCaller.h

@@ -0,0 +1,43 @@
+#ifndef CommonTools_Utils_TypedStringObjectMethodCaller_h
+#define CommonTools_Utils_TypedStringObjectMethodCaller_h
+/* \class TypedStringObjectMethodCaller
+ * 
+ * Object's method (or a chain of methods) caller functor with generic return-type, specified by string expression
+ *
+ */
+
+#include "FWCore/Utilities/interface/EDMException.h"
+#include "CommonTools/Utils/interface/parser/Exception.h"
+#include "CommonTools/Utils/interface/parser/MethodChain.h"
+#include "CommonTools/Utils/interface/parser/MethodChainGrammar.h"
+#include "FWCore/Reflection/interface/ObjectWithDict.h"
+
+template <typename T, typename R, bool DefaultLazyness = false>
+struct TypedStringObjectMethodCaller {
+  TypedStringObjectMethodCaller(const std::string expr, bool lazy = DefaultLazyness) : type_(typeid(T)) {
+    using namespace boost::spirit::classic;
+    reco::parser::MethodChainGrammar grammar(methodchain_, type_, lazy);
+    const char* startingFrom = expr.c_str();
+    try {
+      if (!parse(startingFrom, grammar >> end_p, space_p).full) {
+        throw edm::Exception(edm::errors::Configuration, "failed to parse \"" + expr + "\"");
+      }
+    } catch (reco::parser::BaseException& e) {
+      throw edm::Exception(edm::errors::Configuration)
+          << "MethodChainGrammer parse error:" << reco::parser::baseExceptionWhat(e) << " (char "
+          << e.where - startingFrom << ")\n";
+    }
+  }
+
+  R operator()(const T& t) const {
+    edm::ObjectWithDict o(type_, const_cast<T*>(&t));
+    edm::ObjectWithDict ret = methodchain_->value(o);
+    return *static_cast<R*>(ret.address());
+  }
+
+private:
+  reco::parser::MethodChainPtr methodchain_;
+  edm::TypeWithDict type_;
+};
+
+#endif

CommonTools/Utils/interface/parser/MethodChain.h

@@ -0,0 +1,75 @@
+#ifndef CommonTools_Utils_MethodChain_h
+#define CommonTools_Utils_MethodChain_h
+
+/* \class reco::parser::MethodChain
+ *
+ * Chain of methods
+ * Based on ExpressionBase and ExpressionVar, but remove final conversion to double
+ *
+ */
+
+#include "CommonTools/Utils/interface/parser/MethodInvoker.h"
+#include "CommonTools/Utils/interface/TypeCode.h"
+
+#include <vector>
+#include <oneapi/tbb/concurrent_queue.h>
+
+namespace reco {
+  namespace parser {
+
+    /// Based on Expression, but its value method returns an edm::ObjectWithDict instead of a double
+    class MethodChainBase {
+    public:  // Public Methods
+      virtual ~MethodChainBase() {}
+      virtual edm::ObjectWithDict value(const edm::ObjectWithDict&) const = 0;
+    };
+
+    /// Shared ptr to MethodChainBase
+    typedef std::shared_ptr<MethodChainBase> MethodChainPtr;
+
+    /// Evaluate an object's method or datamember (or chain of them)
+    class MethodChain : public MethodChainBase {
+    private:  // Private Data Members
+      std::vector<MethodInvoker> methods_;
+      using Objects = std::vector<std::pair<edm::ObjectWithDict, bool>>;
+      mutable oneapi::tbb::concurrent_queue<Objects> objectsCache_;
+
+    private:  // Private Methods
+      [[nodiscard]] Objects initObjects_() const;
+
+      Objects borrowObjects() const;
+      void returnObjects(Objects&&) const;
+
+    public:  // Public Static Methods
+      /// allocate an object to hold the result of a given member (if needed)
+      /// this method is used also from the LazyInvoker code
+      /// returns true if objects returned from this will require a destructor
+      static bool makeStorage(edm::ObjectWithDict& obj, const edm::TypeWithDict& retType);
+
+      /// delete an objecty, if needed
+      /// this method is used also from the LazyInvoker code
+      static void delStorage(edm::ObjectWithDict&);
+
+    public:  // Public Methods
+      MethodChain(const std::vector<MethodInvoker>& methods);
+      MethodChain(const MethodChain&);
+      ~MethodChain();
+      edm::ObjectWithDict value(const edm::ObjectWithDict&) const override;
+    };
+
+    /// Same as MethodChain but with lazy resolution of object methods
+    /// using the dynamic type of the object, and not the one fixed at compile time
+    class LazyMethodChain : public MethodChainBase {
+    private:  // Private Data Members
+      std::vector<LazyInvoker> methods_;
+
+    public:
+      LazyMethodChain(const std::vector<LazyInvoker>& methods);
+      ~LazyMethodChain() override;
+      edm::ObjectWithDict value(const edm::ObjectWithDict&) const override;
+    };
+
+  }  // namespace parser
+}  // namespace reco
+
+#endif  // CommonTools_Utils_MethodChain_h

CommonTools/Utils/interface/parser/MethodChainGrammar.h

@@ -0,0 +1,76 @@
+#ifndef CommonTools_Utils_MethodChainGrammar_h
+#define CommonTools_Utils_MethodChainGrammar_h
+/* \class MethodChainGrammer
+ *
+ * subset grammar of the full Grammer (CommonTools/Utils/interface/parser/Grammar.h), allowing only a chain of methods 
+ *
+ */
+
+#include "boost/spirit/include/classic_core.hpp"
+#include "boost/spirit/include/classic_grammar_def.hpp"
+#include "boost/spirit/include/classic_chset.hpp"
+#include "FWCore/Utilities/interface/EDMException.h"
+#include "FWCore/Reflection/interface/ObjectWithDict.h"
+#include "CommonTools/Utils/interface/parser/MethodChain.h"
+#include "CommonTools/Utils/interface/parser/MethodChainSetter.h"
+#include "CommonTools/Utils/interface/parser/MethodSetter.h"
+#include "CommonTools/Utils/interface/parser/MethodArgumentSetter.h"
+#include "CommonTools/Utils/interface/parser/MethodInvoker.h"
+#include "CommonTools/Utils/interface/parser/MethodStack.h"
+#include "CommonTools/Utils/interface/parser/TypeStack.h"
+#include "CommonTools/Utils/interface/parser/MethodArgumentStack.h"
+#include "CommonTools/Utils/interface/parser/AnyMethodArgument.h"
+#include "CommonTools/Utils/interface/parser/Exception.h"
+
+namespace reco {
+  namespace parser {
+    struct MethodChainGrammar : public boost::spirit::classic::grammar<MethodChainGrammar> {
+      MethodChainPtr* methchain_;
+      bool lazy_;
+      mutable MethodStack methStack;
+      mutable LazyMethodStack lazyMethStack;
+      mutable MethodArgumentStack methArgStack;
+      mutable TypeStack typeStack;
+
+      MethodChainGrammar(MethodChainPtr& methchain, const edm::TypeWithDict& iType, bool lazy = false)
+          : methchain_(&methchain), lazy_(lazy) {
+        typeStack.push_back(iType);
+      }
+
+      template <typename ScannerT>
+      struct definition {
+        typedef boost::spirit::classic::rule<ScannerT> rule;
+        rule metharg, method, arrayAccess, methodchain;
+        definition(const MethodChainGrammar& self) {
+          using namespace boost::spirit::classic;
+
+          MethodArgumentSetter methodArg_s(self.methArgStack);
+          MethodSetter method_s(self.methStack, self.lazyMethStack, self.typeStack, self.methArgStack, self.lazy_);
+          MethodChainSetter methodchain_s(*self.methchain_, self.methStack, self.lazyMethStack, self.typeStack);
+
+          BOOST_SPIRIT_DEBUG_RULE(methodchain);
+          BOOST_SPIRIT_DEBUG_RULE(arrayAccess);
+          BOOST_SPIRIT_DEBUG_RULE(method);
+          BOOST_SPIRIT_DEBUG_RULE(metharg);
+
+          boost::spirit::classic::assertion<SyntaxErrors> expectParenthesis(kMissingClosingParenthesis);
+          boost::spirit::classic::assertion<SyntaxErrors> expect(kSyntaxError);
+
+          metharg = (strict_real_p[methodArg_s]) | (int_p[methodArg_s]) |
+                    (ch_p('"') >> *(~ch_p('"')) >> ch_p('"'))[methodArg_s] |
+                    (ch_p('\'') >> *(~ch_p('\'')) >> ch_p('\''))[methodArg_s];
+          method =  // alnum_p doesn't accept underscores, so we use chset<>; lexeme_d needed to avoid whitespace skipping within method names
+              (lexeme_d[alpha_p >> *chset<>("a-zA-Z0-9_")] >> ch_p('(') >> metharg >> *(ch_p(',') >> metharg) >>
+               expectParenthesis(ch_p(')')))[method_s] |
+              ((lexeme_d[alpha_p >> *chset<>("a-zA-Z0-9_")])[method_s] >> !(ch_p('(') >> ch_p(')')));
+          arrayAccess = (ch_p('[') >> metharg >> *(ch_p(',') >> metharg) >> expectParenthesis(ch_p(']')))[method_s];
+          methodchain = (method >> *(arrayAccess | (ch_p('.') >> expect(method))))[methodchain_s];
+        }
+
+        rule const& start() const { return methodchain; }
+      };
+    };
+  }  // namespace parser
+}  // namespace reco
+
+#endif

CommonTools/Utils/interface/parser/MethodChainSetter.h

@@ -0,0 +1,36 @@
+#ifndef CommonTools_Utils_MethodChainSetter_h
+#define CommonTools_Utils_MethodChainSetter_h
+/* \class reco::parser::MethodChainSetter
+ *
+ * Method Chain setter, used to construct MethodChain when parsing with boost::spirit
+ *
+ */
+#include "CommonTools/Utils/interface/parser/MethodChain.h"
+#include "CommonTools/Utils/interface/parser/MethodStack.h"
+#include "CommonTools/Utils/interface/parser/TypeStack.h"
+
+#include <iostream>
+
+namespace reco {
+  namespace parser {
+    struct MethodChainSetter {
+      MethodChainSetter(MethodChainPtr &methchain,
+                        MethodStack &methStack,
+                        LazyMethodStack &lazyMethStack,
+                        TypeStack &typeStack)
+          : methchain_(methchain), methStack_(methStack), lazyMethStack_(lazyMethStack), typeStack_(typeStack) {}
+      void operator()(const char *, const char *) const;
+
+    private:
+      void push(const char *, const char *) const;
+      void lazyPush(const char *, const char *) const;
+
+      MethodChainPtr &methchain_;
+      MethodStack &methStack_;
+      LazyMethodStack &lazyMethStack_;
+      TypeStack &typeStack_;
+    };
+  }  // namespace parser
+}  // namespace reco
+
+#endif

CommonTools/Utils/src/MethodChain.cc

@@ -0,0 +1,122 @@
+#include "CommonTools/Utils/interface/parser/MethodChain.h"
+#include "CommonTools/Utils/interface/parser/MethodInvoker.h"
+
+#include "FWCore/Reflection/interface/ObjectWithDict.h"
+#include "FWCore/Reflection/interface/FunctionWithDict.h"
+#include "FWCore/Reflection/interface/MemberWithDict.h"
+#include "FWCore/Reflection/interface/TypeWithDict.h"
+
+#include <cassert>
+#include <map>
+
+using namespace reco::parser;
+using namespace std;
+
+MethodChain::Objects MethodChain::initObjects_() const {
+  Objects objects(methods_.size(), {edm::ObjectWithDict(), false});
+  assert(objects.size() == methods_.size());
+  auto IO = objects.begin();
+  for (auto const& method : methods_) {
+    if (method.isFunction()) {
+      edm::TypeWithDict retType = method.method().finalReturnType();
+      IO->second = makeStorage(IO->first, retType);
+    } else {
+      *IO = {edm::ObjectWithDict(), false};
+    }
+    ++IO;
+  }
+  return objects;
+}
+
+MethodChain::MethodChain(const vector<MethodInvoker>& methods) : methods_(methods) { returnObjects(initObjects_()); }
+
+MethodChain::MethodChain(const MethodChain& rhs) : methods_(rhs.methods_) { returnObjects(initObjects_()); }
+
+MethodChain::Objects MethodChain::borrowObjects() const {
+  Objects objects;
+  if (objectsCache_.try_pop(objects)) {
+    return objects;
+  }
+  return initObjects_();
+}
+
+void MethodChain::returnObjects(Objects&& iOb) const { objectsCache_.push(std::move(iOb)); }
+
+MethodChain::~MethodChain() {
+  Objects objects;
+  while (objectsCache_.try_pop(objects)) {
+    for (auto& o : objects) {
+      delStorage(o.first);
+    }
+  }
+}
+
+void MethodChain::delStorage(edm::ObjectWithDict& obj) {
+  if (!obj.address()) {
+    return;
+  }
+  if (obj.typeOf().isPointer() || obj.typeOf().isReference()) {
+    // just delete a void*, as that's what it was
+    void** p = static_cast<void**>(obj.address());
+    delete p;
+  } else {
+    //std::cout << "Calling Destruct on a " <<
+    //  obj.typeOf().qualifiedName() << std::endl;
+    obj.typeOf().deallocate(obj.address());
+  }
+}
+
+bool MethodChain::makeStorage(edm::ObjectWithDict& obj, const edm::TypeWithDict& retType) {
+  static const edm::TypeWithDict tVoid(edm::TypeWithDict::byName("void"));
+  bool ret = false;
+  if (retType == tVoid) {
+    obj = edm::ObjectWithDict::byType(tVoid);
+  } else if (retType.isPointer() || retType.isReference()) {
+    // in this case, I have to allocate a void*, not an object!
+    obj = edm::ObjectWithDict(retType, new void*);
+  } else {
+    obj = edm::ObjectWithDict(retType, retType.allocate());
+    ret = retType.isClass();
+    //std::cout << "MethodChain: reserved memory at "  << obj.address() <<
+    //  " for a " << retType.qualifiedName() << " returned by " <<
+    //  member.name() << std::endl;
+  }
+  return ret;
+}
+
+edm::ObjectWithDict MethodChain::value(const edm::ObjectWithDict& obj) const {
+  edm::ObjectWithDict val(obj);
+  auto objects = borrowObjects();
+  auto IO = objects.begin();
+  for (auto& m : methods_) {
+    val = m.invoke(val, IO->first);
+    ++IO;
+  }
+  for (auto RI = objects.rbegin(), RE = objects.rend(); RI != RE; ++RI) {
+    if (RI->second) {
+      RI->first.destruct(false);
+    }
+  }
+  returnObjects(std::move(objects));
+  return val;
+}
+
+LazyMethodChain::LazyMethodChain(const std::vector<LazyInvoker>& methods) : methods_(methods) {}
+
+LazyMethodChain::~LazyMethodChain() {}
+
+edm::ObjectWithDict LazyMethodChain::value(const edm::ObjectWithDict& o) const {
+  edm::ObjectWithDict val = o;
+  std::vector<StorageManager> storage;
+  storage.reserve(methods_.size());
+
+  std::vector<LazyInvoker>::const_iterator I = methods_.begin();
+  std::vector<LazyInvoker>::const_iterator E = methods_.end();
+  for (; I < E; ++I) {
+    val = I->invoke(val, storage);
+  }
+  while (not storage.empty()) {
+    storage.pop_back();
+  }
+  return val;
+}

CommonTools/Utils/src/MethodChainSetter.cc

@@ -0,0 +1,30 @@
+#include "CommonTools/Utils/interface/parser/MethodChainSetter.h"
+#include "CommonTools/Utils/interface/parser/MethodChain.h"
+#include "CommonTools/Utils/interface/returnType.h"
+#include "CommonTools/Utils/interface/parser/Exception.h"
+#include <string>
+
+using namespace reco::parser;
+using namespace std;
+
+void MethodChainSetter::operator()(const char *begin, const char *end) const {
+  //std::cerr << "MethodChainSetter: Pushed [" << std::string(begin,end) << "]" << std::endl;
+  if (!methStack_.empty())
+    push(begin, end);
+  else if (!lazyMethStack_.empty())
+    lazyPush(begin, end);
+  else
+    throw Exception(begin) << " Expression didn't parse neither hastily nor lazyly. This must not happen.\n";
+}
+
+void MethodChainSetter::push(const char *begin, const char *end) const {
+  methchain_ = std::shared_ptr<MethodChainBase>(new MethodChain(methStack_));
+  methStack_.clear();
+  typeStack_.resize(1);
+}
+
+void MethodChainSetter::lazyPush(const char *begin, const char *end) const {
+  methchain_ = std::shared_ptr<MethodChainBase>(new LazyMethodChain(lazyMethStack_));
+  lazyMethStack_.clear();
+  typeStack_.resize(1);
+}