Skip to content

Commit

Permalink
[QC-907] utilities to merge config files
Browse files Browse the repository at this point in the history
  • Loading branch information
@Barthelemy
Barthelemy committed Oct 12, 2023
1 parent 08697fc commit e63c5b1
Show file tree
Hide file tree
Showing 6 changed files with 86 additions and 562 deletions.
9 changes: 6 additions & 3 deletions Framework/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,8 @@ set(EXE_SRCS
src/runUploadRootObjects.cxx
src/runFileMerger.cxx
src/runMetadataUpdater.cxx
src/runBookkeepingBenchmark.cxx)
src/runBookkeepingBenchmark.cxx
src/runConfigMerger.cxx)

set(EXE_NAMES
o2-qc-run-producer
Expand All @@ -177,7 +178,8 @@ set(EXE_NAMES
o2-qc-upload-root-objects
o2-qc-file-merger
o2-qc-metadata-updater
o2-qc-bk-benchmark)
o2-qc-bk-benchmark
o2-qc-config-merger)

# These were the original names before the convention changed. We will get rid
# of them but for the time being we want to create symlinks to avoid confusion.
Expand All @@ -198,7 +200,8 @@ set(EXE_OLD_NAMES
o2-qc-upload-root-objects
o2-qc-file-merger
o2-qc-metadata-updater
o2-qc-bk-benchmark)
o2-qc-bk-benchmark
o2-qc-config-merger)


# As per https://stackoverflow.com/questions/35765106/symbolic-links-cmake
Expand Down
261 changes: 11 additions & 250 deletions Framework/include/QualityControl/ptreeUtils.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,288 +20,49 @@
#include <iostream>
#include <Common/Exceptions.h>
#include <boost/property_tree/ptree.hpp>
#include <boost/property_tree/json_parser.hpp>
#include <string>
#include <stdexcept>
using boost::property_tree::ptree;

using namespace std;
namespace o2::quality_control::core
{

//namespace tree_ops
//{

// TODO when you get at an array this path is finished, just add the items and continue

// TODO what if we don't merge the trees and work with multiple ?ss
//
///// @brief Walk a Boost.PropertyTree tree, invoking the binary function with
///// the full path and the current node.
//template <typename Tree, typename Function>
//void for_each(
// const Tree& tree,
// Function fn,
// const typename Tree::path_type& parent_path = typename Tree::path_type(),
// std::string fullpath="")
//{
// using path_type = typename Tree::path_type;
// cout << "*** " << parent_path.dump() << endl;
// for (auto&& value_pair : tree) {
// cout << "*** '" << value_pair.first << "'" << endl;
// cout << "*** fullpath: '" << fullpath << "'" << endl;
// auto current_path = parent_path / path_type(value_pair.first);
// // fn(current_path, value_pair.second);
// std::string new_path = fullpath.empty() ? value_pair.first : fullpath+"."+value_pair.first;
// fn(parent_path, path_type(value_pair.first), value_pair.second, new_path);
// for_each(value_pair.second, fn, current_path, new_path);
// }
//}

/*/// @brief Return tree with elements in @ref s but not in @ref t.
template <typename Tree>
Tree tree_difference(const Tree& s, const Tree& t)
{
using data_type = typename Tree::data_type;
Tree result;
// Iterate 's', adding to the result when either a node in
// 't' is not present in 's' or the node's values differ.
for_each(s,
[&](const typename Tree::path_type& path, const Tree& node) {
auto value = t.template get_optional<data_type>(path);
if (!value || (value.get() != node.data()))
result.add(path, node.data());
});
return result;
}*/
//
///// @brief Return tree with elements from both @ref s and @ref t.
//template <typename Tree>
//Tree tree_union(const Tree& s, const Tree& t)
//{
// // The result will always contain all values in @ref s.
// Tree result = s;
// // Iterate 't', add values to the result only if the node is
// // either not in 's' or the values are different.
// for_each(t,
// [&](const typename Tree::path_type& parent_path, const typename Tree::path_type& this_path, const Tree& node, std::string fullpath) {
// cout << "--------------------------" << endl;
// cout << "tree_union parent_path : '" << parent_path.dump() << "' / '" << this_path.dump() << "' : " << node.data() << endl;
// cout << "fullpath : " << fullpath << endl;
// auto current_path = parent_path / this_path;
// auto child = s.get_child_optional(current_path);
//
// // if(child)
// // cout << "tree_union : child.data : " << child->data() << endl;
// // else
// // cout << "tree_union : no child "<< endl;
// if (!child || (child->data() != node.data())) {
// if(this_path.empty()) {
// cout << "EMPTY" << endl;
// // special case when the key is empty. It is probably a comment ?
// auto& result_parent= result.get_child(parent_path);
// result_parent.push_back(ptree::value_type("", node.data()));
// } else {
// result.add(fullpath, node.data());
// }
// }
// });
// return result;
//}
//
//}

/*/// @brief Return tree with elements from both @ref s and @ref t.
template <typename Tree>
Tree tree_union(const Tree& s, const Tree& t)
{
// The result will always contain all values in @ref s.
Tree result = s;
// Iterate 't', add values to the result only if the node is
// either not in 's' or the values are different.
for_each(t,
[&](const typename Tree::path_type& path, const Tree& node) {
cout << "tree_union : '" << path.dump() << "' : " << node.data() << endl;
auto child = s.get_child_optional(path);
// if(child)
// cout << "tree_union : child.data : " << child->data() << endl;
// else
// cout << "tree_union : no child "<< endl;
if (!child || (child->data() != node.data()))
result.add(path, node.data());
});
return result;
}*/
/*
/// @brief Return tree with elements common to @ref s and @ref t.
template <typename Tree>
Tree tree_intersection(const Tree& s, const Tree& t)
{
using data_type = typename Tree::data_type;
Tree result;
// Iterate 's', adding common elements found in 't' that have the same
// value.
for_each(s,
[&](const typename Tree::path_type& path, const Tree& node) {
auto value = t.template get_optional<data_type>(path);
if (value && (value.get() == node.data()))
result.add(path, node.data());
});
return result;
}
/// @brief Return tree with elements in either @ref s or @ref t, but not
/// both.
template <typename Tree>
Tree tree_symmetric_difference(const Tree& s, const Tree& t)
{
return tree_difference(tree_union(s, t), tree_intersection(s, t));
}
} // namespace tree_ops
// Expose mathematical tree operations with operators.
/// @brief Return tree with elements in @ref lhs but not in @ref rhs.
boost::property_tree::ptree operator-(
const boost::property_tree::ptree& lhs,
const boost::property_tree::ptree& rhs)
{
return tree_ops::tree_difference(lhs, rhs);
}*/

///// @brief Return tree with elements in both @ref lhs and @ref rhs.
//boost::property_tree::ptree operator|(
// const boost::property_tree::ptree& lhs,
// const boost::property_tree::ptree& rhs)
//{
// return tree_ops::tree_union(lhs, rhs);
//}

/*/// @brief Return tree with elements common to @ref lhs and @ref rhs.
boost::property_tree::ptree operator&(
const boost::property_tree::ptree& lhs,
const boost::property_tree::ptree& rhs)
{
return tree_ops::tree_intersection(lhs, rhs);
}
/// @brief Return tree with elements in either @ref lhs or @ref rhs, but not
/// both.
boost::property_tree::ptree operator^(
const boost::property_tree::ptree& lhs,
const boost::property_tree::ptree& rhs)
{
return tree_ops::tree_symmetric_difference(lhs, rhs);
}*/


// Create necessary escape sequences from illegal characters
//template<class Ch>
//std::basic_string<Ch> create_escapes(const std::basic_string<Ch> &s)
//{
// std::basic_string<Ch> result;
// typename std::basic_string<Ch>::const_iterator b = s.begin();
// typename std::basic_string<Ch>::const_iterator e = s.end();
// while (b != e)
// {
// if (*b == Ch('\0')) result += Ch('\\'), result += Ch('0');
// else if (*b == Ch('\a')) result += Ch('\\'), result += Ch('a');
// else if (*b == Ch('\b')) result += Ch('\\'), result += Ch('b');
// else if (*b == Ch('\f')) result += Ch('\\'), result += Ch('f');
// else if (*b == Ch('\n')) result += Ch('\\'), result += Ch('n');
// else if (*b == Ch('\r')) result += Ch('\\'), result += Ch('r');
// else if (*b == Ch('\v')) result += Ch('\\'), result += Ch('v');
// else if (*b == Ch('"')) result += Ch('\\'), result += Ch('"');
// else if (*b == Ch('\\')) result += Ch('\\'), result += Ch('\\');
// else
// result += *b;
// ++b;
// }
// return result;
//}

// Freely inspired from write_json_helper in boost
template <class Ptree>
void mergeInto(const Ptree& pt, /*ptree& destination, */ptree& parent, const std::string& fullPath , int indent)
void mergeInto(const Ptree& pt, ptree& parent, const std::string& fullPath , int indent)
{
typedef typename Ptree::key_type::value_type Ch;
typedef typename std::basic_string<Ch> Str;

// Value or object or array
if (indent > 0 && pt.empty()) { // Handle value
Str data = /*create_escapes(*/pt.template get_value<Str>()/*)*/;
Str data = pt.template get_value<Str>();
parent.add(fullPath, data);
} else if (indent > 0 && pt.count(Str()) == pt.size()) { // Handle array
typename Ptree::const_iterator it = pt.begin();
cout << "ARRAY " << endl;

// does the array exist in the destination ?
// ptree children;
// if ( parent.count(fullPath) > 0) {
// cout << " already exist in parent" << endl;
// children = parent.get_child(fullPath);
// } else {
// cout << " does not exist in parent" << endl;
// }

for (; it != pt.end(); ++it) {
ptree object;
// std::string newFullPath = fullPath.empty() ? it->first : fullPath + "." + it->first;
cout << " processing item :" << it->first << endl;
cout << " fullPath :" << fullPath << endl;
// cout << " newFullPath:" << newFullPath << endl;
// mergeInto(it->second, /*destination, */object, "", indent + 1);
// boost::property_tree::write_json(std::cout, object);
// cout << " object: " << endl;
// boost::property_tree::write_json(std::cout, it->second);
parent.add_child(fullPath+".", it->second);
// ptree test = object.get_child("").get_child("");
//
// if(test.empty())
// cout << "Node doesn't have children" << endl;
//
// if(test.data().empty())
// cout << "Node doesn't have data" << endl;
//
// if (object.empty() && !object.data().empty()) { // terminal
// cout << " terminal " << endl;
// children.put("", object.data());
// } else {
// cout << " not termianl " << endl;
// children.add_child("", object.get_child(""));
// }

// parent.add_child(fullPath, object);
// parent.add_child("", object);
// this check is to avoid problems when there is a node with data and we try to add children to it (forbidden in json)
if(parent.get_child_optional(fullPath+".") && !parent.get_child(fullPath+".").data().empty()) {
cerr << "could not add item to array '" + fullPath + ".' because it already contains some data." << endl;
} else {
parent.add_child(fullPath+".", it->second);
}
}

// cout << " children:" << endl;
// boost::property_tree::write_json(std::cout, children);
//
// if ( parent.count(fullPath) == 0) {
// cout << " add to parent" << endl;
// parent.add_child(fullPath, children);
// }

// cout << " add to parent" << endl;
// TODO: check if array exist in parent, if so push_basck the object otherwise create it
// parent.add_child(fullPath, children);
} else { // Handle object
typename Ptree::const_iterator it = pt.begin();
for (; it != pt.end(); ++it) {
std::string newFullPath = fullPath.empty() ? it->first : fullPath + "." + it->first;
mergeInto(it->second, /*destination, */parent, newFullPath, indent + 1);
mergeInto(it->second, parent, newFullPath, indent + 1);
}
}
}

template <class Ptree>
void mergeInto(const Ptree& pt, ptree& destination)
{
mergeInto(pt, destination, destination, "", 1);
mergeInto(pt, destination, "", 1);
}

} // namespace o2::quality_control::test
} // namespace o2::quality_control::core

#endif // QC_PTREE_UTILS_H
8 changes: 8 additions & 0 deletions Framework/include/QualityControl/stringUtils.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,14 @@ bool decodeBool(const std::string& value);
/// @throw std::runtime_error the value is not a bool
bool parseBoolParam(const CustomParameters& customParameters, const std::string& name, const std::string& runType = "default", const std::string& beamType = "default");

/**
* Split a string using delim as delimiter.
* @param str
* @param delim
* @return
*/
std::vector<std::string> splitString(const std::string& str, char delim);

} // namespace o2::quality_control::core

#endif // QC_STRING_UTILS_H
Loading

0 comments on commit e63c5b1

Please sign in to comment.