From 15f54d94eeb6e8821949b5c0b33e0e9d87c4af1a Mon Sep 17 00:00:00 2001 From: Alexander Lednev <57529355+iceseer@users.noreply.github.com> Date: Fri, 8 Apr 2022 15:06:00 +0300 Subject: [PATCH] Feature/rdb burrow storage (#2065) * rdb burrow schema Signed-off-by: iceseer Signed-off-by: Alexander Lednev <57529355+iceseer@users.noreply.github.com> * burrow storage Signed-off-by: iceseer Signed-off-by: Alexander Lednev <57529355+iceseer@users.noreply.github.com> * rdb command executor Signed-off-by: iceseer Signed-off-by: Alexander Lednev <57529355+iceseer@users.noreply.github.com> * rdb burrow spec query executor Signed-off-by: iceseer Signed-off-by: Alexander Lednev <57529355+iceseer@users.noreply.github.com> * tests Signed-off-by: iceseer Signed-off-by: Alexander Lednev <57529355+iceseer@users.noreply.github.com> --- irohad/ametsuchi/CMakeLists.txt | 21 +- .../ametsuchi/impl/rocksdb_burrow_storage.cpp | 165 ++++++++++ .../ametsuchi/impl/rocksdb_burrow_storage.hpp | 65 ++++ .../impl/rocksdb_command_executor.cpp | 86 ++++- .../impl/rocksdb_command_executor.hpp | 4 + irohad/ametsuchi/impl/rocksdb_common.hpp | 307 +++++++++++++++++- .../impl/rocksdb_specific_query_executor.cpp | 121 ++++++- .../impl/rocksdb_specific_query_executor.hpp | 1 + .../ametsuchi/impl/rocksdb_storage_impl.cpp | 9 +- libs/common/to_lower.hpp | 14 + test/integration/executor/CMakeLists.txt | 1 + .../executor_fixture_param_rocksdb.cpp | 25 +- .../executor_fixture_param_rocksdb.hpp | 6 +- .../irohad/ametsuchi/rocksdb_common_test.cpp | 84 +++++ .../ametsuchi/rocksdb_executor_test.cpp | 10 +- 15 files changed, 895 insertions(+), 24 deletions(-) create mode 100644 irohad/ametsuchi/impl/rocksdb_burrow_storage.cpp create mode 100644 irohad/ametsuchi/impl/rocksdb_burrow_storage.hpp diff --git a/irohad/ametsuchi/CMakeLists.txt b/irohad/ametsuchi/CMakeLists.txt index bee6e31d2c1..7626d703512 100644 --- a/irohad/ametsuchi/CMakeLists.txt +++ b/irohad/ametsuchi/CMakeLists.txt @@ -185,15 +185,31 @@ target_link_libraries(rocksdb_indexer RocksDB::rocksdb ) +add_library(common_burrow_storage + impl/burrow_storage.cpp + ) +target_link_libraries(common_burrow_storage + common + ) + +add_library(rocksdb_burrow_storage + impl/rocksdb_burrow_storage.cpp + ) +target_link_libraries(rocksdb_burrow_storage + common_burrow_storage + RocksDB::rocksdb + ) + target_compile_definitions(postgres_indexer PRIVATE SOCI_USE_BOOST HAVE_BOOST ) add_library(postgres_burrow_storage - impl/burrow_storage.cpp impl/postgres_burrow_storage.cpp ) -target_link_libraries(postgres_burrow_storage common) +target_link_libraries(postgres_burrow_storage + common_burrow_storage + ) add_library(default_vm_call INTERFACE) if(USE_BURROW) @@ -222,6 +238,7 @@ target_link_libraries(ametsuchi libs_files common postgres_burrow_storage + rocksdb_burrow_storage postgres_options shared_model_interfaces shared_model_plain_backend diff --git a/irohad/ametsuchi/impl/rocksdb_burrow_storage.cpp b/irohad/ametsuchi/impl/rocksdb_burrow_storage.cpp new file mode 100644 index 00000000000..2fec2c4b408 --- /dev/null +++ b/irohad/ametsuchi/impl/rocksdb_burrow_storage.cpp @@ -0,0 +1,165 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "ametsuchi/impl/rocksdb_burrow_storage.hpp" + +#include +#include + +#include "ametsuchi/impl/rocksdb_common.hpp" +#include "common/obj_utils.hpp" +#include "common/result.hpp" +#include "common/to_lower.hpp" + +using namespace iroha::ametsuchi; +using namespace iroha::expected; + +#define MAKE_LOWER_ON_STACK(name, source, sz) \ + static_assert(sz > 0ull, "Unexpected size " #sz); \ + assert(source.size() <= sz); \ + char name##_buffer[sz]; \ + auto name = toLower(source, name##_buffer); + +RocksdbBurrowStorage::RocksdbBurrowStorage( + RocksDbCommon &common, + std::string_view tx_hash, + shared_model::interface::types::CommandIndexType cmd_index) + : common_(common), tx_hash_(tx_hash), cmd_index_(cmd_index) {} + +Result, std::string> +RocksdbBurrowStorage::getAccount(std::string_view address) { + MAKE_LOWER_ON_STACK(address_lc, address, 128); + RDB_TRY_GET_VALUE_OR_STR_ERR( + opt_data, + forCallEngineAccount( + common_, address_lc)); + if (opt_data) + return expected::makeValue(std::string(opt_data->data(), opt_data->size())); + + return std::nullopt; +} + +Result RocksdbBurrowStorage::updateAccount( + std::string_view address, std::string_view account) { + MAKE_LOWER_ON_STACK(address_lc, address, 128); + common_.valueBuffer().assign(account.data(), account.size()); + RDB_ERROR_CHECK_TO_STR( + forCallEngineAccount(common_, address_lc)); + return {}; +} + +Result RocksdbBurrowStorage::removeAccount( + std::string_view address) { + MAKE_LOWER_ON_STACK(address_lc, address, 128); + RDB_ERROR_CHECK_TO_STR( + forCallEngineAccount( + common_, address_lc)); + + auto const &[_, status] = + common_.filterDelete(std::numeric_limits::max(), + RocksDBPort::ColumnFamilyType::kWsv, + fmtstrings::kPathEngineStorage, + address_lc); + + if (!status.ok() && !status.IsNotFound()) + return expected::makeError(fmt::format( + "Delete CallEngine storage with address '{}' failed.", address_lc)); + + return {}; +} + +Result, std::string> +RocksdbBurrowStorage::getStorage(std::string_view address, + std::string_view key) { + MAKE_LOWER_ON_STACK(address_lc, address, 128); + + std::string key_lc; + toLowerAppend(key, key_lc); + + RDB_TRY_GET_VALUE_OR_STR_ERR( + opt_value, + forCallEngineStorage( + common_, address_lc, key_lc)); + if (opt_value) + return expected::makeValue( + std::string(opt_value->data(), opt_value->size())); + + return std::nullopt; +} + +Result RocksdbBurrowStorage::setStorage( + std::string_view address, std::string_view key, std::string_view value) { + MAKE_LOWER_ON_STACK(address_lc, address, 128); + + std::string key_lc; + toLowerAppend(key, key_lc); + + common_.valueBuffer().assign(value.data(), value.size()); + RDB_ERROR_CHECK_TO_STR( + forCallEngineStorage(common_, address_lc, key_lc)); + return {}; +} + +Result RocksdbBurrowStorage::initCallId() { + if (!call_id_cache_) { + RDB_ERROR_CHECK_TO_STR( + forCallEngineCallIds( + common_, tx_hash_, cmd_index_)); + RDB_TRY_GET_VALUE_OR_STR_ERR( + opt_call_id, + forCallEngineNextCallIds( + common_)); + if (opt_call_id) + call_id_cache_ = std::make_pair(*opt_call_id, 0ull); + else + call_id_cache_ = std::make_pair(0ull, 0ull); + + common_.encode(call_id_cache_->first); + RDB_ERROR_CHECK_TO_STR(forCallEngineCallIds( + common_, tx_hash_, cmd_index_)); + + common_.encode(call_id_cache_->first + 1ull); + RDB_ERROR_CHECK_TO_STR( + forCallEngineNextCallIds(common_)); + } + return {}; +} + +Result RocksdbBurrowStorage::storeLog( + std::string_view address, + std::string_view data, + std::vector topics) { + if (!call_id_cache_) { + RDB_ERROR_CHECK(initCallId()); + } + + uint64_t log_idx = 0ull; + RDB_TRY_GET_VALUE_OR_STR_ERR( + opt_log_idx, + forCallEngineNextLogIx(common_)); + if (opt_log_idx) + log_idx = *opt_log_idx; + + common_.encode(log_idx + 1ull); + RDB_ERROR_CHECK_TO_STR(forCallEngineNextLogIx(common_)); + + MAKE_LOWER_ON_STACK(address_lc, address, 128); + common_.valueBuffer() = std::to_string(log_idx); + common_.valueBuffer() += '#'; + common_.valueBuffer() += address_lc; + common_.valueBuffer() += '#'; + common_.valueBuffer() += data; + RDB_ERROR_CHECK_TO_STR(forCallEngineLogs( + common_, call_id_cache_->first, call_id_cache_->second++)); + + for (uint64_t ix = 0; ix < topics.size(); ++ix) { + auto const &topic = topics[ix]; + common_.valueBuffer().assign(topic.data(), topic.size()); + RDB_ERROR_CHECK_TO_STR( + forCallEngineTopics(common_, log_idx, ix)); + } + + return {}; +} diff --git a/irohad/ametsuchi/impl/rocksdb_burrow_storage.hpp b/irohad/ametsuchi/impl/rocksdb_burrow_storage.hpp new file mode 100644 index 00000000000..e40bd0bad5c --- /dev/null +++ b/irohad/ametsuchi/impl/rocksdb_burrow_storage.hpp @@ -0,0 +1,65 @@ +/** + * Copyright Soramitsu Co., Ltd. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef IROHA_RDB_BURROW_STORAGE_HPP +#define IROHA_RDB_BURROW_STORAGE_HPP + +#include "ametsuchi/burrow_storage.hpp" + +#include +#include + +#include "interfaces/common_objects/types.hpp" + +namespace iroha::ametsuchi { + class RocksDbCommon; + + class RocksdbBurrowStorage : public BurrowStorage { + public: + RocksdbBurrowStorage( + RocksDbCommon &common, + std::string_view tx_hash, + shared_model::interface::types::CommandIndexType cmd_index); + + expected::Result, std::string> getAccount( + std::string_view address) override; + + expected::Result updateAccount( + std::string_view address, std::string_view account) override; + + expected::Result removeAccount( + std::string_view address) override; + + expected::Result, std::string> getStorage( + std::string_view address, std::string_view key) override; + + expected::Result setStorage( + std::string_view address, + std::string_view key, + std::string_view value) override; + + expected::Result storeLog( + std::string_view address, + std::string_view data, + std::vector topics) override; + + std::optional getCallId() const { + if (call_id_cache_) + return call_id_cache_->first; + return std::nullopt; + } + + expected::Result initCallId(); + + private: + RocksDbCommon &common_; + std::string_view tx_hash_; + shared_model::interface::types::CommandIndexType cmd_index_; + std::optional> call_id_cache_; + }; + +} // namespace iroha::ametsuchi + +#endif // IROHA_RDB_BURROW_STORAGE_HPP diff --git a/irohad/ametsuchi/impl/rocksdb_command_executor.cpp b/irohad/ametsuchi/impl/rocksdb_command_executor.cpp index 5e943272490..215aaa9c4c5 100644 --- a/irohad/ametsuchi/impl/rocksdb_command_executor.cpp +++ b/irohad/ametsuchi/impl/rocksdb_command_executor.cpp @@ -10,6 +10,8 @@ #include #include #include "ametsuchi/impl/executor_common.hpp" +#include "ametsuchi/impl/rocksdb_burrow_storage.hpp" +#include "ametsuchi/impl/rocksdb_specific_query_executor.hpp" #include "ametsuchi/setting_query.hpp" #include "ametsuchi/vm_caller.hpp" #include "common/to_lower.hpp" @@ -34,6 +36,7 @@ #include "interfaces/commands/set_setting_value.hpp" #include "interfaces/commands/subtract_asset_quantity.hpp" #include "interfaces/commands/transfer_asset.hpp" +#include "interfaces/common_objects/string_view_types.hpp" #include "main/rdb_status.hpp" #include "main/subscription.hpp" @@ -49,9 +52,11 @@ using shared_model::interface::RolePermissionSet; RocksDbCommandExecutor::RocksDbCommandExecutor( std::shared_ptr db_context, std::shared_ptr perm_converter, + std::shared_ptr specific_query_executor, std::optional> vm_caller) : db_context_(std::move(db_context)), perm_converter_{std::move(perm_converter)}, + specific_query_executor_{std::move(specific_query_executor)}, vm_caller_{vm_caller}, db_transaction_(db_context_) { assert(db_context_); @@ -342,10 +347,85 @@ RocksDbCommandExecutor::ExecutionResult RocksDbCommandExecutor::operator()( const shared_model::interface::CallEngine &command, const shared_model::interface::types::AccountIdType &creator_account_id, const std::string &tx_hash, - shared_model::interface::types::CommandIndexType /*cmd_index*/, - bool /*do_validation*/, + shared_model::interface::types::CommandIndexType cmd_index, + bool do_validation, shared_model::interface::RolePermissionSet const &creator_permissions) { - return makeError(ErrorCodes::kNoImplementation, "Not implemented"); + if (!vm_caller_) + return makeError(ErrorCodes::kNotConfigured, + "Engine is not configured."); + + auto const &[creator_account_name, creator_domain_id] = + staticSplitId<2>(creator_account_id); + + GrantablePermissionSet granted_account_permissions; + RDB_TRY_GET_VALUE( + opt_permissions, + forGrantablePermissions( + common, creator_account_name, creator_domain_id, command.caller())); + if (opt_permissions) + granted_account_permissions = *opt_permissions; + + if (do_validation) + RDB_ERROR_CHECK(checkPermissions(creator_permissions, + granted_account_permissions, + Role::kCallEngine, + Grantable::kCallEngineOnMyBehalf)); + + RocksdbBurrowStorage burrow_storage(common, tx_hash, cmd_index); + return vm_caller_->get() + .call( + tx_hash, + cmd_index, + shared_model::interface::types::EvmCodeHexStringView{command.input()}, + command.caller(), + command.callee() + ? std::optional{command.callee() + ->get()} + : std::optional{std::nullopt}, + burrow_storage, + *this, + *specific_query_executor_) + .match( + [&](const auto &value) -> RocksDbCommandExecutor::ExecutionResult { + if (!burrow_storage.getCallId()) + if (auto result = burrow_storage.initCallId(); + expected::hasError(result)) + return makeError(ErrorCodes::kNotConfigured, + "initCallId error: {}", + result.assumeError()); + + assert(burrow_storage.getCallId()); + if (command.callee()) { + common.valueBuffer() = *command.callee(); + common.valueBuffer() += '|'; + if (value.value) + common.valueBuffer() += *value.value; + if (auto result = forCallEngineCallResponse( + common, *burrow_storage.getCallId()); + expected::hasError(result)) + return makeError( + result.template assumeError().code, + "CallEngineResponse: {}", + result.template assumeError().description); + } else { + if (value.value) + common.valueBuffer() = *value.value; + if (auto result = forCallEngineDeploy( + common, *burrow_storage.getCallId()); + expected::hasError(result)) + return makeError( + result.template assumeError().code, + "CallEngineDeploy: {}", + result.template assumeError().description); + } + + return {}; + }, + [](auto &&error) -> RocksDbCommandExecutor::ExecutionResult { + return makeError(3, "CallEngine: {}", std::move(error.error)); + }); } RocksDbCommandExecutor::ExecutionResult RocksDbCommandExecutor::operator()( diff --git a/irohad/ametsuchi/impl/rocksdb_command_executor.hpp b/irohad/ametsuchi/impl/rocksdb_command_executor.hpp index b272c73b526..366f708a4e5 100644 --- a/irohad/ametsuchi/impl/rocksdb_command_executor.hpp +++ b/irohad/ametsuchi/impl/rocksdb_command_executor.hpp @@ -44,6 +44,7 @@ namespace shared_model::interface { namespace iroha::ametsuchi { + class RocksDbSpecificQueryExecutor; class VmCaller; class RocksDbCommandExecutor final : public CommandExecutor { @@ -51,6 +52,7 @@ namespace iroha::ametsuchi { using ExecutionResult = expected::Result; enum ErrorCodes { + kNotConfigured = 1, kNoPermissions = 2, kNoAccount = 3, kInvalidAmount = 3, @@ -74,6 +76,7 @@ namespace iroha::ametsuchi { std::shared_ptr db_context, std::shared_ptr perm_converter, + std::shared_ptr specific_query_executor, std::optional> vm_caller); ~RocksDbCommandExecutor(); @@ -273,6 +276,7 @@ namespace iroha::ametsuchi { std::shared_ptr db_context_; std::shared_ptr perm_converter_; + std::shared_ptr specific_query_executor_; std::optional> vm_caller_; RocksDbTransaction db_transaction_; }; diff --git a/irohad/ametsuchi/impl/rocksdb_common.hpp b/irohad/ametsuchi/impl/rocksdb_common.hpp index c7bfa050eb7..cdae2bc4cb8 100644 --- a/irohad/ametsuchi/impl/rocksdb_common.hpp +++ b/irohad/ametsuchi/impl/rocksdb_common.hpp @@ -115,6 +115,29 @@ * | +- * | +- * | + * +-|EVM_STORAGE|-+-|ENGINE_CALLS|-+- + * | | +- + * | | +- + * | | + * | +-|EC_DEPLOYS|-+- + * | | +- + * | | + * | +-|EC_CON_CALLS|-+- + * | | +- + * | | + * | +-|ACCOUNT|-+- + * | | +- + * | | + * | +-|LOGS|-+- + * | | +- + * | | +- + * | | + * | +-|TOPICS|-+- + * | | +- + * | | + * | +-|ACCOUNT_KV|-+- + * | +- + * | * +- * * @@ -147,6 +170,13 @@ * ### OPTIONS ## O ### * ### ADDRESS ## M ### * ### TLS ## N ### + * ### ENGINE_CALLS ## e ### + * ### ACCOUNT_KV ## A ### + * ### EVM_STORAGE ## E ### + * ### EC_DEPLOYS ## W ### + * ### EC_CON_CALLS ## R ### + * ### LOGS ## y ### + * ### TOPICS ## Y ### * ###################################### * * ###################################### @@ -158,6 +188,7 @@ * ### F_PEERS COUNT ## Z ### * ### F_TOTAL COUNT ## V ### * ### F_VERSION ## v ### + * ### F_NEXT_ID ## X ### * ###################################### * * ###################################### @@ -193,6 +224,13 @@ #define RDB_OPTIONS "O" #define RDB_ADDRESS "M" #define RDB_TLS "N" +#define RDB_ENGINE_CALLS "e" +#define RDB_ACCOUNT_KV "A" +#define RDB_EVM_STORAGE "E" +#define RDB_EC_DEPLOYS "W" +#define RDB_EC_CON_CALLS "R" +#define RDB_LOGS "y" +#define RDB_TOPICS "Y" #define RDB_F_QUORUM "q" #define RDB_F_ASSET_SIZE "I" @@ -200,6 +238,7 @@ #define RDB_F_PEERS_COUNT "Z" #define RDB_F_TOTAL_COUNT "V" #define RDB_F_VERSION "v" +#define RDB_F_NEXT_ID "X" #define RDB_PATH_DOMAIN RDB_ROOT /**/ RDB_WSV /**/ RDB_DOMAIN /**/ RDB_XXX #define RDB_PATH_ACCOUNT RDB_PATH_DOMAIN /**/ RDB_ACCOUNTS /**/ RDB_XXX @@ -237,6 +276,11 @@ namespace iroha::ametsuchi::fmtstrings { FMT_STRING(RDB_ROOT /**/ RDB_WSV /**/ RDB_NETWORK /**/ RDB_S_PEERS /**/ RDB_ADDRESS)}; + // hash ➡️ call_id + static auto constexpr kPathEngineCallIds{ + FMT_STRING(RDB_ROOT /**/ RDB_WSV /**/ RDB_EVM_STORAGE /**/ + RDB_ENGINE_CALLS /**/ RDB_XXX)}; + // domain_id/account_name static auto constexpr kPathSignatories{ FMT_STRING(RDB_PATH_ACCOUNT /**/ RDB_SIGNATORIES)}; @@ -245,11 +289,21 @@ namespace iroha::ametsuchi::fmtstrings { static auto constexpr kPathRoles{ FMT_STRING(RDB_ROOT /**/ RDB_WSV /**/ RDB_ROLES)}; + // call_id ➡️ log_ix/address/data + static auto constexpr kPathEngineLogs{ + FMT_STRING(RDB_ROOT /**/ RDB_WSV /**/ RDB_EVM_STORAGE /**/ + RDB_LOGS /**/ RDB_XXX)}; + // account static auto constexpr kPathTransactionByTs{ FMT_STRING(RDB_ROOT /**/ RDB_WSV /**/ RDB_TRANSACTIONS /**/ RDB_ACCOUNTS /**/ RDB_XXX /**/ RDB_TIMESTAMP)}; + // address + static auto constexpr kPathEngineStorage{ + FMT_STRING(RDB_ROOT /**/ RDB_WSV /**/ RDB_EVM_STORAGE /**/ + RDB_ACCOUNT_KV /**/ RDB_XXX)}; + // account static auto constexpr kPathTransactionByPosition{ FMT_STRING(RDB_ROOT /**/ RDB_WSV /**/ RDB_TRANSACTIONS /**/ @@ -263,6 +317,11 @@ namespace iroha::ametsuchi::fmtstrings { static auto constexpr kPathAccountAssets{ FMT_STRING(RDB_PATH_ACCOUNT /**/ RDB_ASSETS)}; + // log_ix ➡️ topic + static auto constexpr kPathEngineTopics{ + FMT_STRING(RDB_ROOT /**/ RDB_WSV /**/ RDB_EVM_STORAGE /**/ + RDB_TOPICS /**/ RDB_XXX)}; + /** * ###################################### * ############# FOLDERS ################ @@ -272,6 +331,41 @@ namespace iroha::ametsuchi::fmtstrings { static auto constexpr kBlockDataInStore{ FMT_STRING(RDB_ROOT /**/ RDB_STORE /**/ RDB_XXX)}; + // hash/index ➡️ call_id + static auto constexpr kEngineCallId{ + FMT_STRING(RDB_ROOT /**/ RDB_WSV /**/ RDB_EVM_STORAGE /**/ + RDB_ENGINE_CALLS /**/ RDB_XXX /**/ RDB_XXX)}; + + // address ➡️ account + static auto constexpr kEngineAccount{ + FMT_STRING(RDB_ROOT /**/ RDB_WSV /**/ RDB_EVM_STORAGE /**/ + RDB_ACCOUNTS /**/ RDB_XXX)}; + + // address/key ➡️ value + static auto constexpr kEngineStorage{ + FMT_STRING(RDB_ROOT /**/ RDB_WSV /**/ RDB_EVM_STORAGE /**/ + RDB_ACCOUNT_KV /**/ RDB_XXX /**/ RDB_XXX)}; + + // call_id ➡️ contract address + static auto constexpr kEngineDeploy{ + FMT_STRING(RDB_ROOT /**/ RDB_WSV /**/ RDB_EVM_STORAGE /**/ + RDB_EC_DEPLOYS /**/ RDB_XXX)}; + + // call_id ➡️ callee/response + static auto constexpr kEngineCallResponse{ + FMT_STRING(RDB_ROOT /**/ RDB_WSV /**/ RDB_EVM_STORAGE /**/ + RDB_EC_CON_CALLS /**/ RDB_XXX)}; + + // call_id/ix ➡️ log_ix/address/data + static auto constexpr kEngineCallLogs{ + FMT_STRING(RDB_ROOT /**/ RDB_WSV /**/ RDB_EVM_STORAGE /**/ + RDB_LOGS /**/ RDB_XXX /**/ RDB_XXX)}; + + // log_ix/ix ➡️ topic + static auto constexpr kEngineCallTopics{ + FMT_STRING(RDB_ROOT /**/ RDB_WSV /**/ RDB_EVM_STORAGE /**/ + RDB_TOPICS /**/ RDB_XXX /**/ RDB_XXX)}; + // account/height/index/ts ➡️ tx_hash static auto constexpr kTransactionByPosition{FMT_STRING( RDB_ROOT /**/ RDB_WSV /**/ RDB_TRANSACTIONS /**/ RDB_ACCOUNTS /**/ @@ -360,6 +454,16 @@ namespace iroha::ametsuchi::fmtstrings { // domain_id ➡️ default role static auto constexpr kDomain{FMT_STRING(RDB_PATH_DOMAIN)}; + // "" ➡️ next_call_id + static auto constexpr kEngineNextCallId{ + FMT_STRING(RDB_ROOT /**/ RDB_WSV /**/ RDB_EVM_STORAGE /**/ + RDB_ENGINE_CALLS /**/ RDB_F_NEXT_ID)}; + + // "" ➡️ next_log_id + static auto constexpr kEngineNextLogId{ + FMT_STRING(RDB_ROOT /**/ RDB_WSV /**/ RDB_EVM_STORAGE /**/ + RDB_LOGS /**/ RDB_F_NEXT_ID)}; + // "" ➡️ height # hash static auto constexpr kTopBlock{ FMT_STRING(RDB_ROOT /**/ RDB_WSV /**/ RDB_NETWORK /**/ RDB_STORE /**/ @@ -675,7 +779,7 @@ namespace iroha::ametsuchi { #define RDB_ERROR_CHECK_TO_STR(...) \ if (auto _tmp_gen_var = (__VA_ARGS__); \ iroha::expected::hasError(_tmp_gen_var)) \ - return _tmp_gen_var.assumeError().description + return iroha::expected::makeError(_tmp_gen_var.assumeError().description) #define RDB_TRY_GET_VALUE(name, ...) \ typename decltype(__VA_ARGS__)::ValueInnerType name; \ @@ -685,12 +789,12 @@ namespace iroha::ametsuchi { else \ name = std::move(_tmp_gen_var.assumeValue()) -#define RDB_TRY_GET_VALUE_OR_STR_ERR(name, ...) \ - typename decltype(__VA_ARGS__)::ValueInnerType name; \ - if (auto _tmp_gen_var = (__VA_ARGS__); \ - iroha::expected::hasError(_tmp_gen_var)) \ - return _tmp_gen_var.assumeError().description; \ - else \ +#define RDB_TRY_GET_VALUE_OR_STR_ERR(name, ...) \ + typename decltype(__VA_ARGS__)::ValueInnerType name; \ + if (auto _tmp_gen_var = (__VA_ARGS__); \ + iroha::expected::hasError(_tmp_gen_var)) \ + return iroha::expected::makeError(_tmp_gen_var.assumeError().description); \ + else \ name = std::move(_tmp_gen_var.assumeValue()) /** @@ -1218,8 +1322,17 @@ namespace iroha::ametsuchi { kOp != kDbOperation::kDel || kSc != kDbEntry::kMustExist, "Delete operation does not report if key existed before deletion!"); - RDB_ERROR_CHECK(checkStatus( - status, std::forward(op_formatter))); + if constexpr (kOp == kDbOperation::kPut) { + RDB_ERROR_CHECK(checkStatus( + status, std::forward(op_formatter))); + } else if constexpr (kOp == kDbOperation::kDel) { + RDB_ERROR_CHECK(checkStatus( + status, std::forward(op_formatter))); + } else { + RDB_ERROR_CHECK(checkStatus( + status, std::forward(op_formatter))); + } + return status; } @@ -1602,6 +1715,182 @@ namespace iroha::ametsuchi { tx_hash.blob().size())); } + /** + * Access to Call Engine Account data + * @tparam kOp @see kDbOperation + * @tparam kSc @see kDbEntry + * @param common @see RocksDbCommon + * @param address is internal evm address with relative account + * @return operation result + */ + template + inline expected::Result, DbError> + forCallEngineAccount(RocksDbCommon &common, std::string_view address) { + return dbCall( + common, + RocksDBPort::ColumnFamilyType::kWsv, + fmtstrings::kEngineAccount, + address); + } + + /** + * Access to Call Engine Storage data + * @tparam kOp @see kDbOperation + * @tparam kSc @see kDbEntry + * @param common @see RocksDbCommon + * @param address is internal evm address with relative account + * @param key for the storage + * @return operation result + */ + template + inline expected::Result, DbError> + forCallEngineStorage(RocksDbCommon &common, + std::string_view address, + std::string_view key) { + return dbCall( + common, + RocksDBPort::ColumnFamilyType::kWsv, + fmtstrings::kEngineStorage, + address, + key); + } + + /** + * Access to Call Engine Call Ids data + * @tparam kOp @see kDbOperation + * @tparam kSc @see kDbEntry + * @param common @see RocksDbCommon + * @param hash of the tx + * @param cmd_index of the command inside the tx + * @return call_id operation result + */ + template + inline expected::Result, DbError> + forCallEngineCallIds(RocksDbCommon &common, + std::string_view hash, + uint32_t cmd_index) { + return dbCall(common, + RocksDBPort::ColumnFamilyType::kWsv, + fmtstrings::kEngineCallId, + hash, + cmd_index); + } + + /** + * Access to Call Engine Deploy data + * @tparam kOp @see kDbOperation + * @tparam kSc @see kDbEntry + * @param common @see RocksDbCommon + * @param call_id of the CallEngine with contract deploy + * @return address of the contract + */ + template + inline expected::Result, DbError> + forCallEngineDeploy(RocksDbCommon &common, uint64_t call_id) { + return dbCall( + common, + RocksDBPort::ColumnFamilyType::kWsv, + fmtstrings::kEngineDeploy, + call_id); + } + + /** + * Access to Call Engine Call Responses data + * @tparam kOp @see kDbOperation + * @tparam kSc @see kDbEntry + * @param common @see RocksDbCommon + * @param call_id of the CallEngine with contract call + * @return callee + engine response + */ + template + inline expected::Result, DbError> + forCallEngineCallResponse(RocksDbCommon &common, uint64_t call_id) { + return dbCall( + common, + RocksDBPort::ColumnFamilyType::kWsv, + fmtstrings::kEngineCallResponse, + call_id); + } + + /** + * Access to Call Engine Topics data + * @tparam kOp @see kDbOperation + * @tparam kSc @see kDbEntry + * @param common @see RocksDbCommon + * @param log_ix of the topic + * @param ix is increment + * @return topic data + */ + template + inline expected::Result, DbError> + forCallEngineTopics(RocksDbCommon &common, uint64_t log_ix, uint64_t ix) { + return dbCall( + common, + RocksDBPort::ColumnFamilyType::kWsv, + fmtstrings::kEngineCallTopics, + log_ix, + ix); + } + + /** + * Access to Call Engine Logs data + * @tparam kOp @see kDbOperation + * @tparam kSc @see kDbEntry + * @param common @see RocksDbCommon + * @param call_id of the topic + * @param ix is increment + * @return logs data + */ + template + inline expected::Result, DbError> + forCallEngineLogs(RocksDbCommon &common, uint64_t call_id, uint64_t ix) { + return dbCall( + common, + RocksDBPort::ColumnFamilyType::kWsv, + fmtstrings::kEngineCallLogs, + call_id, + ix); + } + + /** + * Access to Call Engine Next Call Id + * @tparam kOp @see kDbOperation + * @tparam kSc @see kDbEntry + * @param common @see RocksDbCommon + * @return next call_id operation result + */ + template + inline expected::Result, DbError> + forCallEngineNextCallIds(RocksDbCommon &common) { + return dbCall(common, + RocksDBPort::ColumnFamilyType::kWsv, + fmtstrings::kEngineNextCallId); + } + + /** + * Access to Call Engine Next Log Id + * @tparam kOp @see kDbOperation + * @tparam kSc @see kDbEntry + * @param common @see RocksDbCommon + * @return next log_ix operation result + */ + template + inline expected::Result, DbError> + forCallEngineNextLogIx(RocksDbCommon &common) { + return dbCall(common, + RocksDBPort::ColumnFamilyType::kWsv, + fmtstrings::kEngineNextLogId); + } + /** * Access to transactions by position * @tparam kOp @see kDbOperation diff --git a/irohad/ametsuchi/impl/rocksdb_specific_query_executor.cpp b/irohad/ametsuchi/impl/rocksdb_specific_query_executor.cpp index 4943098d1a5..b9150ba1d78 100644 --- a/irohad/ametsuchi/impl/rocksdb_specific_query_executor.cpp +++ b/irohad/ametsuchi/impl/rocksdb_specific_query_executor.cpp @@ -13,6 +13,7 @@ #include "ametsuchi/impl/executor_common.hpp" #include "ametsuchi/impl/rocksdb_common.hpp" #include "backend/plain/account_detail_record_id.hpp" +#include "backend/plain/engine_receipt.hpp" #include "backend/plain/peer.hpp" #include "common/bind.hpp" #include "common/common.hpp" @@ -994,5 +995,123 @@ operator()( const shared_model::interface::types::AccountIdType &creator_id, const shared_model::interface::types::HashType &query_hash, shared_model::interface::RolePermissionSet const &creator_permissions) { - throw std::runtime_error(fmt::format("Not implemented")); + auto const &[_, creator_domain_id] = staticSplitId<2ull>(creator_id); + + RDB_ERROR_CHECK(checkPermissions(creator_domain_id, + creator_domain_id, + creator_id, + creator_id, + creator_permissions, + Role::kGetAllEngineReceipts, + Role::kGetDomainEngineReceipts, + Role::kGetMyEngineReceipts)); + + std::vector> records; + + std::optional error; + auto status = enumerateKeysAndValues( + common, + [&](auto, auto cid) { + uint64_t call_id; + std::from_chars(cid.data(), cid.data() + cid.size(), call_id); + + std::optional callee; + std::optional + cantract_address; + std::optional + engine_response; + + if (auto result = + forCallEngineCallResponse(common, call_id); + expected::hasError(result)) { + error = fmt::format("CallEngineResponse code: {}, failed: {}", + result.template assumeError().code, + result.template assumeError().description); + return false; + } else if (result.assumeValue()) { + auto const &[callee_, response_] = + staticSplitId<2ull>(*result.assumeValue(), "|"); + callee = callee_; + engine_response = response_; + } + + if (auto result = + forCallEngineDeploy( + common, call_id); + expected::hasError(result)) { + error = fmt::format("CallEngineDeploy code: {}, failed: {}", + result.template assumeError().code, + result.template assumeError().description); + return false; + } else if (result.assumeValue()) { + cantract_address = *result.assumeValue(); + } + + auto record = std::make_unique( + 0ull, //*cmd_index + "", // caller + callee, + cantract_address, + engine_response); + + auto logs_status = enumerateKeysAndValues( + common, + [&](auto, auto l) { + auto const &[log_ix_str, address, data] = + staticSplitId<3ull>(l.ToStringView(), "#"); + uint64_t log_id = std::stoull( + std::string(log_ix_str.data(), log_ix_str.size())); + + auto log = std::make_unique( + shared_model::interface::types::EvmAddressHexString( + address.data(), address.size()), + shared_model::interface::types::EvmDataHexString( + data.data(), data.size())); + + auto topics_status = enumerateKeysAndValues( + common, + [&](auto, auto t) { + auto tstr = t.ToStringView(); + log->addTopic( + shared_model::interface::types::EvmTopicsHexString( + tstr.data(), tstr.size())); + return true; + }, + RocksDBPort::ColumnFamilyType::kWsv, + fmtstrings::kPathEngineTopics, + log_id); + if (!topics_status.ok()) { + error = fmt::format("enumerate CallEngineTopics failed."); + return false; + } + + record->getMutableLogs().emplace_back(std::move(log)); + return true; + }, + RocksDBPort::ColumnFamilyType::kWsv, + fmtstrings::kPathEngineLogs, + call_id); + if (!logs_status.ok()) { + error = fmt::format("enumerate CallEngineLogs failed."); + return false; + } + + records.emplace_back(std::move(record)); + return true; + }, + RocksDBPort::ColumnFamilyType::kWsv, + fmtstrings::kPathEngineCallIds, + query.txHash()); + RDB_ERROR_CHECK(canExist(status, [&] { + return fmt::format("PathEngineCallsIds enumeration failed: {}", + query.txHash()); + })); + + if (error) + return makeError( + ErrorCodes::kGetReceipts, "GetEngineReceipts failed: {}", *error); + + return query_response_factory_->createEngineReceiptsResponse(records, + query_hash); } diff --git a/irohad/ametsuchi/impl/rocksdb_specific_query_executor.hpp b/irohad/ametsuchi/impl/rocksdb_specific_query_executor.hpp index f4b26d9ac8e..38c3cdb193a 100644 --- a/irohad/ametsuchi/impl/rocksdb_specific_query_executor.hpp +++ b/irohad/ametsuchi/impl/rocksdb_specific_query_executor.hpp @@ -52,6 +52,7 @@ namespace iroha::ametsuchi { kQueryHeightOverflow = 3, kAssetNotFound = 4, kNoTransaction = 4, + kGetReceipts = 5, kRetrieveTransactionsFailed = 1010, }; diff --git a/irohad/ametsuchi/impl/rocksdb_storage_impl.cpp b/irohad/ametsuchi/impl/rocksdb_storage_impl.cpp index 7ebba4dea19..63c329fb833 100644 --- a/irohad/ametsuchi/impl/rocksdb_storage_impl.cpp +++ b/irohad/ametsuchi/impl/rocksdb_storage_impl.cpp @@ -99,7 +99,14 @@ namespace iroha::ametsuchi { expected::Result, std::string> RocksDbStorageImpl::createCommandExecutor() { return std::make_unique( - db_context_, permConverter(), vmCaller()); + db_context_, + permConverter(), + std::make_shared(db_context_, + *blockStore(), + pendingTxStorage(), + queryResponseFactory(), + permConverter()), + vmCaller()); } expected::Result, std::string> diff --git a/libs/common/to_lower.hpp b/libs/common/to_lower.hpp index de7f83991c3..bb628dc7d6d 100644 --- a/libs/common/to_lower.hpp +++ b/libs/common/to_lower.hpp @@ -6,6 +6,7 @@ #ifndef IROHA_TO_LOWER_HPP #define IROHA_TO_LOWER_HPP +#include #include namespace iroha { @@ -16,5 +17,18 @@ namespace iroha { return dst; } + template + inline std::string_view toLower(std::string_view src, char (&dst)[N]) { + assert(N >= src.size()); + + char const *from = src.data(); + char const *const end = src.data() + src.size(); + char *ptr = dst; + + while (from != end) *ptr++ = std::tolower(*from++); + + return std::string_view(dst, src.size()); + } + } // namespace iroha #endif // IROHA_TO_LOWER_HPP diff --git a/test/integration/executor/CMakeLists.txt b/test/integration/executor/CMakeLists.txt index e3b362c2203..fee147a4cb0 100644 --- a/test/integration/executor/CMakeLists.txt +++ b/test/integration/executor/CMakeLists.txt @@ -26,6 +26,7 @@ target_link_libraries(executor_fixture_param_postgres add_library(executor_fixture_param_rocksdb executor_fixture_param_rocksdb.cpp) target_link_libraries(executor_fixture_param_rocksdb ametsuchi_rocksdb + rocksdb_burrow_storage executor_fixture_param GTest::gmock rocksdb_indexer diff --git a/test/integration/executor/executor_fixture_param_rocksdb.cpp b/test/integration/executor/executor_fixture_param_rocksdb.cpp index 955a0ded59c..c95b4b8ba26 100644 --- a/test/integration/executor/executor_fixture_param_rocksdb.cpp +++ b/test/integration/executor/executor_fixture_param_rocksdb.cpp @@ -6,8 +6,10 @@ #include "integration/executor/executor_fixture_param_rocksdb.hpp" #include +#include #include "ametsuchi/impl/block_index_impl.hpp" +#include "ametsuchi/impl/rocksdb_burrow_storage.hpp" #include "ametsuchi/impl/rocksdb_command_executor.hpp" #include "ametsuchi/impl/rocksdb_common.hpp" #include "ametsuchi/impl/rocksdb_indexer.hpp" @@ -32,7 +34,8 @@ using namespace iroha::integration_framework; namespace fs = boost::filesystem; namespace { - ExecutorItfTarget createRocksDBExecutorItfTarget( + std::pair> + createRocksDBExecutorItfTarget( std::shared_ptr db_port, VmCaller &); } // namespace @@ -41,7 +44,11 @@ RocksDBExecutorTestParam::RocksDBExecutorTestParam() { db_port_ = std::make_shared(); db_port_->initialize(db_name_); - executor_itf_target_ = createRocksDBExecutorItfTarget(db_port_, *vm_caller_); + auto const &[executor_itf_target, db] = + createRocksDBExecutorItfTarget(db_port_, *vm_caller_); + executor_itf_target_ = executor_itf_target; + db_context_ = db; + common_ = std::make_unique(db_context_); block_indexer_ = std::make_shared( std::make_unique( @@ -62,7 +69,11 @@ void RocksDBExecutorTestParam::clearBackendState() { db_port_ = std::make_shared(); db_port_->initialize(db_name_); - executor_itf_target_ = createRocksDBExecutorItfTarget(db_port_, *vm_caller_); + auto const &[executor_itf_target, db] = + createRocksDBExecutorItfTarget(db_port_, *vm_caller_); + executor_itf_target_ = executor_itf_target; + db_context_ = db; + common_ = std::make_unique(db_context_); block_indexer_ = std::make_shared( std::make_unique( @@ -78,7 +89,7 @@ std::unique_ptr RocksDBExecutorTestParam::makeBurrowStorage( std::string const &tx_hash, shared_model::interface::types::CommandIndexType cmd_index) const { - return std::unique_ptr{}; + return std::make_unique(*common_, tx_hash, cmd_index); } std::shared_ptr @@ -119,7 +130,8 @@ namespace { std::unique_ptr block_storage_; }; - ExecutorItfTarget createRocksDBExecutorItfTarget( + std::pair> + createRocksDBExecutorItfTarget( std::shared_ptr db_port, VmCaller &vm_caller) { ExecutorItfTarget target; @@ -133,9 +145,10 @@ namespace { target.command_executor = std::make_shared( db_context, std::make_shared(), + query_executor, vm_caller); target.query_executor = std::move(query_executor); - return target; + return std::make_pair(target, db_context); } } // namespace diff --git a/test/integration/executor/executor_fixture_param_rocksdb.hpp b/test/integration/executor/executor_fixture_param_rocksdb.hpp index 4e6c3215fc9..0fbe1fd1f1c 100644 --- a/test/integration/executor/executor_fixture_param_rocksdb.hpp +++ b/test/integration/executor/executor_fixture_param_rocksdb.hpp @@ -11,7 +11,9 @@ namespace iroha::ametsuchi { struct RocksDBPort; -} + struct RocksDBContext; + class RocksDbCommon; +} // namespace iroha::ametsuchi namespace executor_testing { @@ -48,6 +50,8 @@ namespace executor_testing { private: std::string db_name_; std::shared_ptr db_port_; + std::shared_ptr db_context_; + std::unique_ptr common_; iroha::integration_framework::ExecutorItfTarget executor_itf_target_; std::shared_ptr block_indexer_; diff --git a/test/module/irohad/ametsuchi/rocksdb_common_test.cpp b/test/module/irohad/ametsuchi/rocksdb_common_test.cpp index afccdfb6ad5..4a28aa30f9b 100644 --- a/test/module/irohad/ametsuchi/rocksdb_common_test.cpp +++ b/test/module/irohad/ametsuchi/rocksdb_common_test.cpp @@ -724,6 +724,90 @@ TEST_F(RocksDBTest, LowerBoundSearch) { } } +TEST_F(RocksDBTest, LogsEnumerator) { + { + RocksDbCommon common(tx_context_); + + common.valueBuffer() = "aaa"; + ASSERT_TRUE(iroha::expected::hasValue( + forCallEngineLogs(common, 50, 0))); + ASSERT_TRUE(iroha::expected::hasValue( + forCallEngineLogs(common, 50, 1))); + ASSERT_TRUE(iroha::expected::hasValue( + forCallEngineLogs(common, 50, 2))); + ASSERT_TRUE(common.commit().ok()); + } + + { + RocksDbCommon common(tx_context_); + bool found[3]; + memset(found, 0, sizeof(found)); + enumerateKeysAndValues(common, + [&](auto key, auto value) { + throw std::runtime_error("Unexpected"); + return true; + }, + RocksDBPort::ColumnFamilyType::kWsv, + fmtstrings::kPathEngineLogs, + 10); + enumerateKeysAndValues(common, + [&](auto key, auto value) { + found[atoll(key.data())] = + (value.ToStringView() == "aaa"); + return true; + }, + RocksDBPort::ColumnFamilyType::kWsv, + fmtstrings::kPathEngineLogs, + 50); + + ASSERT_TRUE(found[0]); + ASSERT_TRUE(found[1]); + ASSERT_TRUE(found[2]); + } +} + +TEST_F(RocksDBTest, TopicsEnumerator) { + { + RocksDbCommon common(tx_context_); + + common.valueBuffer() = "aaa"; + ASSERT_TRUE(iroha::expected::hasValue( + forCallEngineTopics(common, 50, 0))); + ASSERT_TRUE(iroha::expected::hasValue( + forCallEngineTopics(common, 50, 1))); + ASSERT_TRUE(iroha::expected::hasValue( + forCallEngineTopics(common, 50, 2))); + ASSERT_TRUE(common.commit().ok()); + } + + { + RocksDbCommon common(tx_context_); + bool found[3]; + memset(found, 0, sizeof(found)); + enumerateKeysAndValues(common, + [&](auto key, auto value) { + throw std::runtime_error("Unexpected"); + return true; + }, + RocksDBPort::ColumnFamilyType::kWsv, + fmtstrings::kPathEngineTopics, + 10); + enumerateKeysAndValues(common, + [&](auto key, auto value) { + found[atoll(key.data())] = + (value.ToStringView() == "aaa"); + return true; + }, + RocksDBPort::ColumnFamilyType::kWsv, + fmtstrings::kPathEngineTopics, + 50); + + ASSERT_TRUE(found[0]); + ASSERT_TRUE(found[1]); + ASSERT_TRUE(found[2]); + } +} + TEST_F(RocksDBTest, Signatories) { RocksDbCommon common(tx_context_); auto cmd_check = [&](std::string_view pk) { diff --git a/test/module/irohad/ametsuchi/rocksdb_executor_test.cpp b/test/module/irohad/ametsuchi/rocksdb_executor_test.cpp index a6f3344c795..e855503b6d8 100644 --- a/test/module/irohad/ametsuchi/rocksdb_executor_test.cpp +++ b/test/module/irohad/ametsuchi/rocksdb_executor_test.cpp @@ -8,6 +8,7 @@ #include "ametsuchi/impl/executor_common.hpp" #include "ametsuchi/impl/rocksdb_command_executor.hpp" #include "ametsuchi/impl/rocksdb_common.hpp" +#include "ametsuchi/impl/rocksdb_specific_query_executor.hpp" #include "ametsuchi/impl/rocksdb_wsv_query.hpp" #include "backend/protobuf/proto_permission_to_string.hpp" #include "backend/protobuf/proto_query_response_factory.hpp" @@ -65,8 +66,15 @@ namespace iroha::ametsuchi { getTestLogger("WsvQuery")); pending_txs_storage = std::make_shared(); + + auto query_executor = + std::make_shared(tx_context_, + *block_storage_, + pending_txs_storage, + query_response_factory, + perm_converter); executor = std::make_unique( - tx_context_, perm_converter, std::nullopt); + tx_context_, perm_converter, query_executor, std::nullopt); } void SetUp() override {