From a39b5b64049aa06ee2c99d86870a09ed12dfd2a6 Mon Sep 17 00:00:00 2001 From: Matthew Kelly Date: Tue, 4 Jan 2022 16:03:37 +0800 Subject: [PATCH 01/62] Refactored rabbitmq publish method. Moved exchange_name_.empty() check to ternary --- programs/rodeos/streams/rabbitmq.hpp | 23 ++++++----------------- 1 file changed, 6 insertions(+), 17 deletions(-) diff --git a/programs/rodeos/streams/rabbitmq.hpp b/programs/rodeos/streams/rabbitmq.hpp index 7e34609739..7bcc3c8f92 100644 --- a/programs/rodeos/streams/rabbitmq.hpp +++ b/programs/rodeos/streams/rabbitmq.hpp @@ -68,24 +68,13 @@ class rabbitmq : public stream_handler { } void publish(const std::vector& data, const std::string& routing_key) override { - if (exchange_name_.empty()) { - if( publish_immediately_ ) { - amqp_publisher_->publish_message_direct( queue_name_, data, - []( const std::string& err ) { - elog( "AMQP direct message error: ${e}", ("e", err) ); - } ); - } else { - queue_.emplace_back( std::make_pair( queue_name_, data ) ); - } + if( publish_immediately_ ) { + amqp_publisher_->publish_message_direct( exchange_name_.empty() ? queue_name_ : routing_key, data, + []( const std::string& err ) { + elog( "AMQP direct message error: ${e}", ("e", err) ); + } ); } else { - if( publish_immediately_ ) { - amqp_publisher_->publish_message_direct( routing_key, data, - []( const std::string& err ) { - elog( "AMQP direct message error: ${e}", ("e", err) ); - } ); - } else { - queue_.emplace_back( std::make_pair( routing_key, data ) ); - } + queue_.emplace_back( std::make_pair( exchange_name_.empty() ? queue_name_ : routing_key, data ) ); } } From a6f8dabbbfcba0a693bb233dab5550553d5e185e Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 7 Jan 2022 12:00:36 -0600 Subject: [PATCH 02/62] Call accept_transaction on net_plugin thread and avoid hop to main thread --- plugins/net_plugin/net_plugin.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/plugins/net_plugin/net_plugin.cpp b/plugins/net_plugin/net_plugin.cpp index 1dfb8cff29..4adb1df952 100644 --- a/plugins/net_plugin/net_plugin.cpp +++ b/plugins/net_plugin/net_plugin.cpp @@ -3267,9 +3267,8 @@ namespace eosio { peer_dlog( this, "received packed_transaction ${id}", ("id", tid) ); trx_in_progress_size += calc_trx_size( trx ); - app().post( priority::low, [trx{std::move(trx)}, weak = weak_from_this()]() { - my_impl->chain_plug->accept_transaction( trx, - [weak, trx](const std::variant& result) mutable { + my_impl->chain_plug->accept_transaction( trx, + [weak = weak_from_this(), trx](const std::variant& result) mutable { // next (this lambda) called from application thread if (std::holds_alternative(result)) { fc_dlog( logger, "bad packed_transaction : ${m}", ("m", std::get(result)->what()) ); @@ -3285,7 +3284,6 @@ namespace eosio { if( conn ) { conn->trx_in_progress_size -= calc_trx_size( trx ); } - }); }); } From 361b87ef2c6b23d7cf29fc7b478fae690916b4ba Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Mon, 10 Jan 2022 14:37:02 -0600 Subject: [PATCH 03/62] Add comment on threaded call --- plugins/producer_plugin/producer_plugin.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/plugins/producer_plugin/producer_plugin.cpp b/plugins/producer_plugin/producer_plugin.cpp index 4e0dae76ce..765fa39b1e 100644 --- a/plugins/producer_plugin/producer_plugin.cpp +++ b/plugins/producer_plugin/producer_plugin.cpp @@ -362,6 +362,7 @@ class producer_plugin_impl : public std::enable_shared_from_this next) { chain::controller& chain = chain_plug->chain(); const auto max_trx_time_ms = _max_transaction_time_ms.load(); From b7ee630d475199dda11d87833ac05cdacad4a662 Mon Sep 17 00:00:00 2001 From: William Blevins Date: Wed, 12 Jan 2022 10:32:43 -0500 Subject: [PATCH 04/62] Add develop-boxed branch to the list for LRT. --- .cicd/generate-pipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index b8b63704e4..4629840c9a 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -98,7 +98,7 @@ if [[ ! -z ${BUILDKITE_TRIGGERED_FROM_BUILD_ID} ]]; then fi export BUILD_SOURCE=${BUILD_SOURCE:---build \$BUILDKITE_BUILD_ID} # set trigger_job if master/release/develop branch and webhook -if [[ ! $BUILDKITE_PIPELINE_SLUG =~ 'lrt' ]] && [[ $BUILDKITE_BRANCH =~ ^release/[0-9]+\.[0-9]+\.x$ || $BUILDKITE_BRANCH =~ ^master$ || $BUILDKITE_BRANCH =~ ^develop$ || "$SKIP_LONG_RUNNING_TESTS" == 'false' ]]; then +if [[ ! $BUILDKITE_PIPELINE_SLUG =~ 'lrt' ]] && [[ $BUILDKITE_BRANCH =~ ^release/[0-9]+\.[0-9]+\.x$ || $BUILDKITE_BRANCH =~ ^master$ || $BUILDKITE_BRANCH =~ ^develop$ || $BUILDKITE_BRANCH =~ ^develop-boxed$ || "$SKIP_LONG_RUNNING_TESTS" == 'false' ]]; then [[ $BUILDKITE_SOURCE != 'schedule' ]] && export TRIGGER_JOB=true fi # run LRTs synchronously when running full test suite From 2e14f36b0ebebf6f8a388127a0acb532288ef635 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Wed, 13 Oct 2021 10:02:28 -0500 Subject: [PATCH 05/62] Remove long deprecated history_plugin --- plugins/history_api_plugin/CMakeLists.txt | 7 - .../history_api_plugin/history_api_plugin.cpp | 49 -- .../history_api_plugin/history_api_plugin.hpp | 28 - plugins/history_plugin/CMakeLists.txt | 7 - plugins/history_plugin/history_plugin.cpp | 594 ------------------ .../account_control_history_object.hpp | 52 -- .../eosio/history_plugin/history_plugin.hpp | 151 ----- .../public_key_history_object.hpp | 51 -- 8 files changed, 939 deletions(-) delete mode 100644 plugins/history_api_plugin/CMakeLists.txt delete mode 100644 plugins/history_api_plugin/history_api_plugin.cpp delete mode 100644 plugins/history_api_plugin/include/eosio/history_api_plugin/history_api_plugin.hpp delete mode 100644 plugins/history_plugin/CMakeLists.txt delete mode 100644 plugins/history_plugin/history_plugin.cpp delete mode 100644 plugins/history_plugin/include/eosio/history_plugin/account_control_history_object.hpp delete mode 100644 plugins/history_plugin/include/eosio/history_plugin/history_plugin.hpp delete mode 100644 plugins/history_plugin/include/eosio/history_plugin/public_key_history_object.hpp diff --git a/plugins/history_api_plugin/CMakeLists.txt b/plugins/history_api_plugin/CMakeLists.txt deleted file mode 100644 index 1858af33b0..0000000000 --- a/plugins/history_api_plugin/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -file( GLOB HEADERS "include/eosio/history_api_plugin/*.hpp" ) -add_library( history_api_plugin - history_api_plugin.cpp - ${HEADERS} ) - -target_link_libraries( history_api_plugin history_plugin chain_plugin http_plugin appbase ) -target_include_directories( history_api_plugin PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" ) diff --git a/plugins/history_api_plugin/history_api_plugin.cpp b/plugins/history_api_plugin/history_api_plugin.cpp deleted file mode 100644 index c435371f42..0000000000 --- a/plugins/history_api_plugin/history_api_plugin.cpp +++ /dev/null @@ -1,49 +0,0 @@ -#include -#include - -#include - -namespace eosio { - -using namespace eosio; - -static appbase::abstract_plugin& _history_api_plugin = app().register_plugin(); - -history_api_plugin::history_api_plugin(){} -history_api_plugin::~history_api_plugin(){} - -void history_api_plugin::set_program_options(options_description&, options_description&) {} -void history_api_plugin::plugin_initialize(const variables_map&) {} - -#define CALL_WITH_400(api_name, api_handle, api_namespace, call_name, params_type) \ -{std::string("/v1/" #api_name "/" #call_name), \ - [api_handle](string, string body, url_response_callback cb) mutable { \ - try { \ - auto params = parse_params(body);\ - fc::variant result( api_handle.call_name( std::move(params) ) ); \ - cb(200, std::move(result)); \ - } catch (...) { \ - http_plugin::handle_exception(#api_name, #call_name, body, cb); \ - } \ - }} - -#define CHAIN_RO_CALL(call_name, params_type) CALL_WITH_400(history, ro_api, history_apis::read_only, call_name, params_type) -//#define CHAIN_RW_CALL(call_name) CALL(history, rw_api, history_apis::read_write, call_name) - -void history_api_plugin::plugin_startup() { - ilog( "starting history_api_plugin" ); - auto ro_api = app().get_plugin().get_read_only_api(); - //auto rw_api = app().get_plugin().get_read_write_api(); - - app().get_plugin().add_api({ -// CHAIN_RO_CALL(get_transaction), - CHAIN_RO_CALL(get_actions, http_params_types::params_required), - CHAIN_RO_CALL(get_transaction, http_params_types::params_required), - CHAIN_RO_CALL(get_key_accounts, http_params_types::params_required), - CHAIN_RO_CALL(get_controlled_accounts, http_params_types::params_required) - }); -} - -void history_api_plugin::plugin_shutdown() {} - -} diff --git a/plugins/history_api_plugin/include/eosio/history_api_plugin/history_api_plugin.hpp b/plugins/history_api_plugin/include/eosio/history_api_plugin/history_api_plugin.hpp deleted file mode 100644 index 557ac1b48b..0000000000 --- a/plugins/history_api_plugin/include/eosio/history_api_plugin/history_api_plugin.hpp +++ /dev/null @@ -1,28 +0,0 @@ -#pragma once -#include -#include -#include - -#include - -namespace eosio { - - using namespace appbase; - - class history_api_plugin : public plugin { - public: - APPBASE_PLUGIN_REQUIRES((history_plugin)(chain_plugin)(http_plugin)) - - history_api_plugin(); - virtual ~history_api_plugin(); - - virtual void set_program_options(options_description&, options_description&) override; - - void plugin_initialize(const variables_map&); - void plugin_startup(); - void plugin_shutdown(); - - private: - }; - -} diff --git a/plugins/history_plugin/CMakeLists.txt b/plugins/history_plugin/CMakeLists.txt deleted file mode 100644 index 1eca3d247e..0000000000 --- a/plugins/history_plugin/CMakeLists.txt +++ /dev/null @@ -1,7 +0,0 @@ -file(GLOB HEADERS "include/eosio/history_plugin/*.hpp") -add_library( history_plugin - history_plugin.cpp - ${HEADERS} ) - -target_link_libraries( history_plugin chain_plugin eosio_chain appbase ) -target_include_directories( history_plugin PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" ) diff --git a/plugins/history_plugin/history_plugin.cpp b/plugins/history_plugin/history_plugin.cpp deleted file mode 100644 index c655aac830..0000000000 --- a/plugins/history_plugin/history_plugin.cpp +++ /dev/null @@ -1,594 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include - -#include -#include - -namespace eosio { - using namespace chain; - using boost::signals2::scoped_connection; - - static appbase::abstract_plugin& _history_plugin = app().register_plugin(); - - - struct account_history_object : public chainbase::object { - OBJECT_CTOR( account_history_object ); - - id_type id; - account_name account; ///< the name of the account which has this action in its history - uint64_t action_sequence_num = 0; ///< the sequence number of the relevant action (global) - int32_t account_sequence_num = 0; ///< the sequence number for this account (per-account) - }; - - struct action_history_object : public chainbase::object { - - OBJECT_CTOR( action_history_object, (packed_action_trace) ); - - id_type id; - uint64_t action_sequence_num; ///< the sequence number of the relevant action - - shared_string packed_action_trace; - uint32_t block_num; - block_timestamp_type block_time; - transaction_id_type trx_id; - }; - using account_history_id_type = account_history_object::id_type; - using action_history_id_type = action_history_object::id_type; - - - struct by_action_sequence_num; - struct by_account_action_seq; - struct by_trx_id_act_seq; - - using action_history_index = chainbase::shared_multi_index_container< - action_history_object, - indexed_by< - ordered_unique, member>, - ordered_unique, member>, - ordered_unique, - composite_key< action_history_object, - member, - member - > - > - > - >; - - using account_history_index = chainbase::shared_multi_index_container< - account_history_object, - indexed_by< - ordered_unique, member>, - ordered_unique, - composite_key< account_history_object, - member, - member - > - > - > - >; - -} /// namespace eosio - -CHAINBASE_SET_INDEX_TYPE(eosio::account_history_object, eosio::account_history_index) -CHAINBASE_SET_INDEX_TYPE(eosio::action_history_object, eosio::action_history_index) - -namespace eosio { - - template - static void remove(chainbase::database& db, const account_name& account_name, const permission_name& permission) - { - const auto& idx = db.get_index(); - auto& mutable_idx = db.get_mutable_index(); - while(!idx.empty()) { - auto key = boost::make_tuple(account_name, permission); - const auto& itr = idx.lower_bound(key); - if (itr == idx.end()) - break; - - const auto& range_end = idx.upper_bound(key); - if (itr == range_end) - break; - - mutable_idx.remove(*itr); - } - } - - static void add(chainbase::database& db, const vector& keys, const account_name& name, const permission_name& permission) - { - for (auto pub_key_weight : keys ) { - db.create([&](public_key_history_object& obj) { - obj.public_key = pub_key_weight.key; - obj.name = name; - obj.permission = permission; - }); - } - } - - static void add(chainbase::database& db, const vector& controlling_accounts, const account_name& account_name, const permission_name& permission) - { - for (auto controlling_account : controlling_accounts ) { - db.create([&](account_control_history_object& obj) { - obj.controlled_account = account_name; - obj.controlled_permission = permission; - obj.controlling_account = controlling_account.permission.actor; - }); - } - } - - struct filter_entry { - name receiver; - name action; - name actor; - - std::tuple key() const { - return std::make_tuple(receiver, action, actor); - } - - friend bool operator<( const filter_entry& a, const filter_entry& b ) { - return a.key() < b.key(); - } - }; - - class history_plugin_impl { - public: - bool bypass_filter = false; - std::set filter_on; - std::set filter_out; - chain_plugin* chain_plug = nullptr; - std::optional applied_transaction_connection; - - bool filter(const action_trace& act) { - bool pass_on = false; - if (bypass_filter) { - pass_on = true; - } - if (filter_on.find({ act.receiver, {}, {} }) != filter_on.end()) { - pass_on = true; - } - if (filter_on.find({ act.receiver, act.act.name, {} }) != filter_on.end()) { - pass_on = true; - } - for (const auto& a : act.act.authorization) { - if (filter_on.find({ act.receiver, {}, a.actor }) != filter_on.end()) { - pass_on = true; - } - if (filter_on.find({ act.receiver, act.act.name, a.actor }) != filter_on.end()) { - pass_on = true; - } - } - - if (!pass_on) { return false; } - - if (filter_out.find({ act.receiver, {}, {} }) != filter_out.end()) { - return false; - } - if (filter_out.find({ act.receiver, act.act.name, {} }) != filter_out.end()) { - return false; - } - for (const auto& a : act.act.authorization) { - if (filter_out.find({ act.receiver, {}, a.actor }) != filter_out.end()) { - return false; - } - if (filter_out.find({ act.receiver, act.act.name, a.actor }) != filter_out.end()) { - return false; - } - } - - return true; - } - - set account_set( const action_trace& act ) { - set result; - - result.insert( act.receiver ); - for( const auto& a : act.act.authorization ) { - if( bypass_filter || - filter_on.find({ act.receiver, {}, {}}) != filter_on.end() || - filter_on.find({ act.receiver, {}, a.actor}) != filter_on.end() || - filter_on.find({ act.receiver, act.act.name, {}}) != filter_on.end() || - filter_on.find({ act.receiver, act.act.name, a.actor }) != filter_on.end() ) { - if ((filter_out.find({ act.receiver, {}, {} }) == filter_out.end()) && - (filter_out.find({ act.receiver, {}, a.actor }) == filter_out.end()) && - (filter_out.find({ act.receiver, act.act.name, {} }) == filter_out.end()) && - (filter_out.find({ act.receiver, act.act.name, a.actor }) == filter_out.end())) { - result.insert( a.actor ); - } - } - } - return result; - } - - void record_account_action( account_name n, const action_trace& act ) { - auto& chain = chain_plug->chain(); - chainbase::database& db = const_cast( chain.db() ); // Override read-only access to state DB (highly unrecommended practice!) - - const auto& idx = db.get_index(); - auto itr = idx.lower_bound( boost::make_tuple( name(n.to_uint64_t()+1), 0 ) ); - - uint64_t asn = 0; - if( itr != idx.begin() ) --itr; - if( itr->account == n ) - asn = itr->account_sequence_num + 1; - - const auto& a = db.create( [&]( auto& aho ) { - aho.account = n; - aho.action_sequence_num = act.receipt->global_sequence; - aho.account_sequence_num = asn; - }); - } - - void on_system_action( const action_trace& at ) { - auto& chain = chain_plug->chain(); - chainbase::database& db = const_cast( chain.db() ); // Override read-only access to state DB (highly unrecommended practice!) - if( at.act.name == "newaccount"_n ) - { - const auto create = at.act.data_as(); - add(db, create.owner.keys, create.name, "owner"_n); - add(db, create.owner.accounts, create.name, "owner"_n); - add(db, create.active.keys, create.name, "active"_n); - add(db, create.active.accounts, create.name, "active"_n); - } - else if( at.act.name == "updateauth"_n ) - { - const auto update = at.act.data_as(); - remove(db, update.account, update.permission); - remove(db, update.account, update.permission); - add(db, update.auth.keys, update.account, update.permission); - add(db, update.auth.accounts, update.account, update.permission); - } - else if( at.act.name == "deleteauth"_n ) - { - const auto del = at.act.data_as(); - remove(db, del.account, del.permission); - remove(db, del.account, del.permission); - } - } - - void on_action_trace( const action_trace& at ) { - if( filter( at ) ) { - //idump((fc::json::to_pretty_string(at))); - auto& chain = chain_plug->chain(); - chainbase::database& db = const_cast( chain.db() ); // Override read-only access to state DB (highly unrecommended practice!) - - db.create( [&]( auto& aho ) { - auto ps = fc::raw::pack_size( at ); - aho.packed_action_trace.resize_and_fill(ps, [&at](char* data, std::size_t size) { - fc::datastream ds( data, size ); - fc::raw::pack( ds, at ); - }); - aho.action_sequence_num = at.receipt->global_sequence; - aho.block_num = chain.head_block_num() + 1; - aho.block_time = chain.pending_block_time(); - aho.trx_id = at.trx_id; - }); - - auto aset = account_set( at ); - for( auto a : aset ) { - record_account_action( a, at ); - } - } - if( at.receiver == chain::config::system_account_name ) - on_system_action( at ); - } - - void on_applied_transaction( const transaction_trace_ptr& trace ) { - if( !trace->receipt || (trace->receipt->status != transaction_receipt_header::executed && - trace->receipt->status != transaction_receipt_header::soft_fail) ) - return; - for( const auto& atrace : trace->action_traces ) { - if( !atrace.receipt ) continue; - on_action_trace( atrace ); - } - } - }; - - history_plugin::history_plugin() - :my(std::make_shared()) { - } - - history_plugin::~history_plugin() { - } - - - - void history_plugin::set_program_options(options_description& cli, options_description& cfg) { - cfg.add_options() - ("filter-on,f", bpo::value>()->composing(), - "Track actions which match receiver:action:actor. Actor may be blank to include all. Action and Actor both blank allows all from Recieiver. Receiver may not be blank.") - ; - cfg.add_options() - ("filter-out,F", bpo::value>()->composing(), - "Do not track actions which match receiver:action:actor. Action and Actor both blank excludes all from Reciever. Actor blank excludes all from reciever:action. Receiver may not be blank.") - ; - } - - void history_plugin::plugin_initialize(const variables_map& options) { - try { - if( options.count( "filter-on" )) { - auto fo = options.at( "filter-on" ).as>(); - for( auto& s : fo ) { - if( s == "*" || s == "\"*\"" ) { - my->bypass_filter = true; - wlog( "--filter-on * enabled. This can fill shared_mem, causing nodeos to stop." ); - break; - } - std::vector v; - boost::split( v, s, boost::is_any_of( ":" )); - EOS_ASSERT( v.size() == 3, fc::invalid_arg_exception, "Invalid value ${s} for --filter-on", ("s", s)); - filter_entry fe{eosio::chain::name(v[0]), eosio::chain::name(v[1]), eosio::chain::name(v[2])}; - EOS_ASSERT( fe.receiver.to_uint64_t(), fc::invalid_arg_exception, - "Invalid value ${s} for --filter-on", ("s", s)); - my->filter_on.insert( fe ); - } - } - if( options.count( "filter-out" )) { - auto fo = options.at( "filter-out" ).as>(); - for( auto& s : fo ) { - std::vector v; - boost::split( v, s, boost::is_any_of( ":" )); - EOS_ASSERT( v.size() == 3, fc::invalid_arg_exception, "Invalid value ${s} for --filter-out", ("s", s)); - filter_entry fe{eosio::chain::name(v[0]), eosio::chain::name(v[1]), eosio::chain::name(v[2])}; - EOS_ASSERT( fe.receiver.to_uint64_t(), fc::invalid_arg_exception, - "Invalid value ${s} for --filter-out", ("s", s)); - my->filter_out.insert( fe ); - } - } - - my->chain_plug = app().find_plugin(); - EOS_ASSERT( my->chain_plug, chain::missing_chain_plugin_exception, "" ); - auto& chain = my->chain_plug->chain(); - - chainbase::database& db = const_cast( chain.db() ); // Override read-only access to state DB (highly unrecommended practice!) - // TODO: Use separate chainbase database for managing the state of the history_plugin (or remove deprecated history_plugin entirely) - db.add_index(); - db.add_index(); - db.add_index(); - db.add_index(); - - my->applied_transaction_connection.emplace( - chain.applied_transaction.connect( [&]( std::tuple t ) { - my->on_applied_transaction( std::get<0>(t) ); - } )); - } FC_LOG_AND_RETHROW() - } - - void history_plugin::plugin_startup() { - } - - void history_plugin::plugin_shutdown() { - my->applied_transaction_connection.reset(); - } - - - - - namespace history_apis { - read_only::get_actions_result read_only::get_actions( const read_only::get_actions_params& params )const { - edump((params)); - auto& chain = history->chain_plug->chain(); - const auto& db = chain.db(); - const auto abi_serializer_max_time = history->chain_plug->get_abi_serializer_max_time(); - - const auto& idx = db.get_index(); - - int32_t start = 0; - int32_t pos = params.pos ? *params.pos : -1; - int32_t end = 0; - int32_t offset = params.offset ? *params.offset : -20; - auto n = params.account_name; - idump((pos)); - if( pos == -1 ) { - auto itr = idx.lower_bound( boost::make_tuple( name(n.to_uint64_t()+1), 0 ) ); - if( itr == idx.begin() ) { - if( itr->account == n ) - pos = itr->account_sequence_num+1; - } else if( itr != idx.begin() ) --itr; - - if( itr->account == n ) - pos = itr->account_sequence_num + 1; - } - - if( pos== -1 ) pos = 0xfffffff; - - if( offset > 0 ) { - start = pos; - end = start + offset; - } else { - start = pos + offset; - if( start > pos ) start = 0; - end = pos; - } - EOS_ASSERT( end >= start, chain::plugin_exception, "end position is earlier than start position" ); - - idump((start)(end)); - - auto start_itr = idx.lower_bound( boost::make_tuple( n, start ) ); - auto end_itr = idx.upper_bound( boost::make_tuple( n, end) ); - - auto start_time = fc::time_point::now(); - auto end_time = start_time; - - get_actions_result result; - result.last_irreversible_block = chain.last_irreversible_block_num(); - while( start_itr != end_itr ) { - const auto& a = db.get( start_itr->action_sequence_num ); - fc::datastream ds( a.packed_action_trace.data(), a.packed_action_trace.size() ); - action_trace t; - fc::raw::unpack( ds, t ); - result.actions.emplace_back( ordered_action_result{ - start_itr->action_sequence_num, - start_itr->account_sequence_num, - a.block_num, a.block_time, - chain.to_variant_with_abi(t, abi_serializer::create_yield_function( abi_serializer_max_time )) - }); - - end_time = fc::time_point::now(); - if( end_time - start_time > fc::microseconds(100000) ) { - result.time_limit_exceeded_error = true; - break; - } - ++start_itr; - } - return result; - } - - - read_only::get_transaction_result read_only::get_transaction( const read_only::get_transaction_params& p )const { - auto& chain = history->chain_plug->chain(); - const auto abi_serializer_max_time = history->chain_plug->get_abi_serializer_max_time(); - - transaction_id_type input_id; - auto input_id_length = p.id.size(); - try { - FC_ASSERT( input_id_length <= 64, "hex string is too long to represent an actual transaction id" ); - FC_ASSERT( input_id_length >= 8, "hex string representing transaction id should be at least 8 characters long to avoid excessive collisions" ); - input_id = transaction_id_type(p.id); - } EOS_RETHROW_EXCEPTIONS(transaction_id_type_exception, "Invalid transaction ID: ${transaction_id}", ("transaction_id", p.id)) - - auto txn_id_matched = [&input_id, input_id_size = input_id_length/2, no_half_byte_at_end = (input_id_length % 2 == 0)] - ( const transaction_id_type &id ) -> bool // hex prefix comparison - { - bool whole_byte_prefix_matches = memcmp( input_id.data(), id.data(), input_id_size ) == 0; - if( !whole_byte_prefix_matches || no_half_byte_at_end ) - return whole_byte_prefix_matches; - - // check if half byte at end of specified part of input_id matches - return (*(input_id.data() + input_id_size) & 0xF0) == (*(id.data() + input_id_size) & 0xF0); - }; - - const auto& db = chain.db(); - const auto& idx = db.get_index(); - auto itr = idx.lower_bound( boost::make_tuple( input_id ) ); - - bool in_history = (itr != idx.end() && txn_id_matched(itr->trx_id) ); - - if( !in_history && !p.block_num_hint ) { - EOS_THROW(tx_not_found, "Transaction ${id} not found in history and no block hint was given", ("id",p.id)); - } - - get_transaction_result result; - - if( in_history ) { - result.id = itr->trx_id; - result.last_irreversible_block = chain.last_irreversible_block_num(); - result.block_num = itr->block_num; - result.block_time = itr->block_time; - - while( itr != idx.end() && itr->trx_id == result.id ) { - - fc::datastream ds( itr->packed_action_trace.data(), itr->packed_action_trace.size() ); - action_trace t; - fc::raw::unpack( ds, t ); - result.traces.emplace_back( chain.to_variant_with_abi(t, abi_serializer::create_yield_function( abi_serializer_max_time )) ); - - ++itr; - } - - auto blk = chain.fetch_block_by_number( result.block_num ); - if( blk || chain.is_building_block() ) { - const auto& receipts = blk ? blk->transactions : chain.get_pending_trx_receipts(); - for (const auto &receipt: receipts) { - if (std::holds_alternative(receipt.trx)) { - auto &pt = std::get(receipt.trx); - if (pt.id() == result.id) { - fc::mutable_variant_object r("receipt", receipt); - fc::variant v = chain.to_variant_with_abi(pt.get_transaction(), abi_serializer::create_yield_function( abi_serializer_max_time )); - fc::mutable_variant_object tmp(v.get_object()); - const auto* sigs = pt.get_signatures(); - tmp("signatures", sigs != nullptr ? *sigs : vector()); - const auto* context_free_data = pt.get_context_free_data(); - tmp("context_free_data", context_free_data != nullptr ? *context_free_data : vector()); - r("trx", std::move(tmp) ); - result.trx = move(r); - break; - } - } else { - auto &id = std::get(receipt.trx); - if (id == result.id) { - fc::mutable_variant_object r("receipt", receipt); - result.trx = move(r); - break; - } - } - } - } - } else { - auto blk = chain.fetch_block_by_number(*p.block_num_hint); - bool found = false; - if (blk) { - for (const auto& receipt: blk->transactions) { - if (std::holds_alternative(receipt.trx)) { - auto& pt = std::get(receipt.trx); - const auto& id = pt.id(); - if( txn_id_matched(id) ) { - result.id = id; - result.last_irreversible_block = chain.last_irreversible_block_num(); - result.block_num = *p.block_num_hint; - result.block_time = blk->timestamp; - fc::mutable_variant_object r("receipt", receipt); - fc::variant v = chain.to_variant_with_abi(pt.get_transaction(), abi_serializer::create_yield_function( abi_serializer_max_time )); - fc::mutable_variant_object tmp(v.get_object()); - const auto* sigs = pt.get_signatures(); - tmp("signatures", sigs != nullptr ? *sigs : vector()); - const auto* context_free_data = pt.get_context_free_data(); - tmp("context_free_data", context_free_data != nullptr ? *context_free_data : vector()); - r("trx", std::move(tmp) ); - result.trx = move(r); - found = true; - break; - } - } else { - auto& id = std::get(receipt.trx); - if( txn_id_matched(id) ) { - result.id = id; - result.last_irreversible_block = chain.last_irreversible_block_num(); - result.block_num = *p.block_num_hint; - result.block_time = blk->timestamp; - fc::mutable_variant_object r("receipt", receipt); - result.trx = move(r); - found = true; - break; - } - } - } - } - - if (!found) { - EOS_THROW(tx_not_found, "Transaction ${id} not found in history or in block number ${n}", ("id",p.id)("n", *p.block_num_hint)); - } - } - - return result; - } - - read_only::get_key_accounts_results read_only::get_key_accounts(const get_key_accounts_params& params) const { - std::set accounts; - const auto& db = history->chain_plug->chain().db(); - const auto& pub_key_idx = db.get_index(); - auto range = pub_key_idx.equal_range( params.public_key ); - for (auto obj = range.first; obj != range.second; ++obj) - accounts.insert(obj->name); - return {vector(accounts.begin(), accounts.end())}; - } - - read_only::get_controlled_accounts_results read_only::get_controlled_accounts(const get_controlled_accounts_params& params) const { - std::set accounts; - const auto& db = history->chain_plug->chain().db(); - const auto& account_control_idx = db.get_index(); - auto range = account_control_idx.equal_range( params.controlling_account ); - for (auto obj = range.first; obj != range.second; ++obj) - accounts.insert(obj->controlled_account); - return {vector(accounts.begin(), accounts.end())}; - } - - } /// history_apis - - - -} /// namespace eosio diff --git a/plugins/history_plugin/include/eosio/history_plugin/account_control_history_object.hpp b/plugins/history_plugin/include/eosio/history_plugin/account_control_history_object.hpp deleted file mode 100644 index 2952987644..0000000000 --- a/plugins/history_plugin/include/eosio/history_plugin/account_control_history_object.hpp +++ /dev/null @@ -1,52 +0,0 @@ -#pragma once - -#include -#include - -namespace eosio { -using chain::account_name; -using chain::permission_name; -using chain::shared_vector; -using chain::transaction_id_type; -using namespace boost::multi_index; - -class account_control_history_object : public chainbase::object { - OBJECT_CTOR(account_control_history_object) - - id_type id; - account_name controlled_account; - permission_name controlled_permission; - account_name controlling_account; -}; - -struct by_id; -struct by_controlling; -struct by_controlled_authority; -using account_control_history_multi_index = chainbase::shared_multi_index_container< - account_control_history_object, - indexed_by< - ordered_unique, BOOST_MULTI_INDEX_MEMBER(account_control_history_object, account_control_history_object::id_type, id)>, - ordered_unique, - composite_key< account_control_history_object, - member, - member - > - >, - ordered_unique, - composite_key< account_control_history_object, - member, - member, - member - > - > - > ->; - -typedef chainbase::generic_index account_control_history_index; - -} - -CHAINBASE_SET_INDEX_TYPE( eosio::account_control_history_object, eosio::account_control_history_multi_index ) - -FC_REFLECT( eosio::account_control_history_object, (controlled_account)(controlled_permission)(controlling_account) ) - diff --git a/plugins/history_plugin/include/eosio/history_plugin/history_plugin.hpp b/plugins/history_plugin/include/eosio/history_plugin/history_plugin.hpp deleted file mode 100644 index 365d902f42..0000000000 --- a/plugins/history_plugin/include/eosio/history_plugin/history_plugin.hpp +++ /dev/null @@ -1,151 +0,0 @@ -#pragma once -#include - -#include - -namespace fc { class variant; } - -namespace eosio { - using chain::transaction_id_type; - using std::shared_ptr; - using namespace appbase; - using chain::name; - using chain::uint128_t; - - typedef shared_ptr history_ptr; - typedef shared_ptr history_const_ptr; - -namespace history_apis { - -class read_only { - history_const_ptr history; - - public: - read_only(history_const_ptr&& history) - : history(history) {} - - - struct get_actions_params { - chain::account_name account_name; - std::optional pos; /// a absolute sequence positon -1 is the end/last action - std::optional offset; ///< the number of actions relative to pos, negative numbers return [pos-offset,pos), positive numbers return [pos,pos+offset) - }; - - struct ordered_action_result { - uint64_t global_action_seq = 0; - int32_t account_action_seq = 0; - uint32_t block_num; - chain::block_timestamp_type block_time; - fc::variant action_trace; - }; - - struct get_actions_result { - vector actions; - uint32_t last_irreversible_block; - std::optional time_limit_exceeded_error; - }; - - - get_actions_result get_actions( const get_actions_params& )const; - - - struct get_transaction_params { - string id; - std::optional block_num_hint; - }; - - struct get_transaction_result { - transaction_id_type id; - fc::variant trx; - chain::block_timestamp_type block_time; - uint32_t block_num = 0; - uint32_t last_irreversible_block = 0; - vector traces; - }; - - get_transaction_result get_transaction( const get_transaction_params& )const; - - - - - /* - struct ordered_transaction_results { - uint32_t seq_num; - chain::transaction_id_type transaction_id; - fc::variant transaction; - }; - - get_transactions_results get_transactions(const get_transactions_params& params) const; - */ - - - struct get_key_accounts_params { - chain::public_key_type public_key; - }; - struct get_key_accounts_results { - vector account_names; - }; - get_key_accounts_results get_key_accounts(const get_key_accounts_params& params) const; - - - struct get_controlled_accounts_params { - chain::account_name controlling_account; - }; - struct get_controlled_accounts_results { - vector controlled_accounts; - }; - get_controlled_accounts_results get_controlled_accounts(const get_controlled_accounts_params& params) const; -}; - - -} // namespace history_apis - - -/** - * This plugin tracks all actions and keys associated with a set of configured accounts. It enables - * wallets to paginate queries for history. - * - * An action will be included in the account's history if any of the following: - * - receiver - * - any account named in auth list - * - * A key will be linked to an account if the key is referneced in authorities of updateauth or newaccount - */ -class history_plugin : public plugin { - public: - APPBASE_PLUGIN_REQUIRES((chain_plugin)) - - history_plugin(); - virtual ~history_plugin(); - - virtual void set_program_options(options_description& cli, options_description& cfg) override; - - void plugin_initialize(const variables_map& options); - void plugin_startup(); - void plugin_shutdown(); - - history_apis::read_only get_read_only_api()const { return history_apis::read_only(history_const_ptr(my)); } - - private: - history_ptr my; -}; - -} /// namespace eosio - -FC_REFLECT( eosio::history_apis::read_only::get_actions_params, (account_name)(pos)(offset) ) -FC_REFLECT( eosio::history_apis::read_only::get_actions_result, (actions)(last_irreversible_block)(time_limit_exceeded_error) ) -FC_REFLECT( eosio::history_apis::read_only::ordered_action_result, (global_action_seq)(account_action_seq)(block_num)(block_time)(action_trace) ) - -FC_REFLECT( eosio::history_apis::read_only::get_transaction_params, (id)(block_num_hint) ) -FC_REFLECT( eosio::history_apis::read_only::get_transaction_result, (id)(trx)(block_time)(block_num)(last_irreversible_block)(traces) ) -/* -FC_REFLECT(eosio::history_apis::read_only::get_transaction_params, (transaction_id) ) -FC_REFLECT(eosio::history_apis::read_only::get_transaction_results, (transaction_id)(transaction) ) -FC_REFLECT(eosio::history_apis::read_only::get_transactions_params, (account_name)(skip_seq)(num_seq) ) -FC_REFLECT(eosio::history_apis::read_only::ordered_transaction_results, (seq_num)(transaction_id)(transaction) ) -FC_REFLECT(eosio::history_apis::read_only::get_transactions_results, (transactions)(time_limit_exceeded_error) ) -*/ -FC_REFLECT(eosio::history_apis::read_only::get_key_accounts_params, (public_key) ) -FC_REFLECT(eosio::history_apis::read_only::get_key_accounts_results, (account_names) ) -FC_REFLECT(eosio::history_apis::read_only::get_controlled_accounts_params, (controlling_account) ) -FC_REFLECT(eosio::history_apis::read_only::get_controlled_accounts_results, (controlled_accounts) ) diff --git a/plugins/history_plugin/include/eosio/history_plugin/public_key_history_object.hpp b/plugins/history_plugin/include/eosio/history_plugin/public_key_history_object.hpp deleted file mode 100644 index 79b10e36d9..0000000000 --- a/plugins/history_plugin/include/eosio/history_plugin/public_key_history_object.hpp +++ /dev/null @@ -1,51 +0,0 @@ -#pragma once - -#include -#include - -namespace eosio { -using chain::account_name; -using chain::public_key_type; -using chain::permission_name; -using namespace boost::multi_index; - -class public_key_history_object : public chainbase::object { - OBJECT_CTOR(public_key_history_object) - - id_type id; - public_key_type public_key; - account_name name; - permission_name permission; -}; - -struct by_id; -struct by_pub_key; -struct by_account_permission_name; -using public_key_history_multi_index = chainbase::shared_multi_index_container< - public_key_history_object, - indexed_by< - ordered_unique, BOOST_MULTI_INDEX_MEMBER(public_key_history_object, public_key_history_object::id_type, id)>, - ordered_unique, - composite_key< public_key_history_object, - member, - member - > - >, - ordered_unique, - composite_key< public_key_history_object, - member, - member, - member - > - > - > ->; - -typedef chainbase::generic_index public_key_history_index; - -} - -CHAINBASE_SET_INDEX_TYPE( eosio::public_key_history_object, eosio::public_key_history_multi_index ) - -FC_REFLECT( eosio::public_key_history_object, (public_key)(name)(permission) ) - From eb2649083ef162c015de2113c910d98a616d5957 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Wed, 13 Oct 2021 12:28:59 -0500 Subject: [PATCH 06/62] Remove history_api_plugin use --- tests/p2p_tests/dawn_515/test.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/p2p_tests/dawn_515/test.sh b/tests/p2p_tests/dawn_515/test.sh index 081fe3477f..5c5640a283 100755 --- a/tests/p2p_tests/dawn_515/test.sh +++ b/tests/p2p_tests/dawn_515/test.sh @@ -46,7 +46,6 @@ plugin = eosio::chain_api_plugin plugin = eosio::net_plugin plugin = eosio::trace_api_plugin trace-no-abis = true -plugin = eosio::history_api_plugin http-server-address = 127.0.0.1:8888 blocks-dir = blocks p2p-listen-endpoint = 0.0.0.0:9876 From 2e2b76f5cbe51a77b091a02471fe791482e8e87b Mon Sep 17 00:00:00 2001 From: Jingjun Zhao Date: Tue, 11 Jan 2022 19:30:52 -0500 Subject: [PATCH 07/62] Fix a long run test --- tests/nodeos_read_terminate_at_block_test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/nodeos_read_terminate_at_block_test.py b/tests/nodeos_read_terminate_at_block_test.py index b049a6c1ab..ce6dc092fe 100755 --- a/tests/nodeos_read_terminate_at_block_test.py +++ b/tests/nodeos_read_terminate_at_block_test.py @@ -156,6 +156,7 @@ def checkHeadOrSpeculative(head, lib): 2 : "--read-mode speculative --terminate-at-block=200", 3 : "--read-mode head --terminate-at-block=250", } + traceNodeosArgs = " --plugin eosio::trace_api_plugin --trace-no-abis " # Kill any existing instances and launch cluster TestHelper.printSystemInfo("BEGIN") @@ -169,7 +170,8 @@ def checkHeadOrSpeculative(head, lib): pnodes=1, useBiosBootFile=False, topo="mesh", - specificExtraNodeosArgs=specificNodeosArgs + specificExtraNodeosArgs=specificNodeosArgs, + extraNodeosArgs=traceNodeosArgs ) producingNodeId = 0 From fbae4af3adcd4c3319892ddd088602532b62dd88 Mon Sep 17 00:00:00 2001 From: Jingjun Zhao Date: Wed, 12 Jan 2022 09:54:02 -0500 Subject: [PATCH 08/62] Increase relaunch timeout in nodeos_irreversible_mode_test.py --- tests/nodeos_irreversible_mode_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/nodeos_irreversible_mode_test.py b/tests/nodeos_irreversible_mode_test.py index a0a4330696..cd71720e6e 100755 --- a/tests/nodeos_irreversible_mode_test.py +++ b/tests/nodeos_irreversible_mode_test.py @@ -25,7 +25,7 @@ Print = Utils.Print errorExit = Utils.errorExit cmdError = Utils.cmdError -relaunchTimeout = 10 +relaunchTimeout = 30 numOfProducers = 4 totalNodes = 10 From 6e4ed5f4d707ed237ed0d54d73cac46f4e94c5e2 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Tue, 14 Dec 2021 11:54:55 -0600 Subject: [PATCH 09/62] move -Werror to CI only --- .cicd/build.sh | 2 +- CMakeLists.txt | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.cicd/build.sh b/.cicd/build.sh index 2d61254cd8..caea86d22b 100755 --- a/.cicd/build.sh +++ b/.cicd/build.sh @@ -4,7 +4,7 @@ set -eo pipefail . ./.cicd/helpers/general.sh mkdir -p "$BUILD_DIR" [[ -z "$DCMAKE_BUILD_TYPE" ]] && export DCMAKE_BUILD_TYPE='Release' -CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_BUILD_TYPE=\"$DCMAKE_BUILD_TYPE\" -DENABLE_MULTIVERSION_PROTOCOL_TEST=\"true\" -DAMQP_CONN_STR=\"amqp://guest:guest@localhost:5672\"" +CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_CXX_FLAGS=\"-Werror\" -DCMAKE_BUILD_TYPE=\"$DCMAKE_BUILD_TYPE\" -DENABLE_MULTIVERSION_PROTOCOL_TEST=\"true\" -DAMQP_CONN_STR=\"amqp://guest:guest@localhost:5672\"" if [[ "$(uname)" == 'Darwin' && "$FORCE_LINUX" != 'true' ]]; then # You can't use chained commands in execute if [[ "$GITHUB_ACTIONS" == 'true' ]]; then diff --git a/CMakeLists.txt b/CMakeLists.txt index 30f72fa558..72cc4e3c00 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,8 +37,6 @@ set( RODEOS_EXECUTABLE_NAME rodeos ) set( TESTER_EXECUTABLE_NAME eosio-tester ) set( CLI_CLIENT_TPM_EXECUTABLE_NAME cleos_tpm ) -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror") - # http://stackoverflow.com/a/18369825 if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8.0) From bb764d921cf1c0f5946f219efd425a6a0eaaaa41 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Tue, 14 Dec 2021 14:42:47 -0600 Subject: [PATCH 10/62] use CMAKE_POSITION_INDEPENDENT_CODE --- .cicd/helpers/clang.make | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.cicd/helpers/clang.make b/.cicd/helpers/clang.make index fbc2380420..6f40993740 100755 --- a/.cicd/helpers/clang.make +++ b/.cicd/helpers/clang.make @@ -3,9 +3,10 @@ set(CMAKE_CXX_COMPILER_WORKS 1) set(CMAKE_C_COMPILER /usr/local/bin/clang) set(CMAKE_CXX_COMPILER /usr/local/bin/clang++) set(CMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES /usr/local/include/c++/v1 /usr/local/include /usr/include) -set(CMAKE_C_FLAGS_INIT "-D_FORTIFY_SOURCE=2 -fstack-protector-strong -fpie") -set(CMAKE_CXX_FLAGS_INIT "-nostdinc++ -D_FORTIFY_SOURCE=2 -fstack-protector-strong -fpie") -set(CMAKE_EXE_LINKER_FLAGS_INIT "-stdlib=libc++ -nostdlib++ -pie") +set(CMAKE_POSITION_INDEPENDENT_CODE ON) +set(CMAKE_C_FLAGS_INIT "-D_FORTIFY_SOURCE=2 -fstack-protector-strong") +set(CMAKE_CXX_FLAGS_INIT "-nostdinc++ -D_FORTIFY_SOURCE=2 -fstack-protector-strong") +set(CMAKE_EXE_LINKER_FLAGS_INIT "-stdlib=libc++ -nostdlib++") if(NOT APPLE) string(APPEND CMAKE_EXE_LINKER_FLAGS_INIT " -Wl,-z,relro,-z,now") endif() From 84c4cb71f88f3442046b4092eeaec76701056b74 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Wed, 15 Dec 2021 08:58:57 -0600 Subject: [PATCH 11/62] update abieos --- libraries/abieos | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/abieos b/libraries/abieos index 4865d4a5b9..708ead4737 160000 --- a/libraries/abieos +++ b/libraries/abieos @@ -1 +1 @@ -Subproject commit 4865d4a5b9174df9bf4f55408921ce5460f114b8 +Subproject commit 708ead4737f5cb89883b25b1bf843d04ae4d701d From 04d7cacddad7585bc12bfc0305f53dbc88edba46 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Wed, 15 Dec 2021 10:04:59 -0600 Subject: [PATCH 12/62] add CMAKE_C_FLAGS="-Werror" to ci build --- .cicd/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cicd/build.sh b/.cicd/build.sh index caea86d22b..ef90eea1f2 100755 --- a/.cicd/build.sh +++ b/.cicd/build.sh @@ -4,7 +4,7 @@ set -eo pipefail . ./.cicd/helpers/general.sh mkdir -p "$BUILD_DIR" [[ -z "$DCMAKE_BUILD_TYPE" ]] && export DCMAKE_BUILD_TYPE='Release' -CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_CXX_FLAGS=\"-Werror\" -DCMAKE_BUILD_TYPE=\"$DCMAKE_BUILD_TYPE\" -DENABLE_MULTIVERSION_PROTOCOL_TEST=\"true\" -DAMQP_CONN_STR=\"amqp://guest:guest@localhost:5672\"" +CMAKE_EXTRAS="$CMAKE_EXTRAS -DCMAKE_C_FLAGS=\"-Werror\" -DCMAKE_CXX_FLAGS=\"-Werror\" -DCMAKE_BUILD_TYPE=\"$DCMAKE_BUILD_TYPE\" -DENABLE_MULTIVERSION_PROTOCOL_TEST=\"true\" -DAMQP_CONN_STR=\"amqp://guest:guest@localhost:5672\"" if [[ "$(uname)" == 'Darwin' && "$FORCE_LINUX" != 'true' ]]; then # You can't use chained commands in execute if [[ "$GITHUB_ACTIONS" == 'true' ]]; then From c4da4e1d6acf2decd250098edb6fb3062dc1c99e Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Wed, 15 Dec 2021 17:04:15 -0600 Subject: [PATCH 13/62] update eos-vm --- libraries/eos-vm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/eos-vm b/libraries/eos-vm index 33087bb6cd..48b4070406 160000 --- a/libraries/eos-vm +++ b/libraries/eos-vm @@ -1 +1 @@ -Subproject commit 33087bb6cd1fb00250da2247c572f4427654305d +Subproject commit 48b40704060791d5dff5f000205b83d26c13f7d3 From 332761641fcfebf2fb43f690d8ebb9d678ace472 Mon Sep 17 00:00:00 2001 From: Huang-Ming Huang Date: Mon, 10 Jan 2022 10:11:26 -0600 Subject: [PATCH 14/62] update abieos --- libraries/abieos | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/abieos b/libraries/abieos index 708ead4737..ea37175ddb 160000 --- a/libraries/abieos +++ b/libraries/abieos @@ -1 +1 @@ -Subproject commit 708ead4737f5cb89883b25b1bf843d04ae4d701d +Subproject commit ea37175ddb02b3fb9532884f6f6e80d0787ec4f9 From 414413d5c4c300924265cae8ad5066d5b3b345ed Mon Sep 17 00:00:00 2001 From: Keke Li Date: Thu, 13 Jan 2022 10:58:16 -0800 Subject: [PATCH 15/62] Remove pinned Ubuntu 16.04 from develop-boxed --- .cicd/base-images.yml | 13 +- .cicd/build-scripts.yml | 12 -- .cicd/generate-pipeline.sh | 14 --- .cicd/metrics/test-metrics.js | 6 +- .../pinned/ubuntu-16.04-pinned.dockerfile | 116 ------------------ scripts/eosio_build_ubuntu.sh | 4 +- scripts/generate_deb.sh | 2 - scripts/helpers/eosio.sh | 2 +- 8 files changed, 7 insertions(+), 162 deletions(-) delete mode 100644 .cicd/platforms/pinned/ubuntu-16.04-pinned.dockerfile diff --git a/.cicd/base-images.yml b/.cicd/base-images.yml index 717193f4fe..dbff4f6eba 100644 --- a/.cicd/base-images.yml +++ b/.cicd/base-images.yml @@ -69,17 +69,6 @@ steps: agents: "queue=mac-anka-node-fleet" timeout: 180 - - label: ":ubuntu: Ubuntu 16.04 - Base Image Pinned" - command: - - "./.cicd/generate-base-images.sh" - env: - FORCE_BASE_IMAGE: true - IMAGE_TAG: ubuntu-16.04-pinned - PLATFORM_TYPE: pinned - agents: - queue: "automation-eks-eos-builder-fleet" - timeout: 180 - - label: ":ubuntu: Ubuntu 18.04 - Base Image Pinned" command: - "./.cicd/generate-base-images.sh" @@ -168,4 +157,4 @@ steps: PLATFORM_TYPE: unpinned agents: queue: "automation-eks-eos-builder-fleet" - timeout: 180 \ No newline at end of file + timeout: 180 diff --git a/.cicd/build-scripts.yml b/.cicd/build-scripts.yml index e66915ec22..ae55a27f86 100644 --- a/.cicd/build-scripts.yml +++ b/.cicd/build-scripts.yml @@ -52,18 +52,6 @@ steps: cd: ~ timeout: 180 - - label: ":ubuntu: Ubuntu 16.04 - Build Pinned" - plugins: - - docker#v3.3.0: - image: "ubuntu:16.04" - always-pull: true - agents: - queue: "automation-eks-eos-builder-fleet" - command: - - "apt update && apt upgrade -y && apt install -y git" - - "./scripts/eosio_build.sh -P -y" - timeout: 180 - - label: ":ubuntu: Ubuntu 18.04 - Build Pinned" plugins: - docker#v3.3.0: diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 4629840c9a..260997d732 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -582,20 +582,6 @@ cat < tcti-swtpm.c.new && \ - mv tcti-swtpm.c.new src/tss2-tcti/tcti-swtpm.c && \ - head -n -14 src/tss2-tcti/tcti-device.c > tcti-device.c.new && \ - mv tcti-device.c.new src/tss2-tcti/tcti-device.c && \ - head -n -14 src/tss2-tcti/tcti-mssim.c > tcti-mssim.c.new && \ - mv tcti-mssim.c.new src/tss2-tcti/tcti-mssim.c && \ - ./configure --disable-tcti-cmd --disable-fapi --disable-shared --enable-nodl && \ - make -j$(nproc) install && \ - cd .. && \ - rm -rf tpm2-tss-3.0.1 -# build dynamic tpm2-tss, do this one last so that the installed pkg-config files reference it -RUN tar xf tpm2-tss-3.0.1.tar.gz && \ - cd tpm2-tss-3.0.1 && \ - ./configure --disable-static --disable-fapi && \ - make -j$(nproc) install && \ - cd .. && \ - rm -rf tpm2-tss-3.0.1* - -# build TPM components used in unitests; tpm2-tools first -RUN curl -L https://github.com/tpm2-software/tpm2-tools/releases/download/4.3.0/tpm2-tools-4.3.0.tar.gz | tar zx && \ - cd tpm2-tools-4.3.0 && \ - ./configure && \ - make -j$(nproc) install && \ - cd .. && \ - rm -rf tpm2-tools-4.3.0 -# build libtpms -RUN git clone -b v0.7.3 https://github.com/stefanberger/libtpms && \ - cd libtpms && \ - autoreconf --install && \ - ./configure --with-tpm2 --with-openssl && \ - make -j$(nproc) install && \ - cd .. && \ - rm -rf libtpms -# build swtpm -RUN git clone -b v0.5.0 https://github.com/stefanberger/swtpm && \ - cd swtpm && \ - pip3 install cryptography==3.2 && \ - autoreconf --install && \ - ./configure && \ - make -j$(nproc) install && \ - cd .. && \ - rm -rf swtpm -RUN ldconfig - -#install nvm -RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.0/install.sh | bash -# load nvm in non-interactive shells -RUN cp ~/.bashrc ~/.bashrc.bak && \ - cat ~/.bashrc.bak | tail -3 > ~/.bashrc && \ - cat ~/.bashrc.bak | head -n '-3' >> ~/.bashrc && \ - rm ~/.bashrc.bak -# install node 10 -RUN bash -c '. ~/.bashrc; nvm install --lts=dubnium' && \ - ln -s "/root/.nvm/versions/node/$(ls -p /root/.nvm/versions/node | sort -Vr | head -1)bin/node" /usr/local/bin/node -RUN curl -sL https://deb.nodesource.com/setup_13.x | sudo -E bash - -RUN apt-get update && apt-get install -y nodejs && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* diff --git a/scripts/eosio_build_ubuntu.sh b/scripts/eosio_build_ubuntu.sh index 75fcdd513e..66a7a1c204 100755 --- a/scripts/eosio_build_ubuntu.sh +++ b/scripts/eosio_build_ubuntu.sh @@ -5,7 +5,7 @@ echo "Physical Memory: ${MEM_GIG}G" echo "Disk space total: ${DISK_TOTAL}G" echo "Disk space available: ${DISK_AVAIL}G" -( [[ $NAME == "Ubuntu" ]] && ( [[ "$(echo ${VERSION_ID})" == "16.04" ]] || [[ "$(echo ${VERSION_ID})" == "18.04" ]] || [[ "$(echo ${VERSION_ID})" == "20.04" ]]) ) || ( echo " - You must be running 16.04.x or 18.04.x or 20.04.x to install EOSIO." && exit 1 ) +( [[ $NAME == "Ubuntu" ]] && ( [[ "$(echo ${VERSION_ID})" == "18.04" ]] || [[ "$(echo ${VERSION_ID})" == "20.04" ]]) ) || ( echo " - You must be running 18.04.x or 20.04.x to install EOSIO." && exit 1 ) [[ $MEM_GIG -lt 7 ]] && echo "Your system must have 7 or more Gigabytes of physical memory installed." && exit 1 [[ "${DISK_AVAIL}" -lt "${DISK_MIN}" ]] && echo " - You must have at least ${DISK_MIN}GB of available storage to install EOSIO." && exit 1 @@ -13,7 +13,7 @@ echo "Disk space available: ${DISK_AVAIL}G" # system clang and build essential for Ubuntu 18 (16 too old) ( [[ $PIN_COMPILER == false ]] && ( [[ $VERSION_ID == "18.04" ]] || [[ $VERSION_ID == "20.04" ]] ) ) && EXTRA_DEPS=(clang,dpkg\ -s llvm-7-dev,dpkg\ -s) # We install clang8 for Ubuntu 16, but we still need something to compile cmake, boost, etc + pinned 18 still needs something to build source -( [[ $VERSION_ID == "16.04" ]] || ( $PIN_COMPILER && ( [[ $VERSION_ID == "18.04" ]] || [[ $VERSION_ID == "20.04" ]]) ) ) && ensure-build-essential +( $PIN_COMPILER && ( [[ $VERSION_ID == "18.04" ]] || [[ $VERSION_ID == "20.04" ]]) ) && ensure-build-essential $ENABLE_COVERAGE_TESTING && EXTRA_DEPS+=(lcov,dpkg\ -s) ensure-apt-packages "${REPO_ROOT}/scripts/eosio_build_ubuntu_deps" $(echo ${EXTRA_DEPS[@]}) echo "" diff --git a/scripts/generate_deb.sh b/scripts/generate_deb.sh index c9c63ef916..03d9f6e931 100755 --- a/scripts/generate_deb.sh +++ b/scripts/generate_deb.sh @@ -26,8 +26,6 @@ fi DISTRIB_MAJOR_VERSION="$(echo "$DISTRIB_RELEASE" | cut -d '.' -f '1')" if (( "$DISTRIB_MAJOR_VERSION" >= 18 )); then RELEASE_SPECIFIC_DEPS='libssl1.1' -elif (( "$DISTRIB_MAJOR_VERSION" >= 16 )); then - RELEASE_SPECIFIC_DEPS='libssl1.0.0' else echo "Found Ubuntu $DISTRIB_MAJOR_VERSION, but not sure which version of libssl to use. Exiting..." exit 2 diff --git a/scripts/helpers/eosio.sh b/scripts/helpers/eosio.sh index 9d27052f24..7fef803008 100755 --- a/scripts/helpers/eosio.sh +++ b/scripts/helpers/eosio.sh @@ -130,7 +130,7 @@ function print_supported_linux_distros_and_exit() { function ensure-compiler() { # Support build-essentials on ubuntu - if [[ $NAME == "CentOS Linux" ]] || [[ $VERSION_ID == "16.04" ]] || ( $PIN_COMPILER && ( [[ $VERSION_ID == "18.04" ]] || [[ $VERSION_ID == "20.04" ]] ) ); then + if [[ $NAME == "CentOS Linux" ]] || ( $PIN_COMPILER && ( [[ $VERSION_ID == "18.04" ]] || [[ $VERSION_ID == "20.04" ]] ) ); then export CXX=${CXX:-'g++'} export CC=${CC:-'gcc'} fi From a9d9a3a2444748ed917ee0e16529b7c42090e600 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Wed, 12 Jan 2022 13:19:16 -0500 Subject: [PATCH 16/62] Whitespace fixes --- .cicd/base-images.yml | 12 +++++----- .cicd/build-scripts.yml | 9 ++++--- .cicd/generate-pipeline.sh | 49 +++++++++++++++++++------------------- 3 files changed, 34 insertions(+), 36 deletions(-) diff --git a/.cicd/base-images.yml b/.cicd/base-images.yml index dbff4f6eba..e4d4dfe686 100644 --- a/.cicd/base-images.yml +++ b/.cicd/base-images.yml @@ -11,7 +11,7 @@ steps: agents: queue: "automation-eks-eos-builder-fleet" timeout: 180 - + - label: ":centos: CentOS 7.7 - Base Image Pinned" command: - "./.cicd/generate-base-images.sh" @@ -22,7 +22,7 @@ steps: agents: queue: "automation-eks-eos-builder-fleet" timeout: 180 - + - label: ":darwin: macOS 10.14 - Base Image Pinned" command: - "git clone git@github.com:EOSIO/eos.git eos && cd eos && git checkout -f $BUILDKITE_BRANCH" @@ -68,7 +68,7 @@ steps: cd: ~ agents: "queue=mac-anka-node-fleet" timeout: 180 - + - label: ":ubuntu: Ubuntu 18.04 - Base Image Pinned" command: - "./.cicd/generate-base-images.sh" @@ -90,7 +90,7 @@ steps: agents: queue: "automation-eks-eos-builder-fleet" timeout: 180 - + - label: ":centos: CentOS 7.7 - Base Image Unpinned" command: - "./.cicd/generate-base-images.sh" @@ -101,7 +101,7 @@ steps: agents: queue: "automation-eks-eos-builder-fleet" timeout: 180 - + - label: ":darwin: macOS 10.14 - Base Image Unpinned" command: - "git clone git@github.com:EOSIO/eos.git eos && cd eos && git checkout -f $BUILDKITE_BRANCH" @@ -147,7 +147,7 @@ steps: cd: ~ agents: "queue=mac-anka-node-fleet" timeout: 180 - + - label: ":ubuntu: Ubuntu 18.04 - Base Image Unpinned" command: - "./.cicd/generate-base-images.sh" diff --git a/.cicd/build-scripts.yml b/.cicd/build-scripts.yml index ae55a27f86..6c5e57cc9d 100644 --- a/.cicd/build-scripts.yml +++ b/.cicd/build-scripts.yml @@ -10,7 +10,7 @@ steps: command: - "./scripts/eosio_build.sh -P -y" timeout: 180 - + - label: ":centos: CentOS 7.7 - Build Pinned" plugins: - docker#v3.3.0: @@ -22,7 +22,6 @@ steps: - "./scripts/eosio_build.sh -P -y" timeout: 180 - - label: ":darwin: macOS 10.15 - Build Pinned" env: REPO: "git@github.com:EOSIO/eos.git" @@ -51,7 +50,7 @@ steps: - EOSIO/skip-checkout#v0.1.1: cd: ~ timeout: 180 - + - label: ":ubuntu: Ubuntu 18.04 - Build Pinned" plugins: - docker#v3.3.0: @@ -89,7 +88,7 @@ steps: command: - "./scripts/eosio_build.sh -y" timeout: 180 - + - label: ":centos: CentOS 7.7 - Build UnPinned" plugins: - docker#v3.3.0: @@ -129,7 +128,7 @@ steps: - EOSIO/skip-checkout#v0.1.1: cd: ~ timeout: 180 - + - label: ":ubuntu: Ubuntu 18.04 - Build UnPinned" plugins: - docker#v3.3.0: diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 260997d732..cde10ce088 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -76,8 +76,8 @@ for FILE in $(ls "$CICD_DIR/platforms/$PLATFORM_TYPE"); do export ANKA_TAG_BASE='' export ANKA_TEMPLATE_NAME='' fi - export PLATFORMS_JSON_ARRAY=$(echo $PLATFORMS_JSON_ARRAY | jq -c '. += [{ - "FILE_NAME": env.FILE_NAME, + export PLATFORMS_JSON_ARRAY=$(echo $PLATFORMS_JSON_ARRAY | jq -c '. += [{ + "FILE_NAME": env.FILE_NAME, "PLATFORM_NAME": env.PLATFORM_NAME, "PLATFORM_SKIP_VAR": env.PLATFORM_SKIP_VAR, "PLATFORM_NAME_UPCASE": env.PLATFORM_NAME_UPCASE, @@ -162,7 +162,7 @@ EOF failover-registries: - 'registry_1' - 'registry_2' - pre-commands: + pre-commands: - "rm -rf mac-anka-fleet; git clone git@github.com:EOSIO/mac-anka-fleet.git && cd mac-anka-fleet && . ./ensure-tag.bash -u 12 -r 25G -a '-n'" - EOSIO/skip-checkout#v0.1.1: cd: ~ @@ -341,26 +341,26 @@ EOF skip: $(echo "$PLATFORM_JSON" | jq -r '.PLATFORM_SKIP_VAR | env[.] // empty')${SKIP_SERIAL_TESTS} EOF - elif [ $TEST_NAME != 'rodeos_test_eosvmoc' -a \ - $TEST_NAME != 'rodeos_multi_ship_eos_vm_oc_idle_test' -a \ - $TEST_NAME != 'rodeos_multi_ship_eos_vm_oc_load_test' -a \ - $TEST_NAME != 'rodeos_idle_restart_producer_eos-vm-oc_test' -a \ - $TEST_NAME != 'rodeos_idle_restart_rodeos_eos-vm-oc_test' -a \ - $TEST_NAME != 'rodeos_idle_restart_ship_eos-vm-oc_test' -a \ - $TEST_NAME != 'rodeos_idle_restart_rodeos_producer_eos-vm-oc_test' -a \ - $TEST_NAME != 'rodeos_multi_ship_kill_clean_restart_eos_vm_oc_idle_test' -a \ - $TEST_NAME != 'rodeos_multi_ship_kill_clean_restart_eos_vm_oc_load_test' -a \ - $TEST_NAME != 'rodeos_multi_ship_kill_restart_eos_vm_oc_idle_test' -a \ - $TEST_NAME != 'rodeos_multi_ship_kill_restart_eos_vm_oc_load_test' -a \ - $TEST_NAME != 'rodeos_multi_ship_kill_clean_restart_eos_vm_oc_idle_unixsocket_test' -a \ - $TEST_NAME != 'rodeos_multi_ship_kill_clean_restart_eos_vm_oc_load_unixsocket_test' -a \ - $TEST_NAME != 'rodeos_multi_ship_kill_restart_eos_vm_oc_idle_unixsocket_test' -a \ - $TEST_NAME != 'rodeos_multi_ship_kill_restart_eos_vm_oc_load_unixsocket_test' -a \ - $TEST_NAME != 'rodeos_idle_restart_ship_unix-socket_eos-vm-oc_test' -a \ - $TEST_NAME != 'rodeos_idle_restart_rodeos_unix-socket_eos-vm-oc_test' -a \ - $TEST_NAME != 'rodeos_wasmQL_http_timeout_unix_socket_eos_vm_oc' -a \ - $TEST_NAME != 'rodeos_wasmQL_http_timeout_eos_vm_oc' ] - then + elif [ $TEST_NAME != 'rodeos_test_eosvmoc' -a \ + $TEST_NAME != 'rodeos_multi_ship_eos_vm_oc_idle_test' -a \ + $TEST_NAME != 'rodeos_multi_ship_eos_vm_oc_load_test' -a \ + $TEST_NAME != 'rodeos_idle_restart_producer_eos-vm-oc_test' -a \ + $TEST_NAME != 'rodeos_idle_restart_rodeos_eos-vm-oc_test' -a \ + $TEST_NAME != 'rodeos_idle_restart_ship_eos-vm-oc_test' -a \ + $TEST_NAME != 'rodeos_idle_restart_rodeos_producer_eos-vm-oc_test' -a \ + $TEST_NAME != 'rodeos_multi_ship_kill_clean_restart_eos_vm_oc_idle_test' -a \ + $TEST_NAME != 'rodeos_multi_ship_kill_clean_restart_eos_vm_oc_load_test' -a \ + $TEST_NAME != 'rodeos_multi_ship_kill_restart_eos_vm_oc_idle_test' -a \ + $TEST_NAME != 'rodeos_multi_ship_kill_restart_eos_vm_oc_load_test' -a \ + $TEST_NAME != 'rodeos_multi_ship_kill_clean_restart_eos_vm_oc_idle_unixsocket_test' -a \ + $TEST_NAME != 'rodeos_multi_ship_kill_clean_restart_eos_vm_oc_load_unixsocket_test' -a \ + $TEST_NAME != 'rodeos_multi_ship_kill_restart_eos_vm_oc_idle_unixsocket_test' -a \ + $TEST_NAME != 'rodeos_multi_ship_kill_restart_eos_vm_oc_load_unixsocket_test' -a \ + $TEST_NAME != 'rodeos_idle_restart_ship_unix-socket_eos-vm-oc_test' -a \ + $TEST_NAME != 'rodeos_idle_restart_rodeos_unix-socket_eos-vm-oc_test' -a \ + $TEST_NAME != 'rodeos_wasmQL_http_timeout_unix_socket_eos_vm_oc' -a \ + $TEST_NAME != 'rodeos_wasmQL_http_timeout_eos_vm_oc' ] + then cat < Date: Wed, 12 Jan 2022 13:27:39 -0500 Subject: [PATCH 17/62] Remove quotes from a TIMEOUT var --- .cicd/generate-pipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index cde10ce088..bdf35c4a18 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -659,7 +659,7 @@ cat < Date: Wed, 12 Jan 2022 13:31:46 -0500 Subject: [PATCH 18/62] Put the squigly ('~') after the wait step, before continue_on_failure --- .cicd/generate-pipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index bdf35c4a18..3c15b7bf68 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -544,7 +544,7 @@ EOF fi # pipeline tail cat < Date: Wed, 12 Jan 2022 13:34:24 -0500 Subject: [PATCH 19/62] Re-enable sync tests on tagged builds --- .cicd/generate-pipeline.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 3c15b7bf68..71aae42127 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -507,7 +507,6 @@ EOF - label: ":chains: Sync from Genesis Test" trigger: "eosio-sync-from-genesis" async: false - if: build.env("BUILDKITE_TAG") == null build: message: "Triggered by $BUILDKITE_PIPELINE_SLUG build $BUILDKITE_BUILD_NUMBER" commit: "${BUILDKITE_COMMIT}" @@ -527,7 +526,6 @@ EOF - label: ":outbox_tray: Resume from State Test" trigger: "eosio-resume-from-state" async: false - if: build.env("BUILDKITE_TAG") == null build: message: "Triggered by $BUILDKITE_PIPELINE_SLUG build $BUILDKITE_BUILD_NUMBER" commit: "${BUILDKITE_COMMIT}" From 78dd0d3a6b2c04726a979498879cb01296573f38 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Wed, 12 Jan 2022 14:18:09 -0500 Subject: [PATCH 20/62] Add wait step to beginning of build scripts pipeline --- .cicd/build-scripts.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.cicd/build-scripts.yml b/.cicd/build-scripts.yml index 6c5e57cc9d..1e9d544e56 100644 --- a/.cicd/build-scripts.yml +++ b/.cicd/build-scripts.yml @@ -1,4 +1,5 @@ steps: + - wait - label: ":aws: Amazon_Linux 2 - Build Pinned" plugins: From 106ca858aaa28a14a8f45d70fa0d2561bd196f24 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Wed, 12 Jan 2022 14:28:14 -0500 Subject: [PATCH 21/62] Add SKIP steps to base image and build script pipelines --- .cicd/base-images.yml | 10 ++++++++++ .cicd/build-scripts.yml | 10 ++++++++++ 2 files changed, 20 insertions(+) diff --git a/.cicd/base-images.yml b/.cicd/base-images.yml index e4d4dfe686..b8571e90b7 100644 --- a/.cicd/base-images.yml +++ b/.cicd/base-images.yml @@ -11,6 +11,7 @@ steps: agents: queue: "automation-eks-eos-builder-fleet" timeout: 180 + skip: ${SKIP_AMAZON_LINUX_2}${SKIP_LINUX} - label: ":centos: CentOS 7.7 - Base Image Pinned" command: @@ -22,6 +23,7 @@ steps: agents: queue: "automation-eks-eos-builder-fleet" timeout: 180 + skip: ${SKIP_CENTOS_7_7}${SKIP_LINUX} - label: ":darwin: macOS 10.14 - Base Image Pinned" command: @@ -45,6 +47,7 @@ steps: cd: ~ agents: "queue=mac-anka-node-fleet" timeout: 180 + skip: ${SKIP_MACOS_10_14}${SKIP_MAC} - label: ":darwin: macOS 10.15 - Base Image Pinned" command: @@ -68,6 +71,7 @@ steps: cd: ~ agents: "queue=mac-anka-node-fleet" timeout: 180 + skip: ${SKIP_MACOS_10_15}${SKIP_MAC} - label: ":ubuntu: Ubuntu 18.04 - Base Image Pinned" command: @@ -79,6 +83,7 @@ steps: agents: queue: "automation-eks-eos-builder-fleet" timeout: 180 + skip: ${SKIP_UBUNTU_18_04}${SKIP_LINUX} - label: ":aws: Amazon_Linux 2 - Base Image Unpinned" command: @@ -90,6 +95,7 @@ steps: agents: queue: "automation-eks-eos-builder-fleet" timeout: 180 + skip: ${SKIP_AMAZON_LINUX_2}${SKIP_LINUX} - label: ":centos: CentOS 7.7 - Base Image Unpinned" command: @@ -101,6 +107,7 @@ steps: agents: queue: "automation-eks-eos-builder-fleet" timeout: 180 + skip: ${SKIP_CENTOS_7_7}${SKIP_LINUX} - label: ":darwin: macOS 10.14 - Base Image Unpinned" command: @@ -124,6 +131,7 @@ steps: cd: ~ agents: "queue=mac-anka-node-fleet" timeout: 180 + skip: ${SKIP_MACOS_10_14}${SKIP_MAC} - label: ":darwin: macOS 10.15 - Base Image Unpinned" command: @@ -147,6 +155,7 @@ steps: cd: ~ agents: "queue=mac-anka-node-fleet" timeout: 180 + skip: ${SKIP_MACOS_10_15}${SKIP_MAC} - label: ":ubuntu: Ubuntu 18.04 - Base Image Unpinned" command: @@ -158,3 +167,4 @@ steps: agents: queue: "automation-eks-eos-builder-fleet" timeout: 180 + skip: ${SKIP_UBUNTU_18_04}${SKIP_LINUX} diff --git a/.cicd/build-scripts.yml b/.cicd/build-scripts.yml index 1e9d544e56..4e1c5ab124 100644 --- a/.cicd/build-scripts.yml +++ b/.cicd/build-scripts.yml @@ -11,6 +11,7 @@ steps: command: - "./scripts/eosio_build.sh -P -y" timeout: 180 + skip: ${SKIP_AMAZON_LINUX_2}${SKIP_LINUX} - label: ":centos: CentOS 7.7 - Build Pinned" plugins: @@ -22,6 +23,7 @@ steps: command: - "./scripts/eosio_build.sh -P -y" timeout: 180 + skip: ${SKIP_CENTOS_7_7}${SKIP_LINUX} - label: ":darwin: macOS 10.15 - Build Pinned" env: @@ -51,6 +53,7 @@ steps: - EOSIO/skip-checkout#v0.1.1: cd: ~ timeout: 180 + skip: ${SKIP_MACOS_10_15}${SKIP_MAC} - label: ":ubuntu: Ubuntu 18.04 - Build Pinned" plugins: @@ -63,6 +66,7 @@ steps: - "apt update && apt upgrade -y && apt install -y git" - "./scripts/eosio_build.sh -P -y" timeout: 180 + skip: ${SKIP_UBUNTU_18_04}${SKIP_LINUX} - label: ":ubuntu: Ubuntu 20.04 - Build Pinned" env: @@ -78,6 +82,7 @@ steps: - "apt update && apt upgrade -y && apt install -y git" - "./scripts/eosio_build.sh -P -y" timeout: 180 + skip: ${SKIP_UBUNTU_20_04}${SKIP_LINUX} - label: ":aws: Amazon_Linux 2 - Build UnPinned" plugins: @@ -89,6 +94,7 @@ steps: command: - "./scripts/eosio_build.sh -y" timeout: 180 + skip: ${SKIP_AMAZON_LINUX_2}${SKIP_LINUX} - label: ":centos: CentOS 7.7 - Build UnPinned" plugins: @@ -100,6 +106,7 @@ steps: command: - "./scripts/eosio_build.sh -y" timeout: 180 + skip: ${SKIP_CENTOS_7_7}${SKIP_LINUX} - label: ":darwin: macOS 10.15 - Build UnPinned" env: @@ -129,6 +136,7 @@ steps: - EOSIO/skip-checkout#v0.1.1: cd: ~ timeout: 180 + skip: ${SKIP_MACOS_10_15}${SKIP_MAC} - label: ":ubuntu: Ubuntu 18.04 - Build UnPinned" plugins: @@ -141,6 +149,7 @@ steps: - "apt update && apt upgrade -y && apt install -y git" - "./scripts/eosio_build.sh -y" timeout: 180 + skip: ${SKIP_UBUNTU_18_04}${SKIP_LINUX} - label: ":ubuntu: Ubuntu 20.04 - Build UnPinned" env: @@ -156,3 +165,4 @@ steps: - "apt update && apt upgrade -y && apt install -y git g++" - "./scripts/eosio_build.sh -y" timeout: 180 + skip: ${SKIP_UBUNTU_20_04}${SKIP_LINUX} From e0cba6c0472b3ce9b4ab4183a344513023bf03a8 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Wed, 12 Jan 2022 14:33:40 -0500 Subject: [PATCH 22/62] Copy Ubuntu 20.04 steps from eos:b3de424/.cicd/base-images.yml --- .cicd/base-images.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/.cicd/base-images.yml b/.cicd/base-images.yml index b8571e90b7..1d6bb66460 100644 --- a/.cicd/base-images.yml +++ b/.cicd/base-images.yml @@ -85,6 +85,18 @@ steps: timeout: 180 skip: ${SKIP_UBUNTU_18_04}${SKIP_LINUX} + - label: ":ubuntu: Ubuntu 20.04 - Base Image Pinned" + command: + - "./.cicd/generate-base-images.sh" + env: + FORCE_BASE_IMAGE: true + IMAGE_TAG: ubuntu-20.04-pinned + PLATFORM_TYPE: pinned + agents: + queue: "automation-eks-eos-builder-fleet" + timeout: 180 + skip: ${SKIP_UBUNTU_20_04}${SKIP_LINUX} + - label: ":aws: Amazon_Linux 2 - Base Image Unpinned" command: - "./.cicd/generate-base-images.sh" @@ -168,3 +180,15 @@ steps: queue: "automation-eks-eos-builder-fleet" timeout: 180 skip: ${SKIP_UBUNTU_18_04}${SKIP_LINUX} + + - label: ":ubuntu: Ubuntu 20.04 - Base Image Unpinned" + command: + - "./.cicd/generate-base-images.sh" + env: + FORCE_BASE_IMAGE: true + IMAGE_TAG: ubuntu-20.04-unpinned + PLATFORM_TYPE: unpinned + agents: + queue: "automation-eks-eos-builder-fleet" + timeout: 180 + skip: ${SKIP_UBUNTU_20_04}${SKIP_LINUX} From bec4baa73aae9d4c54e9eda9c2d547ac40dd4d61 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Wed, 12 Jan 2022 14:35:49 -0500 Subject: [PATCH 23/62] Copy homebrew bottle syntax modernization from eos:b3de424/scripts/generate_bottle.sh --- scripts/generate_bottle.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/generate_bottle.sh b/scripts/generate_bottle.sh index f9417679c3..9868633165 100644 --- a/scripts/generate_bottle.sh +++ b/scripts/generate_bottle.sh @@ -32,6 +32,8 @@ export SSUBPREFIX hash=`openssl dgst -sha256 ${NAME}.tar.gz | awk 'NF>1{print $NF}'` echo "class Eosio < Formula + # typed: false + # frozen_string_literal: true homepage \"${URL}\" revision 0 @@ -49,7 +51,7 @@ echo "class Eosio < Formula bottle do root_url \"https://github.com/eosio/eos/releases/download/v${VERSION}\" - sha256 \"${hash}\" => :${MAC_VERSION} + sha256 ${MAC_VERSION}: \"${hash}\" end def install raise \"Error, only supporting binary packages at this time\" From 1529ad3838feb895d229659d7c9ad10ed6435ee4 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Wed, 12 Jan 2022 14:44:11 -0500 Subject: [PATCH 24/62] Only upload pipeline documentation one time --- .cicd/generate-pipeline.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 71aae42127..dbe12b3121 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -13,6 +13,7 @@ if [[ "$BUILDKITE" == 'true' && "$RETRY" == '0' ]]; then echo "This documentation is also available on [GitHub]($DOCS_URL)." | buildkite-agent annotate --append --style 'info' --context 'documentation' cat .cicd/README.md | buildkite-agent annotate --append --style 'info' --context 'documentation' fi +buildkite-agent meta-data set pipeline-upload-retries "$(( $RETRY + 1 ))" # Determine if it's a forked PR and make sure to add git fetch so we don't have to git clone the forked repo's url if [[ $BUILDKITE_BRANCH =~ ^pull/[0-9]+/head: ]]; then PR_ID=$(echo $BUILDKITE_BRANCH | cut -d/ -f2) From efc23621eaba337cd82a71a1f17f1b5eb4f7861a Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Wed, 12 Jan 2022 15:00:56 -0500 Subject: [PATCH 25/62] Support running generate-pipeline.sh locally --- .cicd/generate-pipeline.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index dbe12b3121..d1c6e4796a 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -8,12 +8,12 @@ BUILDKITE_BUILD_AGENT_QUEUE='automation-eks-eos-builder-fleet' BUILDKITE_TEST_AGENT_QUEUE='automation-eks-eos-tester-fleet' # attach pipeline documentation export DOCS_URL="https://github.com/EOSIO/eos/blob/${BUILDKITE_COMMIT:-master}/.cicd/README.md" -export RETRY="$(buildkite-agent meta-data get pipeline-upload-retries --default '0')" +export RETRY="$([[ "$BUILDKITE" == 'true' ]] && buildkite-agent meta-data get pipeline-upload-retries --default '0' || echo "${RETRY:-0}")" if [[ "$BUILDKITE" == 'true' && "$RETRY" == '0' ]]; then echo "This documentation is also available on [GitHub]($DOCS_URL)." | buildkite-agent annotate --append --style 'info' --context 'documentation' cat .cicd/README.md | buildkite-agent annotate --append --style 'info' --context 'documentation' fi -buildkite-agent meta-data set pipeline-upload-retries "$(( $RETRY + 1 ))" +[[ "$BUILDKITE" == 'true' ]] && buildkite-agent meta-data set pipeline-upload-retries "$(( $RETRY + 1 ))" # Determine if it's a forked PR and make sure to add git fetch so we don't have to git clone the forked repo's url if [[ $BUILDKITE_BRANCH =~ ^pull/[0-9]+/head: ]]; then PR_ID=$(echo $BUILDKITE_BRANCH | cut -d/ -f2) From 63d8444bb3742795c0edb9dd07d4c4d77ce331ba Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Wed, 12 Jan 2022 15:02:18 -0500 Subject: [PATCH 26/62] Improve documentation links --- .cicd/generate-pipeline.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index d1c6e4796a..b606bf5b30 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -7,10 +7,10 @@ export PLATFORMS_JSON_ARRAY='[]' BUILDKITE_BUILD_AGENT_QUEUE='automation-eks-eos-builder-fleet' BUILDKITE_TEST_AGENT_QUEUE='automation-eks-eos-tester-fleet' # attach pipeline documentation -export DOCS_URL="https://github.com/EOSIO/eos/blob/${BUILDKITE_COMMIT:-master}/.cicd/README.md" +export DOCS_URL="https://github.com/EOSIO/eos/blob/$(git rev-parse HEAD)/.cicd" export RETRY="$([[ "$BUILDKITE" == 'true' ]] && buildkite-agent meta-data get pipeline-upload-retries --default '0' || echo "${RETRY:-0}")" if [[ "$BUILDKITE" == 'true' && "$RETRY" == '0' ]]; then - echo "This documentation is also available on [GitHub]($DOCS_URL)." | buildkite-agent annotate --append --style 'info' --context 'documentation' + echo "This documentation is also available on [GitHub]($DOCS_URL/README.md)." | buildkite-agent annotate --append --style 'info' --context 'documentation' cat .cicd/README.md | buildkite-agent annotate --append --style 'info' --context 'documentation' fi [[ "$BUILDKITE" == 'true' ]] && buildkite-agent meta-data set pipeline-upload-retries "$(( $RETRY + 1 ))" From ab3dba61b34fab0b136fea604ecd9c7770327b9e Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Wed, 12 Jan 2022 15:06:24 -0500 Subject: [PATCH 27/62] Sort serial and long-running tests by name --- .cicd/generate-pipeline.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index b606bf5b30..2fdd4588c8 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -322,7 +322,7 @@ EOF echo ' # serial tests' echo $PLATFORMS_JSON_ARRAY | jq -cr '.[]' | while read -r PLATFORM_JSON; do IFS=$oIFS - SERIAL_TESTS="$(cat tests/CMakeLists.txt | grep nonparallelizable_tests | grep -v "^#" | awk -F" " '{ print $2 }')" + SERIAL_TESTS="$(cat tests/CMakeLists.txt | grep nonparallelizable_tests | grep -v "^#" | awk -F ' ' '{ print $2 }' | sort | uniq)" for TEST_NAME in $SERIAL_TESTS; do if [[ ! "$(echo "$PLATFORM_JSON" | jq -r .FILE_NAME)" =~ 'macos' ]]; then cat < Date: Wed, 12 Jan 2022 15:10:31 -0500 Subject: [PATCH 28/62] Remove excess whitespace in auto-generated pipeline.yml --- .cicd/generate-pipeline.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 2fdd4588c8..e77edcca56 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -261,7 +261,6 @@ EOF EOF fi - echo done # wasm spec tests echo ' # wasm spec tests' @@ -316,7 +315,6 @@ EOF EOF fi - echo done # serial tests echo ' # serial tests' @@ -393,7 +391,6 @@ EOF EOF fi - echo done IFS=$nIFS done @@ -453,7 +450,6 @@ EOF EOF fi - echo done IFS=$nIFS done From 6fe671d2794439108f3704c421dceeff506dda76 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Wed, 12 Jan 2022 15:14:55 -0500 Subject: [PATCH 29/62] Pull in stability testing support from eos:b3de424 --- .cicd/generate-pipeline.sh | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index e77edcca56..eefa2c06ba 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -4,6 +4,7 @@ set -eo pipefail . ./.cicd/helpers/general.sh export PLATFORMS_JSON_ARRAY='[]' [[ -z "$ROUNDS" ]] && export ROUNDS='1' +[[ -z "$ROUND_SIZE" ]] && export ROUND_SIZE='1' BUILDKITE_BUILD_AGENT_QUEUE='automation-eks-eos-builder-fleet' BUILDKITE_TEST_AGENT_QUEUE='automation-eks-eos-tester-fleet' # attach pipeline documentation @@ -184,8 +185,7 @@ EOF EOF fi done -cat < Date: Wed, 12 Jan 2022 15:24:01 -0500 Subject: [PATCH 30/62] Bring in eos:b3de424/.cicd/eosio-test-stability.md and associated code --- .cicd/eosio-test-stability.md | 84 +++++++++++++++++++++++++++++++++++ .cicd/generate-pipeline.sh | 4 ++ 2 files changed, 88 insertions(+) create mode 100644 .cicd/eosio-test-stability.md diff --git a/.cicd/eosio-test-stability.md b/.cicd/eosio-test-stability.md new file mode 100644 index 0000000000..b2ae0c99f8 --- /dev/null +++ b/.cicd/eosio-test-stability.md @@ -0,0 +1,84 @@ +# Stability Testing +Stability testing of EOSIO unit and integration tests is done in the [eosio-test-stability](https://buildkite.com/EOSIO/eosio-test-stability) pipeline. It will take thousands of runs of any given test to identify it as "stable" or "unstable". Runs should be split evenly across "pinned" (fixed dependency version) and "unpinned" (default dependency version) builds because, sometimes, test instability is only expressed in one of these environments. Finally, stability testing should be performed on the Linux fleet first because this fleet is effectively infinite. Once stability is demonstrated on Linux, testing can be performed on the finite macOS Anka fleet. + +
+See More + +## Index +1. [Configuration](eosio-test-stability.md#configuration) + 1. [Variables](eosio-test-stability.md#variables) + 1. [Runs](eosio-test-stability.md#runs) + 1. [Examples](eosio-test-stability.md#examples) +1. [See Also](eosio-test-stability.md#see-also) + +## Configuration +The [eosio-test-stability](https://buildkite.com/EOSIO/eosio-test-stability) pipeline uses the same pipeline upload script as [eosio](https://buildkite.com/EOSIO/eosio), [eosio-build-unpinned](https://buildkite.com/EOSIO/eosio-build-unpinned), and [eosio-lrt](https://buildkite.com/EOSIO/eosio-lrt), so all variables from the [pipeline documentation](README.md) apply. + +### Variables +There are five primary environment variables relevant to stability testing: +```bash +CONTINUE_ON_FAILURE='true|false' # by default, only scheduled builds will continue to the following round if + # any test fails for the current round; however, this setting can be explicitly + # overriden by setting this variable to 'true'. +PINNED='true|false' # whether to perform the test with pinned dependencies, or default dependencies +ROUNDS='ℕ' # natural number defining the number of gated rounds of tests to generate +ROUND_SIZE='ℕ' # number of test steps to generate per operating system, per round +SKIP_MAC='true|false' # conserve finite macOS Anka agents by excluding them from your testing +TEST='name' # PCRE expression defining the tests to run, preceded by '^' and followed by '$' +TIMEOUT='ℕ' # set timeout in minutes for all Buildkite steps +``` +The `TEST` variable is parsed as [pearl-compatible regular expression](https://www.debuggex.com/cheatsheet/regex/pcre) where the expression in `TEST` is preceded by `^` and followed by `$`. To specify one test, set `TEST` equal to the test name (e.g. `TEST='read_only_query'`). Specify two tests as `TEST='(nodeos_short_fork_take_over_lr_test|read_only_query)'`. Or, perhaps, you want all of the `restart_scenarios` tests. Then, you could define `TEST='restart-scenario-test-.*'` and Buildkite will generate `ROUND_SIZE` steps each round for each operating system for all three restart scenarios tests. + +### Runs +The number of total test runs will be: +```bash +RUNS = ROUNDS * ROUND_SIZE * OS_COUNT * TEST_COUNT # where: +OS_COUNT = 'ℕ' # the number of supported operating systems +TEST_COUNT = 'ℕ' # the number of tests matching the PCRE filter in TEST +``` + +### Examples +We recommend stability testing one test per build with two builds per test, on Linux at first. Kick off one pinned build on Linux... +```bash +PINNED='true' +ROUNDS='42' +ROUND_SIZE'5' +SKIP_MAC='true' +TEST='read_only_query' +``` +...and one unpinned build on Linux: +```bash +PINNED='true' +ROUNDS='42' +ROUND_SIZE'5' +SKIP_MAC='true' +TEST='read_only_query' +``` +Once the Linux runs have proven stable, and if instability was observed on macOS, kick off two equivalent builds on macOS instead of Linux. One pinned build on macOS... +```bash +PINNED='true' +ROUNDS='42' +ROUND_SIZE'5' +SKIP_LINUX='true' +SKIP_MAC='false' +TEST='read_only_query' +``` +...and one unpinned build on macOS: +```bash +PINNED='true' +ROUNDS='42' +ROUND_SIZE'5' +SKIP_LINUX='true' +SKIP_MAC='false' +TEST='read_only_query' +``` +If these runs are against `eos:develop` and `develop` has five supported operating systems, this pattern would consist of 2,100 runs per test across all four builds. If the runs are against `eos:release/2.1.x` which, at the time of this writing, supports eight operating systems, this pattern would consist of 3,360 runs per test across all four builds. This gives you and your team strong confidence that any test instability occurs less than 1% of the time. + +# See Also +- Buildkite + - [DevDocs](https://github.com/EOSIO/devdocs/wiki/Buildkite) + - [EOSIO Pipelines](https://github.com/EOSIO/eos/blob/HEAD/.cicd/README.md) + - [Run Your First Build](https://buildkite.com/docs/tutorials/getting-started#run-your-first-build) +- [#help-automation](https://blockone.slack.com/archives/CMTAZ9L4D) Slack Channel + +
diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index eefa2c06ba..5841b8156e 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -13,6 +13,10 @@ export RETRY="$([[ "$BUILDKITE" == 'true' ]] && buildkite-agent meta-data get pi if [[ "$BUILDKITE" == 'true' && "$RETRY" == '0' ]]; then echo "This documentation is also available on [GitHub]($DOCS_URL/README.md)." | buildkite-agent annotate --append --style 'info' --context 'documentation' cat .cicd/README.md | buildkite-agent annotate --append --style 'info' --context 'documentation' + if [[ "$BUILDKITE_PIPELINE_SLUG" == 'eosio-test-stability' ]]; then + echo "This documentation is also available on [GitHub]($DOCS_URL/eosio-test-stability.md)." | buildkite-agent annotate --append --style 'info' --context 'test-stability' + cat .cicd/eosio-test-stability.md | buildkite-agent annotate --append --style 'info' --context 'test-stability' + fi fi [[ "$BUILDKITE" == 'true' ]] && buildkite-agent meta-data set pipeline-upload-retries "$(( $RETRY + 1 ))" # Determine if it's a forked PR and make sure to add git fetch so we don't have to git clone the forked repo's url From 6c44bc2dd0d3f86728be7d0acef5ed22624283ca Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Wed, 12 Jan 2022 15:24:54 -0500 Subject: [PATCH 31/62] Guard against spawning too many test stability jobs --- .cicd/generate-pipeline.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 5841b8156e..65b83c1798 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -19,6 +19,15 @@ if [[ "$BUILDKITE" == 'true' && "$RETRY" == '0' ]]; then fi fi [[ "$BUILDKITE" == 'true' ]] && buildkite-agent meta-data set pipeline-upload-retries "$(( $RETRY + 1 ))" +# guard against accidentally spawning too many jobs +if (( $ROUNDS > 1 || $ROUND_SIZE > 1 )) && [[ "$BUILDKITE_PIPELINE_SLUG" != 'eosio-test-stability' ]]; then + echo '+++ :no_entry: WARNING: Your parameters will spawn a very large number of jobs!' 1>&2 + echo "Setting ROUNDS='$ROUNDS' and/or ROUND_SIZE='$ROUND_SIZE' in the environment will cause ALL tests to be run $(( $ROUNDS * $ROUND_SIZE )) times, which will consume a large number of agents!" 1>&2 + [[ "$BUILDKITE" == 'true' ]] && cat | buildkite-agent annotate --append --style 'error' --context 'no-TEST' <<-MD +Your build was cancelled because you set \`ROUNDS\` and/or \`ROUND_SIZE\` outside the [eosio-test-stability](https://buildkite.com/EOSIO/eosio-test-stability) pipeline. +MD + exit 255 +fi # Determine if it's a forked PR and make sure to add git fetch so we don't have to git clone the forked repo's url if [[ $BUILDKITE_BRANCH =~ ^pull/[0-9]+/head: ]]; then PR_ID=$(echo $BUILDKITE_BRANCH | cut -d/ -f2) From a97b7b88fc2a4f315ef491e34307a16a50d84ce4 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Wed, 12 Jan 2022 15:26:47 -0500 Subject: [PATCH 32/62] Change SKIP_MACOS_10_14 to SKIP_MACOS_10_15 --- .cicd/generate-pipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 65b83c1798..46f130af4f 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -677,7 +677,7 @@ EOF agents: queue: "automation-basic-builder-fleet" timeout: ${TIMEOUT:-5} - skip: ${SKIP_PACKAGE_BUILDER}${SKIP_MAC}${SKIP_MACOS_10_14} + skip: ${SKIP_PACKAGE_BUILDER}${SKIP_MAC}${SKIP_MACOS_10_15} - label: ":docker: :ubuntu: Docker - Build 18.04 Docker Image" command: "./.cicd/create-docker-from-binary.sh" From 076f603dbb0f275a50526ce635f69d0c6feea672 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Wed, 12 Jan 2022 15:50:10 -0500 Subject: [PATCH 33/62] Copy in package tests from eos:b3de424 --- .cicd/generate-pipeline.sh | 86 +++++++++++++++++++++++++++++++++++- .cicd/libfunctions.sh | 6 +++ .cicd/test-package.anka.sh | 13 ++++++ .cicd/test-package.docker.sh | 9 ++++ .cicd/test-package.run.sh | 31 +++++++++++++ 5 files changed, 144 insertions(+), 1 deletion(-) create mode 100755 .cicd/libfunctions.sh create mode 100755 .cicd/test-package.anka.sh create mode 100755 .cicd/test-package.docker.sh create mode 100755 .cicd/test-package.run.sh diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 46f130af4f..da906dff59 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -595,6 +595,33 @@ EOF PKGTYPE: "rpm" agents: queue: "$BUILDKITE_TEST_AGENT_QUEUE" + key: "centos7pb" + timeout: ${TIMEOUT:-10} + skip: ${SKIP_CENTOS_7_7}${SKIP_PACKAGE_BUILDER}${SKIP_LINUX} + + - label: ":centos: CentOS 7 - Test Package" + command: + - "buildkite-agent artifact download '*.rpm' . --step ':centos: CentOS 7.7 - Package Builder' --agent-access-token \$\$BUILDKITE_AGENT_ACCESS_TOKEN" + - "./.cicd/test-package.docker.sh" + env: + IMAGE: "centos:7" + agents: + queue: "$BUILDKITE_TEST_AGENT_QUEUE" + depends_on: "centos7pb" + allow_dependency_failure: false + timeout: ${TIMEOUT:-10} + skip: ${SKIP_CENTOS_7_7}${SKIP_PACKAGE_BUILDER}${SKIP_LINUX} + + - label: ":aws: Amazon Linux 2 - Test Package" + command: + - "buildkite-agent artifact download '*.rpm' . --step ':centos: CentOS 7.7 - Package Builder' --agent-access-token \$\$BUILDKITE_AGENT_ACCESS_TOKEN" + - "./.cicd/test-package.docker.sh" + env: + IMAGE: "amazonlinux:2" + agents: + queue: "$BUILDKITE_TEST_AGENT_QUEUE" + depends_on: "centos7pb" + allow_dependency_failure: false timeout: ${TIMEOUT:-10} skip: ${SKIP_CENTOS_7_7}${SKIP_PACKAGE_BUILDER}${SKIP_LINUX} @@ -609,6 +636,20 @@ EOF PKGTYPE: "deb" agents: queue: "$BUILDKITE_TEST_AGENT_QUEUE" + key: "ubuntu1804pb" + timeout: ${TIMEOUT:-10} + skip: ${SKIP_UBUNTU_18_04}${SKIP_PACKAGE_BUILDER}${SKIP_LINUX} + + - label: ":ubuntu: Ubuntu 18.04 - Test Package" + command: + - "buildkite-agent artifact download '*.deb' . --step ':ubuntu: Ubuntu 18.04 - Package Builder' --agent-access-token \$\$BUILDKITE_AGENT_ACCESS_TOKEN" + - "./.cicd/test-package.docker.sh" + env: + IMAGE: "ubuntu:18.04" + agents: + queue: "$BUILDKITE_TEST_AGENT_QUEUE" + depends_on: "ubuntu1804pb" + allow_dependency_failure: false timeout: ${TIMEOUT:-10} skip: ${SKIP_UBUNTU_18_04}${SKIP_PACKAGE_BUILDER}${SKIP_LINUX} @@ -623,7 +664,21 @@ EOF PKGTYPE: "deb" agents: queue: "$BUILDKITE_TEST_AGENT_QUEUE" - timeout: ${TIMEOUT:-20} + key: "ubuntu2004pb" + timeout: ${TIMEOUT:-10} + skip: ${SKIP_UBUNTU_20_04}${SKIP_PACKAGE_BUILDER}${SKIP_LINUX} + + - label: ":ubuntu: Ubuntu 20.04 - Test Package" + command: + - "buildkite-agent artifact download '*.deb' . --step ':ubuntu: Ubuntu 20.04 - Package Builder' --agent-access-token \$\$BUILDKITE_AGENT_ACCESS_TOKEN" + - "./.cicd/test-package.docker.sh" + env: + IMAGE: "ubuntu:20.04" + agents: + queue: "$BUILDKITE_TEST_AGENT_QUEUE" + depends_on: "ubuntu2004pb" + allow_dependency_failure: false + timeout: ${TIMEOUT:-10} skip: ${SKIP_UBUNTU_20_04}${SKIP_PACKAGE_BUILDER}${SKIP_LINUX} - label: ":darwin: macOS 10.15 - Package Builder" @@ -649,9 +704,38 @@ EOF cd: ~ agents: - "queue=mac-anka-node-fleet" + key: "macos1015pb" timeout: ${TIMEOUT:-30} skip: ${SKIP_MACOS_10_15}${SKIP_PACKAGE_BUILDER}${SKIP_MAC} + - label: ":darwin: macOS 10.15 - Test Package" + command: + - "git clone \$BUILDKITE_REPO eos && cd eos && $GIT_FETCH git checkout -f \$BUILDKITE_COMMIT" + - "cd eos && buildkite-agent artifact download '*' . --step ':darwin: macOS 10.15 - Package Builder' --agent-access-token \$\$BUILDKITE_AGENT_ACCESS_TOKEN" + - "cd eos && ./.cicd/test-package.anka.sh" + plugins: + - EOSIO/anka#v0.6.1: + no-volume: true + inherit-environment-vars: true + vm-name: 10.15.5_6C_14G_80G + vm-registry-tag: "clean::cicd::git-ssh::nas::brew::buildkite-agent" + always-pull: true + debug: true + wait-network: true + pre-execute-sleep: 5 + pre-execute-ping-sleep: github.com + failover-registries: + - 'registry_1' + - 'registry_2' + - EOSIO/skip-checkout#v0.1.1: + cd: ~ + agents: + - "queue=mac-anka-node-fleet" + depends_on: "macos1015pb" + allow_dependency_failure: false + timeout: ${TIMEOUT:-10} + skip: ${SKIP_MACOS_10_15}${SKIP_PACKAGE_BUILDER}${SKIP_MAC} + - label: ":docker: Docker - Label Container with Git Branch and Git Tag" command: .cicd/docker-tag.sh env: diff --git a/.cicd/libfunctions.sh b/.cicd/libfunctions.sh new file mode 100755 index 0000000000..9c820a364b --- /dev/null +++ b/.cicd/libfunctions.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +function perform { + echo "$ $1" + eval $1 +} diff --git a/.cicd/test-package.anka.sh b/.cicd/test-package.anka.sh new file mode 100755 index 0000000000..06be17a299 --- /dev/null +++ b/.cicd/test-package.anka.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -euo pipefail + +. "${0%/*}/libfunctions.sh" + +echo '--- :anka: Pretest Setup' + +if [[ ! $(python3 --version 2>/dev/null) ]]; then + perform 'brew update' + perform 'brew install python3' +fi + +perform "./.cicd/test-package.run.sh" diff --git a/.cicd/test-package.docker.sh b/.cicd/test-package.docker.sh new file mode 100755 index 0000000000..9508e216ff --- /dev/null +++ b/.cicd/test-package.docker.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -euo pipefail + +. "${0%/*}/libfunctions.sh" + +echo '--- :docker: Pretest Setup' + +perform "docker pull $IMAGE" +perform "docker run --rm -v \"\$(pwd):/eos\" -w '/eos' -it $IMAGE ./.cicd/test-package.run.sh" diff --git a/.cicd/test-package.run.sh b/.cicd/test-package.run.sh new file mode 100755 index 0000000000..748be92772 --- /dev/null +++ b/.cicd/test-package.run.sh @@ -0,0 +1,31 @@ +#!/bin/bash +set -euo pipefail + +. "${0%/*}/libfunctions.sh" + +echo '+++ :minidisc: Installing EOSIO' + +if [[ $(apt-get --version 2>/dev/null) ]]; then # debian family packaging + perform 'apt-get update' + perform 'apt-get install -y /eos/*.deb' +elif [[ $(yum --version 2>/dev/null) ]]; then # RHEL family packaging + perform 'yum check-update || :' + perform 'yum install -y /eos/*.rpm' +elif [[ $(brew --version 2>/dev/null) ]]; then # homebrew packaging + perform 'brew update' + perform 'mkdir homebrew-eosio' + perform 'git init homebrew-eosio' + perform 'cp *.rb homebrew-eosio' + perform "sed -i.bk -e 's/url \".*\"/url \"http:\/\/127.0.0.1:7800\"/' homebrew-eosio/*.rb" + perform "pushd homebrew-eosio && git add *.rb && git commit -m 'test it!' && popd" + perform "brew tap eosio/eosio homebrew-eosio" + perform '{ python3 -m http.server 7800 & } && export HTTP_SERVER_PID=$!' + perform 'sleep 20s' + perform 'brew install eosio' + perform 'kill $HTTP_SERVER_PID' +else + echo 'ERROR: Package manager not detected!' + exit 3 +fi + +nodeos --full-version From 447f8136fb523d5a9cd9e97c7873ec60f968b9ff Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Wed, 12 Jan 2022 15:51:47 -0500 Subject: [PATCH 34/62] Only publish containers which pass the package tests --- .cicd/generate-pipeline.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index da906dff59..afd0bcd7fe 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -736,6 +736,8 @@ EOF timeout: ${TIMEOUT:-10} skip: ${SKIP_MACOS_10_15}${SKIP_PACKAGE_BUILDER}${SKIP_MAC} + - wait + - label: ":docker: Docker - Label Container with Git Branch and Git Tag" command: .cicd/docker-tag.sh env: @@ -746,8 +748,6 @@ EOF timeout: ${TIMEOUT:-10} skip: ${SKIP_INSTALL}${SKIP_LINUX}${SKIP_DOCKER}${SKIP_PACKAGE_BUILDER} - - wait - - label: ":git: Git Submodule Regression Check" command: "./.cicd/submodule-regression-check.sh" agents: From 2c1bd1798a697c02c8d344c286112339922c3d5a Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Wed, 12 Jan 2022 15:59:03 -0500 Subject: [PATCH 35/62] Simplify package tests by moving them into their own group instead of using DAG dependencies --- .cicd/generate-pipeline.sh | 115 ++++++++++++++++++------------------- 1 file changed, 57 insertions(+), 58 deletions(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index afd0bcd7fe..7f66379877 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -595,33 +595,6 @@ EOF PKGTYPE: "rpm" agents: queue: "$BUILDKITE_TEST_AGENT_QUEUE" - key: "centos7pb" - timeout: ${TIMEOUT:-10} - skip: ${SKIP_CENTOS_7_7}${SKIP_PACKAGE_BUILDER}${SKIP_LINUX} - - - label: ":centos: CentOS 7 - Test Package" - command: - - "buildkite-agent artifact download '*.rpm' . --step ':centos: CentOS 7.7 - Package Builder' --agent-access-token \$\$BUILDKITE_AGENT_ACCESS_TOKEN" - - "./.cicd/test-package.docker.sh" - env: - IMAGE: "centos:7" - agents: - queue: "$BUILDKITE_TEST_AGENT_QUEUE" - depends_on: "centos7pb" - allow_dependency_failure: false - timeout: ${TIMEOUT:-10} - skip: ${SKIP_CENTOS_7_7}${SKIP_PACKAGE_BUILDER}${SKIP_LINUX} - - - label: ":aws: Amazon Linux 2 - Test Package" - command: - - "buildkite-agent artifact download '*.rpm' . --step ':centos: CentOS 7.7 - Package Builder' --agent-access-token \$\$BUILDKITE_AGENT_ACCESS_TOKEN" - - "./.cicd/test-package.docker.sh" - env: - IMAGE: "amazonlinux:2" - agents: - queue: "$BUILDKITE_TEST_AGENT_QUEUE" - depends_on: "centos7pb" - allow_dependency_failure: false timeout: ${TIMEOUT:-10} skip: ${SKIP_CENTOS_7_7}${SKIP_PACKAGE_BUILDER}${SKIP_LINUX} @@ -636,20 +609,6 @@ EOF PKGTYPE: "deb" agents: queue: "$BUILDKITE_TEST_AGENT_QUEUE" - key: "ubuntu1804pb" - timeout: ${TIMEOUT:-10} - skip: ${SKIP_UBUNTU_18_04}${SKIP_PACKAGE_BUILDER}${SKIP_LINUX} - - - label: ":ubuntu: Ubuntu 18.04 - Test Package" - command: - - "buildkite-agent artifact download '*.deb' . --step ':ubuntu: Ubuntu 18.04 - Package Builder' --agent-access-token \$\$BUILDKITE_AGENT_ACCESS_TOKEN" - - "./.cicd/test-package.docker.sh" - env: - IMAGE: "ubuntu:18.04" - agents: - queue: "$BUILDKITE_TEST_AGENT_QUEUE" - depends_on: "ubuntu1804pb" - allow_dependency_failure: false timeout: ${TIMEOUT:-10} skip: ${SKIP_UBUNTU_18_04}${SKIP_PACKAGE_BUILDER}${SKIP_LINUX} @@ -664,20 +623,6 @@ EOF PKGTYPE: "deb" agents: queue: "$BUILDKITE_TEST_AGENT_QUEUE" - key: "ubuntu2004pb" - timeout: ${TIMEOUT:-10} - skip: ${SKIP_UBUNTU_20_04}${SKIP_PACKAGE_BUILDER}${SKIP_LINUX} - - - label: ":ubuntu: Ubuntu 20.04 - Test Package" - command: - - "buildkite-agent artifact download '*.deb' . --step ':ubuntu: Ubuntu 20.04 - Package Builder' --agent-access-token \$\$BUILDKITE_AGENT_ACCESS_TOKEN" - - "./.cicd/test-package.docker.sh" - env: - IMAGE: "ubuntu:20.04" - agents: - queue: "$BUILDKITE_TEST_AGENT_QUEUE" - depends_on: "ubuntu2004pb" - allow_dependency_failure: false timeout: ${TIMEOUT:-10} skip: ${SKIP_UBUNTU_20_04}${SKIP_PACKAGE_BUILDER}${SKIP_LINUX} @@ -704,10 +649,66 @@ EOF cd: ~ agents: - "queue=mac-anka-node-fleet" - key: "macos1015pb" timeout: ${TIMEOUT:-30} skip: ${SKIP_MACOS_10_15}${SKIP_PACKAGE_BUILDER}${SKIP_MAC} + - wait + + - label: ":aws: Amazon Linux 2 - Test Package" + command: + - "buildkite-agent artifact download '*.rpm' . --step ':centos: CentOS 7.7 - Package Builder' --agent-access-token \$\$BUILDKITE_AGENT_ACCESS_TOKEN" + - "./.cicd/test-package.docker.sh" + env: + IMAGE: "amazonlinux:2" + agents: + queue: "$BUILDKITE_TEST_AGENT_QUEUE" + timeout: ${TIMEOUT:-10} + skip: ${SKIP_CENTOS_7_7}${SKIP_PACKAGE_BUILDER}${SKIP_LINUX} + + - label: ":centos: CentOS 7 - Test Package" + command: + - "buildkite-agent artifact download '*.rpm' . --step ':centos: CentOS 7.7 - Package Builder' --agent-access-token \$\$BUILDKITE_AGENT_ACCESS_TOKEN" + - "./.cicd/test-package.docker.sh" + env: + IMAGE: "centos:7" + agents: + queue: "$BUILDKITE_TEST_AGENT_QUEUE" + timeout: ${TIMEOUT:-10} + skip: ${SKIP_CENTOS_7_7}${SKIP_PACKAGE_BUILDER}${SKIP_LINUX} + + - label: ":ubuntu: Ubuntu 16.04 - Test Package" + command: + - "buildkite-agent artifact download '*.deb' . --step ':ubuntu: Ubuntu 16.04 - Package Builder' --agent-access-token \$\$BUILDKITE_AGENT_ACCESS_TOKEN" + - "./.cicd/test-package.docker.sh" + env: + IMAGE: "ubuntu:16.04" + agents: + queue: "$BUILDKITE_TEST_AGENT_QUEUE" + timeout: ${TIMEOUT:-10} + skip: ${SKIP_UBUNTU_16_04}${SKIP_PACKAGE_BUILDER}${SKIP_LINUX} + + - label: ":ubuntu: Ubuntu 18.04 - Test Package" + command: + - "buildkite-agent artifact download '*.deb' . --step ':ubuntu: Ubuntu 18.04 - Package Builder' --agent-access-token \$\$BUILDKITE_AGENT_ACCESS_TOKEN" + - "./.cicd/test-package.docker.sh" + env: + IMAGE: "ubuntu:18.04" + agents: + queue: "$BUILDKITE_TEST_AGENT_QUEUE" + timeout: ${TIMEOUT:-10} + skip: ${SKIP_UBUNTU_18_04}${SKIP_PACKAGE_BUILDER}${SKIP_LINUX} + + - label: ":ubuntu: Ubuntu 20.04 - Test Package" + command: + - "buildkite-agent artifact download '*.deb' . --step ':ubuntu: Ubuntu 20.04 - Package Builder' --agent-access-token \$\$BUILDKITE_AGENT_ACCESS_TOKEN" + - "./.cicd/test-package.docker.sh" + env: + IMAGE: "ubuntu:20.04" + agents: + queue: "$BUILDKITE_TEST_AGENT_QUEUE" + timeout: ${TIMEOUT:-10} + skip: ${SKIP_UBUNTU_20_04}${SKIP_PACKAGE_BUILDER}${SKIP_LINUX} + - label: ":darwin: macOS 10.15 - Test Package" command: - "git clone \$BUILDKITE_REPO eos && cd eos && $GIT_FETCH git checkout -f \$BUILDKITE_COMMIT" @@ -731,8 +732,6 @@ EOF cd: ~ agents: - "queue=mac-anka-node-fleet" - depends_on: "macos1015pb" - allow_dependency_failure: false timeout: ${TIMEOUT:-10} skip: ${SKIP_MACOS_10_15}${SKIP_PACKAGE_BUILDER}${SKIP_MAC} From 036b6d588e337a731580eb122c30cfd9be3c9212 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Wed, 12 Jan 2022 16:00:22 -0500 Subject: [PATCH 36/62] Make it clearer what the package tests are doing --- .cicd/generate-pipeline.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 7f66379877..542269bfa8 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -654,7 +654,7 @@ EOF - wait - - label: ":aws: Amazon Linux 2 - Test Package" + - label: ":aws: Amazon Linux 2 - Install Package" command: - "buildkite-agent artifact download '*.rpm' . --step ':centos: CentOS 7.7 - Package Builder' --agent-access-token \$\$BUILDKITE_AGENT_ACCESS_TOKEN" - "./.cicd/test-package.docker.sh" @@ -665,7 +665,7 @@ EOF timeout: ${TIMEOUT:-10} skip: ${SKIP_CENTOS_7_7}${SKIP_PACKAGE_BUILDER}${SKIP_LINUX} - - label: ":centos: CentOS 7 - Test Package" + - label: ":centos: CentOS 7 - Install Package" command: - "buildkite-agent artifact download '*.rpm' . --step ':centos: CentOS 7.7 - Package Builder' --agent-access-token \$\$BUILDKITE_AGENT_ACCESS_TOKEN" - "./.cicd/test-package.docker.sh" @@ -676,7 +676,7 @@ EOF timeout: ${TIMEOUT:-10} skip: ${SKIP_CENTOS_7_7}${SKIP_PACKAGE_BUILDER}${SKIP_LINUX} - - label: ":ubuntu: Ubuntu 16.04 - Test Package" + - label: ":ubuntu: Ubuntu 16.04 - Install Package" command: - "buildkite-agent artifact download '*.deb' . --step ':ubuntu: Ubuntu 16.04 - Package Builder' --agent-access-token \$\$BUILDKITE_AGENT_ACCESS_TOKEN" - "./.cicd/test-package.docker.sh" @@ -687,7 +687,7 @@ EOF timeout: ${TIMEOUT:-10} skip: ${SKIP_UBUNTU_16_04}${SKIP_PACKAGE_BUILDER}${SKIP_LINUX} - - label: ":ubuntu: Ubuntu 18.04 - Test Package" + - label: ":ubuntu: Ubuntu 18.04 - Install Package" command: - "buildkite-agent artifact download '*.deb' . --step ':ubuntu: Ubuntu 18.04 - Package Builder' --agent-access-token \$\$BUILDKITE_AGENT_ACCESS_TOKEN" - "./.cicd/test-package.docker.sh" @@ -698,7 +698,7 @@ EOF timeout: ${TIMEOUT:-10} skip: ${SKIP_UBUNTU_18_04}${SKIP_PACKAGE_BUILDER}${SKIP_LINUX} - - label: ":ubuntu: Ubuntu 20.04 - Test Package" + - label: ":ubuntu: Ubuntu 20.04 - Install Package" command: - "buildkite-agent artifact download '*.deb' . --step ':ubuntu: Ubuntu 20.04 - Package Builder' --agent-access-token \$\$BUILDKITE_AGENT_ACCESS_TOKEN" - "./.cicd/test-package.docker.sh" @@ -709,7 +709,7 @@ EOF timeout: ${TIMEOUT:-10} skip: ${SKIP_UBUNTU_20_04}${SKIP_PACKAGE_BUILDER}${SKIP_LINUX} - - label: ":darwin: macOS 10.15 - Test Package" + - label: ":darwin: macOS 10.15 - Install Package" command: - "git clone \$BUILDKITE_REPO eos && cd eos && $GIT_FETCH git checkout -f \$BUILDKITE_COMMIT" - "cd eos && buildkite-agent artifact download '*' . --step ':darwin: macOS 10.15 - Package Builder' --agent-access-token \$\$BUILDKITE_AGENT_ACCESS_TOKEN" From 2bbec71721f25a070663a67202f791dc5e814321 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Wed, 12 Jan 2022 15:44:05 -0500 Subject: [PATCH 37/62] Make the Linux command steps less error-prone --- .cicd/generate-pipeline.sh | 100 ++++++++++++++++++------------------- 1 file changed, 50 insertions(+), 50 deletions(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 542269bfa8..e6845500fe 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -226,9 +226,9 @@ if [[ "$DCMAKE_BUILD_TYPE" != 'Debug' ]]; then if [[ ! "$(echo "$PLATFORM_JSON" | jq -r .FILE_NAME)" =~ 'macos' ]]; then cat <> ~/.ssh/known_hosts" - - "git clone \$BUILDKITE_REPO ." - - "$GIT_FETCH git checkout -f \$BUILDKITE_COMMIT" - - "echo '+++ :compression: Extracting Test Metrics Code'" - - "tar -zxf .cicd/metrics/test-metrics.tar.gz" - - "echo '+++ :javascript: Running test-metrics.js'" - - "node --max-old-space-size=32768 test-metrics.js" + command: | + ssh-keyscan -H github.com >> ~/.ssh/known_hosts + git clone \$BUILDKITE_REPO . + $GIT_FETCH git checkout -f \$BUILDKITE_COMMIT + echo '+++ :compression: Extracting Test Metrics Code' + tar -zxf .cicd/metrics/test-metrics.tar.gz + echo '+++ :javascript: Running test-metrics.js' + node --max-old-space-size=32768 test-metrics.js plugins: - EOSIO/skip-checkout#v0.1.1: cd: ~ @@ -585,9 +585,9 @@ EOF # packaging - label: ":centos: CentOS 7.7 - Package Builder" - command: - - "buildkite-agent artifact download build.tar.gz . --step ':centos: CentOS 7.7 - Build' && tar -xzf build.tar.gz" - - "./.cicd/package.sh" + command: | + buildkite-agent artifact download build.tar.gz . --step ':centos: CentOS 7.7 - Build' && tar -xzf build.tar.gz + ./.cicd/package.sh env: IMAGE_TAG: "centos-7.7-$PLATFORM_TYPE" PLATFORM_TYPE: $PLATFORM_TYPE @@ -599,9 +599,9 @@ EOF skip: ${SKIP_CENTOS_7_7}${SKIP_PACKAGE_BUILDER}${SKIP_LINUX} - label: ":ubuntu: Ubuntu 18.04 - Package Builder" - command: - - "buildkite-agent artifact download build.tar.gz . --step ':ubuntu: Ubuntu 18.04 - Build' && tar -xzf build.tar.gz" - - "./.cicd/package.sh" + command: | + buildkite-agent artifact download build.tar.gz . --step ':ubuntu: Ubuntu 18.04 - Build' && tar -xzf build.tar.gz + ./.cicd/package.sh env: IMAGE_TAG: "ubuntu-18.04-$PLATFORM_TYPE" PLATFORM_TYPE: $PLATFORM_TYPE @@ -613,9 +613,9 @@ EOF skip: ${SKIP_UBUNTU_18_04}${SKIP_PACKAGE_BUILDER}${SKIP_LINUX} - label: ":ubuntu: Ubuntu 20.04 - Package Builder" - command: - - "buildkite-agent artifact download build.tar.gz . --step ':ubuntu: Ubuntu 20.04 - Build' && tar -xzf build.tar.gz" - - "./.cicd/package.sh" + command: | + buildkite-agent artifact download build.tar.gz . --step ':ubuntu: Ubuntu 20.04 - Build' && tar -xzf build.tar.gz + ./.cicd/package.sh env: IMAGE_TAG: "ubuntu-20.04-$PLATFORM_TYPE" PLATFORM_TYPE: $PLATFORM_TYPE @@ -655,9 +655,9 @@ EOF - wait - label: ":aws: Amazon Linux 2 - Install Package" - command: - - "buildkite-agent artifact download '*.rpm' . --step ':centos: CentOS 7.7 - Package Builder' --agent-access-token \$\$BUILDKITE_AGENT_ACCESS_TOKEN" - - "./.cicd/test-package.docker.sh" + command: | + buildkite-agent artifact download '*.rpm' . --step ':centos: CentOS 7.7 - Package Builder' --agent-access-token \$\$BUILDKITE_AGENT_ACCESS_TOKEN + ./.cicd/test-package.docker.sh env: IMAGE: "amazonlinux:2" agents: @@ -666,9 +666,9 @@ EOF skip: ${SKIP_CENTOS_7_7}${SKIP_PACKAGE_BUILDER}${SKIP_LINUX} - label: ":centos: CentOS 7 - Install Package" - command: - - "buildkite-agent artifact download '*.rpm' . --step ':centos: CentOS 7.7 - Package Builder' --agent-access-token \$\$BUILDKITE_AGENT_ACCESS_TOKEN" - - "./.cicd/test-package.docker.sh" + command: | + buildkite-agent artifact download '*.rpm' . --step ':centos: CentOS 7.7 - Package Builder' --agent-access-token \$\$BUILDKITE_AGENT_ACCESS_TOKEN + ./.cicd/test-package.docker.sh env: IMAGE: "centos:7" agents: @@ -677,9 +677,9 @@ EOF skip: ${SKIP_CENTOS_7_7}${SKIP_PACKAGE_BUILDER}${SKIP_LINUX} - label: ":ubuntu: Ubuntu 16.04 - Install Package" - command: - - "buildkite-agent artifact download '*.deb' . --step ':ubuntu: Ubuntu 16.04 - Package Builder' --agent-access-token \$\$BUILDKITE_AGENT_ACCESS_TOKEN" - - "./.cicd/test-package.docker.sh" + command: | + buildkite-agent artifact download '*.deb' . --step ':ubuntu: Ubuntu 16.04 - Package Builder' --agent-access-token \$\$BUILDKITE_AGENT_ACCESS_TOKEN + ./.cicd/test-package.docker.sh env: IMAGE: "ubuntu:16.04" agents: @@ -688,9 +688,9 @@ EOF skip: ${SKIP_UBUNTU_16_04}${SKIP_PACKAGE_BUILDER}${SKIP_LINUX} - label: ":ubuntu: Ubuntu 18.04 - Install Package" - command: - - "buildkite-agent artifact download '*.deb' . --step ':ubuntu: Ubuntu 18.04 - Package Builder' --agent-access-token \$\$BUILDKITE_AGENT_ACCESS_TOKEN" - - "./.cicd/test-package.docker.sh" + command: | + buildkite-agent artifact download '*.deb' . --step ':ubuntu: Ubuntu 18.04 - Package Builder' --agent-access-token \$\$BUILDKITE_AGENT_ACCESS_TOKEN + ./.cicd/test-package.docker.sh env: IMAGE: "ubuntu:18.04" agents: @@ -699,9 +699,9 @@ EOF skip: ${SKIP_UBUNTU_18_04}${SKIP_PACKAGE_BUILDER}${SKIP_LINUX} - label: ":ubuntu: Ubuntu 20.04 - Install Package" - command: - - "buildkite-agent artifact download '*.deb' . --step ':ubuntu: Ubuntu 20.04 - Package Builder' --agent-access-token \$\$BUILDKITE_AGENT_ACCESS_TOKEN" - - "./.cicd/test-package.docker.sh" + command: | + buildkite-agent artifact download '*.deb' . --step ':ubuntu: Ubuntu 20.04 - Package Builder' --agent-access-token \$\$BUILDKITE_AGENT_ACCESS_TOKEN + ./.cicd/test-package.docker.sh env: IMAGE: "ubuntu:20.04" agents: @@ -754,9 +754,9 @@ EOF timeout: ${TIMEOUT:-5} - label: ":beer: Brew Updater" - command: - - "buildkite-agent artifact download eosio.rb . --step ':darwin: macOS 10.15 - Package Builder'" - - "buildkite-agent artifact upload eosio.rb" + command: | + buildkite-agent artifact download eosio.rb . --step ':darwin: macOS 10.15 - Package Builder' + buildkite-agent artifact upload eosio.rb agents: queue: "automation-basic-builder-fleet" timeout: ${TIMEOUT:-5} From ef855eaef6f428e0f1cd0621214c6cf259c82903 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Wed, 12 Jan 2022 16:35:11 -0500 Subject: [PATCH 38/62] Move libfunctions.sh into lib folder and give it a more obvious name --- .cicd/{libfunctions.sh => helpers/perform.sh} | 0 .cicd/test-package.anka.sh | 2 +- .cicd/test-package.docker.sh | 2 +- .cicd/test-package.run.sh | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename .cicd/{libfunctions.sh => helpers/perform.sh} (100%) diff --git a/.cicd/libfunctions.sh b/.cicd/helpers/perform.sh similarity index 100% rename from .cicd/libfunctions.sh rename to .cicd/helpers/perform.sh diff --git a/.cicd/test-package.anka.sh b/.cicd/test-package.anka.sh index 06be17a299..3f9c6b8e3d 100755 --- a/.cicd/test-package.anka.sh +++ b/.cicd/test-package.anka.sh @@ -1,7 +1,7 @@ #!/bin/bash set -euo pipefail -. "${0%/*}/libfunctions.sh" +. "${0%/*}/helpers/perform.sh" echo '--- :anka: Pretest Setup' diff --git a/.cicd/test-package.docker.sh b/.cicd/test-package.docker.sh index 9508e216ff..b96f0e9271 100755 --- a/.cicd/test-package.docker.sh +++ b/.cicd/test-package.docker.sh @@ -1,7 +1,7 @@ #!/bin/bash set -euo pipefail -. "${0%/*}/libfunctions.sh" +. "${0%/*}/helpers/perform.sh" echo '--- :docker: Pretest Setup' diff --git a/.cicd/test-package.run.sh b/.cicd/test-package.run.sh index 748be92772..fc017b3f8e 100755 --- a/.cicd/test-package.run.sh +++ b/.cicd/test-package.run.sh @@ -1,7 +1,7 @@ #!/bin/bash set -euo pipefail -. "${0%/*}/libfunctions.sh" +. "${0%/*}/helpers/perform.sh" echo '+++ :minidisc: Installing EOSIO' From 91ef80e0e919cf94b0a8eba232f6fe0bab04cc69 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Wed, 12 Jan 2022 16:36:24 -0500 Subject: [PATCH 39/62] Update documentation --- .cicd/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.cicd/README.md b/.cicd/README.md index 93ad073ae5..d2480f1742 100644 --- a/.cicd/README.md +++ b/.cicd/README.md @@ -94,12 +94,14 @@ Pipeline | Details [eosio-lrt](https://buildkite.com/EOSIO/eosio-lrt) | runs tests that need more time on merge commits [eosio-resume-from-state](https://buildkite.com/EOSIO/eosio-resume-from-state) | loads the current version of `nodeos` from state files generated by specific previous versions of `nodeos` in each [eosio](https://buildkite.com/EOSIO/eosio) build ([Documentation](https://github.com/EOSIO/auto-eks-sync-nodes/blob/master/pipelines/eosio-resume-from-state/README.md)) [eosio-sync-from-genesis](https://buildkite.com/EOSIO/eosio-sync-from-genesis) | sync the current version of `nodeos` past genesis from peers on common public chains as a smoke test, for each [eosio](https://buildkite.com/EOSIO/eosio) build +[eosio-test-stability](https://buildkite.com/EOSIO/eosio-test-stability) | prove or disprove test stability by running a test thousands of times ## See Also - Buildkite - [DevDocs](https://github.com/EOSIO/devdocs/wiki/Buildkite) - [eosio-resume-from-state Documentation](https://github.com/EOSIO/auto-eks-sync-nodes/blob/master/pipelines/eosio-resume-from-state/README.md) - [Run Your First Build](https://buildkite.com/docs/tutorials/getting-started#run-your-first-build) + - [Stability Testing](https://github.com/EOSIO/eos/blob/HEAD/.cicd/eosio-test-stability.md) - [#help-automation](https://blockone.slack.com/archives/CMTAZ9L4D) Slack Channel From 6cf783a6573fcd7a45eecb300dc2f368c9e5c231 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Wed, 12 Jan 2022 16:47:40 -0500 Subject: [PATCH 40/62] Remove remaining code or documentation for macOS 10.14 Mojave --- .cicd/README.md | 1 - .cicd/base-images.yml | 48 -------------------------------------- .cicd/generate-pipeline.sh | 13 +++-------- 3 files changed, 3 insertions(+), 59 deletions(-) diff --git a/.cicd/README.md b/.cicd/README.md index d2480f1742..13a7b42cc3 100644 --- a/.cicd/README.md +++ b/.cicd/README.md @@ -32,7 +32,6 @@ RUN_ALL_TESTS='true' # run all tests in the current build (inclu SKIP_AMAZON_LINUX_2='true|false' # skip all steps for Amazon Linux 2 SKIP_CENTOS_7_7='true|false' # skip all steps for Centos 7.7 SKIP_CENTOS_8='true|false' # skip all steps for Centos 8 -SKIP_MACOS_10_14='true|false' # skip all steps for MacOS 10.14 SKIP_MACOS_10_15='true|false' # skip all steps for MacOS 10.15 SKIP_MACOS_11='true|false' # skip all steps for MacOS 11 SKIP_UBUNTU_16_04='true|false' # skip all steps for Ubuntu 16.04 diff --git a/.cicd/base-images.yml b/.cicd/base-images.yml index 1d6bb66460..dc3280ecc4 100644 --- a/.cicd/base-images.yml +++ b/.cicd/base-images.yml @@ -25,30 +25,6 @@ steps: timeout: 180 skip: ${SKIP_CENTOS_7_7}${SKIP_LINUX} - - label: ":darwin: macOS 10.14 - Base Image Pinned" - command: - - "git clone git@github.com:EOSIO/eos.git eos && cd eos && git checkout -f $BUILDKITE_BRANCH" - - "cd eos && ./.cicd/platforms/pinned/macos-10.14-pinned.sh" - plugins: - - EOSIO/anka#v0.6.1: - debug: true - vm-name: "10.14.6_6C_14G_80G" - no-volume: true - always-pull: true - wait-network: true - pre-execute-sleep: 5 - pre-execute-ping-sleep: github.com - vm-registry-tag: "clean::cicd::git-ssh::nas::brew::buildkite-agent" - failover-registries: - - "registry_1" - - "registry_2" - inherit-environment-vars: true - - EOSIO/skip-checkout#v0.1.1: - cd: ~ - agents: "queue=mac-anka-node-fleet" - timeout: 180 - skip: ${SKIP_MACOS_10_14}${SKIP_MAC} - - label: ":darwin: macOS 10.15 - Base Image Pinned" command: - "git clone git@github.com:EOSIO/eos.git eos && cd eos && git checkout -f $BUILDKITE_BRANCH" @@ -121,30 +97,6 @@ steps: timeout: 180 skip: ${SKIP_CENTOS_7_7}${SKIP_LINUX} - - label: ":darwin: macOS 10.14 - Base Image Unpinned" - command: - - "git clone git@github.com:EOSIO/eos.git eos && cd eos && git checkout -f $BUILDKITE_BRANCH" - - "cd eos && ./.cicd/platforms/unpinned/macos-10.14-unpinned.sh" - plugins: - - EOSIO/anka#v0.6.1: - debug: true - vm-name: "10.14.6_6C_14G_80G" - no-volume: true - always-pull: true - wait-network: true - pre-execute-sleep: 5 - pre-execute-ping-sleep: github.com - vm-registry-tag: "clean::cicd::git-ssh::nas::brew::buildkite-agent" - failover-registries: - - "registry_1" - - "registry_2" - inherit-environment-vars: true - - EOSIO/skip-checkout#v0.1.1: - cd: ~ - agents: "queue=mac-anka-node-fleet" - timeout: 180 - skip: ${SKIP_MACOS_10_14}${SKIP_MAC} - - label: ":darwin: macOS 10.15 - Base Image Unpinned" command: - "git clone git@github.com:EOSIO/eos.git eos && cd eos && git checkout -f $BUILDKITE_BRANCH" diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index e6845500fe..e004a7a1dc 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -48,13 +48,8 @@ for FILE in $(ls "$CICD_DIR/platforms/$PLATFORM_TYPE"); do export SKIP_PACKAGE_BUILDER=${SKIP_PACKAGE_BUILDER:-true} fi export FILE_NAME="$(echo "$FILE" | awk '{split($0,a,/\.(d|s)/); print a[1] }')" - # macos-10.14 + # macos-10.15 # ubuntu-16.04 - # skip Mojave if it's anything but the post-merge build - if [[ "$FILE_NAME" =~ 'macos-10.14' && "$SKIP_MACOS_10_14" != 'false' && "$RUN_ALL_TESTS" != 'true' && ( "$BUILDKITE_SOURCE" != 'webhook' || "$BUILDKITE_PULL_REQUEST" != 'false' || ! "$BUILDKITE_MESSAGE" =~ 'Merge pull request' ) ]]; then - export SKIP_MACOS_10_14='true' - continue - fi export PLATFORM_NAME="$(echo $FILE_NAME | cut -d- -f1 | sed 's/os/OS/g')" # macOS # ubuntu @@ -68,7 +63,7 @@ for FILE in $(ls "$CICD_DIR/platforms/$PLATFORM_TYPE"); do # _14 # _04 export VERSION_FULL="$(echo $FILE_NAME | cut -d- -f2)" - # 10.14 + # 10.15 # 16.04 OLDIFS=$IFS IFS='_' @@ -83,9 +78,7 @@ for FILE in $(ls "$CICD_DIR/platforms/$PLATFORM_TYPE"); do export PLATFORM_SKIP_VAR="SKIP_${PLATFORM_NAME_UPCASE}_${VERSION_MAJOR}${VERSION_MINOR}" # Anka Template and Tags export ANKA_TAG_BASE='clean::cicd::git-ssh::nas::brew::buildkite-agent' - if [[ $FILE_NAME =~ 'macos-10.14' ]]; then - export ANKA_TEMPLATE_NAME='10.14.6_6C_14G_80G' - elif [[ $FILE_NAME =~ 'macos-10.15' ]]; then + if [[ $FILE_NAME =~ 'macos-10.15' ]]; then export ANKA_TEMPLATE_NAME='10.15.5_6C_14G_80G' else # Linux export ANKA_TAG_BASE='' From 38b31710f677e0257542c91ea2635db1a9f564d8 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Wed, 12 Jan 2022 16:51:43 -0500 Subject: [PATCH 41/62] Delete Ubuntu 16.04 --- .cicd/README.md | 1 - .cicd/generate-pipeline.sh | 15 ++------------- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/.cicd/README.md b/.cicd/README.md index 13a7b42cc3..66c306ec31 100644 --- a/.cicd/README.md +++ b/.cicd/README.md @@ -34,7 +34,6 @@ SKIP_CENTOS_7_7='true|false' # skip all steps for Centos 7.7 SKIP_CENTOS_8='true|false' # skip all steps for Centos 8 SKIP_MACOS_10_15='true|false' # skip all steps for MacOS 10.15 SKIP_MACOS_11='true|false' # skip all steps for MacOS 11 -SKIP_UBUNTU_16_04='true|false' # skip all steps for Ubuntu 16.04 SKIP_UBUNTU_18_04='true|false' # skip all steps for Ubuntu 18.04 SKIP_UBUNTU_20_04='true|false' # skip all steps for Ubuntu 20.04 ``` diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index e004a7a1dc..27f0432c6b 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -49,7 +49,7 @@ for FILE in $(ls "$CICD_DIR/platforms/$PLATFORM_TYPE"); do fi export FILE_NAME="$(echo "$FILE" | awk '{split($0,a,/\.(d|s)/); print a[1] }')" # macos-10.15 - # ubuntu-16.04 + # ubuntu-20.04 export PLATFORM_NAME="$(echo $FILE_NAME | cut -d- -f1 | sed 's/os/OS/g')" # macOS # ubuntu @@ -64,7 +64,7 @@ for FILE in $(ls "$CICD_DIR/platforms/$PLATFORM_TYPE"); do # _04 export VERSION_FULL="$(echo $FILE_NAME | cut -d- -f2)" # 10.15 - # 16.04 + # 20.04 OLDIFS=$IFS IFS='_' set $PLATFORM_NAME @@ -669,17 +669,6 @@ EOF timeout: ${TIMEOUT:-10} skip: ${SKIP_CENTOS_7_7}${SKIP_PACKAGE_BUILDER}${SKIP_LINUX} - - label: ":ubuntu: Ubuntu 16.04 - Install Package" - command: | - buildkite-agent artifact download '*.deb' . --step ':ubuntu: Ubuntu 16.04 - Package Builder' --agent-access-token \$\$BUILDKITE_AGENT_ACCESS_TOKEN - ./.cicd/test-package.docker.sh - env: - IMAGE: "ubuntu:16.04" - agents: - queue: "$BUILDKITE_TEST_AGENT_QUEUE" - timeout: ${TIMEOUT:-10} - skip: ${SKIP_UBUNTU_16_04}${SKIP_PACKAGE_BUILDER}${SKIP_LINUX} - - label: ":ubuntu: Ubuntu 18.04 - Install Package" command: | buildkite-agent artifact download '*.deb' . --step ':ubuntu: Ubuntu 18.04 - Package Builder' --agent-access-token \$\$BUILDKITE_AGENT_ACCESS_TOKEN From 70ea87d81e62fe16ab4bdfd8e31eb6e47636fa34 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Wed, 12 Jan 2022 16:52:49 -0500 Subject: [PATCH 42/62] Remove reference to CentOS 8 --- .cicd/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/.cicd/README.md b/.cicd/README.md index 66c306ec31..6f71b401c8 100644 --- a/.cicd/README.md +++ b/.cicd/README.md @@ -31,7 +31,6 @@ Configure which operating systems are built, tested, and packaged: RUN_ALL_TESTS='true' # run all tests in the current build (including LRTs, overridden by SKIP* variables) SKIP_AMAZON_LINUX_2='true|false' # skip all steps for Amazon Linux 2 SKIP_CENTOS_7_7='true|false' # skip all steps for Centos 7.7 -SKIP_CENTOS_8='true|false' # skip all steps for Centos 8 SKIP_MACOS_10_15='true|false' # skip all steps for MacOS 10.15 SKIP_MACOS_11='true|false' # skip all steps for MacOS 11 SKIP_UBUNTU_18_04='true|false' # skip all steps for Ubuntu 18.04 From 8b6a0a419b7bad44befc305f8e4c2b9a9822a063 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Wed, 12 Jan 2022 16:54:23 -0500 Subject: [PATCH 43/62] Document FORCE_BASE_IMAGE flag --- .cicd/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.cicd/README.md b/.cicd/README.md index 6f71b401c8..0d78bf82ba 100644 --- a/.cicd/README.md +++ b/.cicd/README.md @@ -51,6 +51,7 @@ SKIP_PACKAGE_BUILDER='true|false' # skip all packaging steps Configure how the steps are executed: ```bash +FORCE_BASE_IMAGE='true|false' # force the CI system to build base images from scratch, but do not overwrite any existing copies in the cloud PINNED='true|false' # use specific versions of dependencies instead of whatever version is provided by default on a given platform TIMEOUT='##' # set timeout in minutes for all steps ``` From 684328d6c0543191391d3e51c18a8fd80760fa73 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Wed, 12 Jan 2022 16:56:28 -0500 Subject: [PATCH 44/62] Document the OVERWRITE_BASE_IMAGE flag --- .cicd/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/.cicd/README.md b/.cicd/README.md index 0d78bf82ba..1b800e1b9b 100644 --- a/.cicd/README.md +++ b/.cicd/README.md @@ -52,6 +52,7 @@ SKIP_PACKAGE_BUILDER='true|false' # skip all packaging steps Configure how the steps are executed: ```bash FORCE_BASE_IMAGE='true|false' # force the CI system to build base images from scratch, but do not overwrite any existing copies in the cloud +OVERWRITE_BASE_IMAGE='true|false' # force the CI system to build base images from scratch and overwrite the copies in the cloud, if successful PINNED='true|false' # use specific versions of dependencies instead of whatever version is provided by default on a given platform TIMEOUT='##' # set timeout in minutes for all steps ``` From 7045cebd275663a13254e0117093676808ce05a8 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Wed, 12 Jan 2022 17:20:52 -0500 Subject: [PATCH 45/62] Remove ssh-keyscan on github.com, which is done fleet-wide in a more secure manner --- .cicd/generate-pipeline.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 27f0432c6b..77a19b0030 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -557,7 +557,6 @@ cat <> ~/.ssh/known_hosts git clone \$BUILDKITE_REPO . $GIT_FETCH git checkout -f \$BUILDKITE_COMMIT echo '+++ :compression: Extracting Test Metrics Code' From 1562c7c0681ffc489c96215679307ad142a8c26b Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Wed, 12 Jan 2022 17:24:45 -0500 Subject: [PATCH 46/62] Fold documentation on Buildkite but not GitHub --- .cicd/README.md | 5 ++--- .cicd/eosio-test-stability.md | 5 ++--- .cicd/generate-pipeline.sh | 4 ++-- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.cicd/README.md b/.cicd/README.md index 1b800e1b9b..edb46d2f10 100644 --- a/.cicd/README.md +++ b/.cicd/README.md @@ -3,8 +3,7 @@ The [eosio](https://buildkite.com/EOSIO/eosio) and [eosio-build-unpinned](https: The [eosio](https://buildkite.com/EOSIO/eosio) pipeline further triggers the [eosio-sync-from-genesis](https://buildkite.com/EOSIO/eosio-sync-from-genesis) and [eosio-resume-from-state](https://buildkite.com/EOSIO/eosio-resume-from-state) pipelines on each build, and the the [eosio-lrt](https://buildkite.com/EOSIO/eosio-lrt) pipeline on merge commits. Each of these pipelines are described in more detail below and in their respective READMEs. -
-See More + ## Index 1. [Configuration](README.md#configuration) @@ -103,4 +102,4 @@ Pipeline | Details - [Stability Testing](https://github.com/EOSIO/eos/blob/HEAD/.cicd/eosio-test-stability.md) - [#help-automation](https://blockone.slack.com/archives/CMTAZ9L4D) Slack Channel -
+ diff --git a/.cicd/eosio-test-stability.md b/.cicd/eosio-test-stability.md index b2ae0c99f8..798e54d3da 100644 --- a/.cicd/eosio-test-stability.md +++ b/.cicd/eosio-test-stability.md @@ -1,8 +1,7 @@ # Stability Testing Stability testing of EOSIO unit and integration tests is done in the [eosio-test-stability](https://buildkite.com/EOSIO/eosio-test-stability) pipeline. It will take thousands of runs of any given test to identify it as "stable" or "unstable". Runs should be split evenly across "pinned" (fixed dependency version) and "unpinned" (default dependency version) builds because, sometimes, test instability is only expressed in one of these environments. Finally, stability testing should be performed on the Linux fleet first because this fleet is effectively infinite. Once stability is demonstrated on Linux, testing can be performed on the finite macOS Anka fleet. -
-See More + ## Index 1. [Configuration](eosio-test-stability.md#configuration) @@ -81,4 +80,4 @@ If these runs are against `eos:develop` and `develop` has five supported operati - [Run Your First Build](https://buildkite.com/docs/tutorials/getting-started#run-your-first-build) - [#help-automation](https://blockone.slack.com/archives/CMTAZ9L4D) Slack Channel -
+ diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 77a19b0030..1a2a9d94b1 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -12,10 +12,10 @@ export DOCS_URL="https://github.com/EOSIO/eos/blob/$(git rev-parse HEAD)/.cicd" export RETRY="$([[ "$BUILDKITE" == 'true' ]] && buildkite-agent meta-data get pipeline-upload-retries --default '0' || echo "${RETRY:-0}")" if [[ "$BUILDKITE" == 'true' && "$RETRY" == '0' ]]; then echo "This documentation is also available on [GitHub]($DOCS_URL/README.md)." | buildkite-agent annotate --append --style 'info' --context 'documentation' - cat .cicd/README.md | buildkite-agent annotate --append --style 'info' --context 'documentation' + cat .cicd/README.md | sed 's__
\nSee More_' | sed 's__
_' | buildkite-agent annotate --append --style 'info' --context 'documentation' if [[ "$BUILDKITE_PIPELINE_SLUG" == 'eosio-test-stability' ]]; then echo "This documentation is also available on [GitHub]($DOCS_URL/eosio-test-stability.md)." | buildkite-agent annotate --append --style 'info' --context 'test-stability' - cat .cicd/eosio-test-stability.md | buildkite-agent annotate --append --style 'info' --context 'test-stability' + cat .cicd/eosio-test-stability.md | sed 's__
\nSee More_' | sed 's__
_' | buildkite-agent annotate --append --style 'info' --context 'test-stability' fi fi [[ "$BUILDKITE" == 'true' ]] && buildkite-agent meta-data set pipeline-upload-retries "$(( $RETRY + 1 ))" From 6a6fc4e13a4f934d2fad2c87d1924b6f4caabeab Mon Sep 17 00:00:00 2001 From: William Blevins Date: Thu, 13 Jan 2022 16:25:56 -0500 Subject: [PATCH 47/62] Update package install step timeout to 10 minutes. --- .cicd/generate-pipeline.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 1a2a9d94b1..ef62fee2ff 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -654,7 +654,7 @@ EOF IMAGE: "amazonlinux:2" agents: queue: "$BUILDKITE_TEST_AGENT_QUEUE" - timeout: ${TIMEOUT:-10} + timeout: ${TIMEOUT:-20} skip: ${SKIP_CENTOS_7_7}${SKIP_PACKAGE_BUILDER}${SKIP_LINUX} - label: ":centos: CentOS 7 - Install Package" @@ -665,7 +665,7 @@ EOF IMAGE: "centos:7" agents: queue: "$BUILDKITE_TEST_AGENT_QUEUE" - timeout: ${TIMEOUT:-10} + timeout: ${TIMEOUT:-20} skip: ${SKIP_CENTOS_7_7}${SKIP_PACKAGE_BUILDER}${SKIP_LINUX} - label: ":ubuntu: Ubuntu 18.04 - Install Package" @@ -676,7 +676,7 @@ EOF IMAGE: "ubuntu:18.04" agents: queue: "$BUILDKITE_TEST_AGENT_QUEUE" - timeout: ${TIMEOUT:-10} + timeout: ${TIMEOUT:-20} skip: ${SKIP_UBUNTU_18_04}${SKIP_PACKAGE_BUILDER}${SKIP_LINUX} - label: ":ubuntu: Ubuntu 20.04 - Install Package" @@ -687,7 +687,7 @@ EOF IMAGE: "ubuntu:20.04" agents: queue: "$BUILDKITE_TEST_AGENT_QUEUE" - timeout: ${TIMEOUT:-10} + timeout: ${TIMEOUT:-20} skip: ${SKIP_UBUNTU_20_04}${SKIP_PACKAGE_BUILDER}${SKIP_LINUX} - label: ":darwin: macOS 10.15 - Install Package" @@ -713,7 +713,7 @@ EOF cd: ~ agents: - "queue=mac-anka-node-fleet" - timeout: ${TIMEOUT:-10} + timeout: ${TIMEOUT:-20} skip: ${SKIP_MACOS_10_15}${SKIP_PACKAGE_BUILDER}${SKIP_MAC} - wait From cf14a6eedcd88692cb454150dedcadd1e1c04543 Mon Sep 17 00:00:00 2001 From: William Blevins Date: Thu, 13 Jan 2022 17:34:55 -0500 Subject: [PATCH 48/62] Update package build step timeout to 20 minutes also. --- .cicd/generate-pipeline.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index ef62fee2ff..5ce0cb0302 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -587,7 +587,7 @@ EOF PKGTYPE: "rpm" agents: queue: "$BUILDKITE_TEST_AGENT_QUEUE" - timeout: ${TIMEOUT:-10} + timeout: ${TIMEOUT:-20} skip: ${SKIP_CENTOS_7_7}${SKIP_PACKAGE_BUILDER}${SKIP_LINUX} - label: ":ubuntu: Ubuntu 18.04 - Package Builder" @@ -601,7 +601,7 @@ EOF PKGTYPE: "deb" agents: queue: "$BUILDKITE_TEST_AGENT_QUEUE" - timeout: ${TIMEOUT:-10} + timeout: ${TIMEOUT:-20} skip: ${SKIP_UBUNTU_18_04}${SKIP_PACKAGE_BUILDER}${SKIP_LINUX} - label: ":ubuntu: Ubuntu 20.04 - Package Builder" @@ -615,7 +615,7 @@ EOF PKGTYPE: "deb" agents: queue: "$BUILDKITE_TEST_AGENT_QUEUE" - timeout: ${TIMEOUT:-10} + timeout: ${TIMEOUT:-20} skip: ${SKIP_UBUNTU_20_04}${SKIP_PACKAGE_BUILDER}${SKIP_LINUX} - label: ":darwin: macOS 10.15 - Package Builder" From 2f46bec9d58719a6c2a05a1d4172df8d26a41256 Mon Sep 17 00:00:00 2001 From: William Blevins Date: Fri, 14 Jan 2022 10:04:14 -0500 Subject: [PATCH 49/62] Disable resume-from-state with ticket number to fix it. --- .cicd/generate-pipeline.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 5ce0cb0302..4632dcff80 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -531,7 +531,7 @@ EOF EOF fi # trigger eosio-resume-from-state for every build - if [[ -z "$TEST" && "$BUILDKITE_PIPELINE_SLUG" == 'eosio' && -z "${SKIP_INSTALL}${SKIP_LINUX}${SKIP_DOCKER}${SKIP_SYNC_TESTS}" ]]; then + if [[ -z "$TEST" && "$BUILDKITE_PIPELINE_SLUG" == 'eosio' && -z "${SKIP_INSTALL}${SKIP_LINUX}${SKIP_DOCKER}${SKIP_SYNC_TESTS}BLU-30414" ]]; then cat < Date: Fri, 14 Jan 2022 10:15:46 -0500 Subject: [PATCH 50/62] Try @heifner's suggestion --- .cicd/generate-pipeline.sh | 2 +- pipeline.jsonc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 4632dcff80..5ce0cb0302 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -531,7 +531,7 @@ EOF EOF fi # trigger eosio-resume-from-state for every build - if [[ -z "$TEST" && "$BUILDKITE_PIPELINE_SLUG" == 'eosio' && -z "${SKIP_INSTALL}${SKIP_LINUX}${SKIP_DOCKER}${SKIP_SYNC_TESTS}BLU-30414" ]]; then + if [[ -z "$TEST" && "$BUILDKITE_PIPELINE_SLUG" == 'eosio' && -z "${SKIP_INSTALL}${SKIP_LINUX}${SKIP_DOCKER}${SKIP_SYNC_TESTS}" ]]; then cat < Date: Fri, 14 Jan 2022 12:44:34 -0500 Subject: [PATCH 51/62] Temporarily disable eosio-resume-from-state tests with a reference to the JIRA bug --- .cicd/generate-pipeline.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 5ce0cb0302..6c897ccc57 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -546,6 +546,7 @@ EOF SKIP_KYLIN: "${SKIP_KYLIN}" SKIP_MAIN: "${SKIP_MAIN}" TIMEOUT: "${TIMEOUT}" + skip: "See BLU-30414" EOF fi From 849142dec6806283fd9026e7d2dda7c714368f94 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Tue, 11 Jan 2022 18:25:08 -0500 Subject: [PATCH 52/62] Delete GitLab CI --- .gitlab-ci.yml | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 .gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index a51b6e9564..0000000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,7 +0,0 @@ -image: docker:latest - -build: - script: - - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY - - docker build -t $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA -f Docker/Dockerfile . - - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA From aceeed09b856d6671bd4643670a459b6ca320eb0 Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Tue, 11 Jan 2022 19:58:11 -0500 Subject: [PATCH 53/62] Update .gitignore --- .gitignore | 152 +++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 113 insertions(+), 39 deletions(-) diff --git a/.gitignore b/.gitignore index 7c0785ba35..06edcb806e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,33 +1,114 @@ -*.a -*.sw* -*.dylib -*.ll -*.bc -*.wast -*.wast.hpp -*.s -*.dot -*.abi.hpp +# archives +*.7z +*.7zip +*.rar +*.tar.gz +*.tgz +*.zip + +# binary files +*.bin + +# build artifacts +**/build +**/.dist +.dist.tar.gz + +# cmake *.cmake -!.cicd !CMakeModules/*.cmake -*.ninja -\#* -\.#* CMakeCache.txt CMakeFiles cmake_install.cmake cmake-build-debug/ cmake-build-release/ Makefile -compile_commands.json -moc_* -*.moc -.clangd/ +# data files +*.csv +/*.json +*.tsv +*.xls +*.xlsx + +# doxygen +doxygen +eos.doxygen + +# editor files +*.sw? +.vscode +.idea/ + +# environment variable files +.env + +# eslint +.eslintcache + +# istanbul/jest +**/coverage +junit.xml + +# keys and certs +*.cer +*.crt +*.key +*.pem + +# logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +*.txt + +# macOS finder cache +**/*.DS_Store + +# nodeJS + TypeScript +**/build +**/node_modules +.node_repl_history +**/.npm +*.tgz +**/typings + +# nodeos genesis.json hardfork.hpp +wallet.json +witness_node_data_dir +*.wallet +# pipeline-upload.sh artifacts +/pipeline.yml +/pipeline.yaml + +# python +*.pyc +*.pyo + +# swagger +!*.swagger.* + +# terraform +plan.out +**/.terraform +*.tfstate +*.tfstate.* +*.terraform.lock.hcl +**/terraform/*.out + +# wasm +*.wast +*.wast.hpp + +# yarn +.yarn-integrity + +### COMPONENTS ### libraries/egenesis/egenesis_brief.cpp libraries/egenesis/egenesis_full.cpp libraries/egenesis/embed_genesis @@ -36,7 +117,6 @@ libraries/types/types_test libraries/fc/test/crypto/test_cypher_suites libraries/testing/chain_tester - libraries/wallet/Doxyfile libraries/wallet/api_documentation.cpp libraries/wallet/doxygen @@ -63,30 +143,24 @@ scripts/tn_init.sh tests/plugin_test unittests/unit_test -doxygen -eos.doxygen - -wallet.json -witness_node_data_dir - -*.wallet - -*.pyc -*.pyo +### OTHER ### +*.a +*.dylib +*.ll +*.bc +*.s +*.dot +*.abi.hpp +*.ninja +\#* +\.#* +moc_* +*.moc +.clangd/ Testing/* -build.tar.gz -build/* build-debug/* -node_modules -package-lock.json -etc/eosio/node_* -var/lib/node_* -.vscode -.idea/ *.iws -.DS_Store -!*.swagger.* .cache From 3c2905ffdae2c3e080761db2ac89288997d6885d Mon Sep 17 00:00:00 2001 From: Zach Butler Date: Mon, 17 Jan 2022 19:49:18 -0500 Subject: [PATCH 54/62] Disable macOS 10.15 Catalina package steps - see BLU-30502 --- .cicd/generate-pipeline.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.cicd/generate-pipeline.sh b/.cicd/generate-pipeline.sh index 6c897ccc57..3e6c08bf3b 100755 --- a/.cicd/generate-pipeline.sh +++ b/.cicd/generate-pipeline.sh @@ -643,7 +643,7 @@ EOF agents: - "queue=mac-anka-node-fleet" timeout: ${TIMEOUT:-30} - skip: ${SKIP_MACOS_10_15}${SKIP_PACKAGE_BUILDER}${SKIP_MAC} + skip: "See BLU-30502" - wait @@ -715,7 +715,7 @@ EOF agents: - "queue=mac-anka-node-fleet" timeout: ${TIMEOUT:-20} - skip: ${SKIP_MACOS_10_15}${SKIP_PACKAGE_BUILDER}${SKIP_MAC} + skip: "See BLU-30502" - wait @@ -742,7 +742,7 @@ EOF agents: queue: "automation-basic-builder-fleet" timeout: ${TIMEOUT:-5} - skip: ${SKIP_PACKAGE_BUILDER}${SKIP_MAC}${SKIP_MACOS_10_15} + skip: "See BLU-30502" - label: ":docker: :ubuntu: Docker - Build 18.04 Docker Image" command: "./.cicd/create-docker-from-binary.sh" From f9cf973d49245691286322036b90238f0c28d5b7 Mon Sep 17 00:00:00 2001 From: Dmytro Sydorchenko Date: Fri, 7 Jan 2022 17:22:08 -0500 Subject: [PATCH 55/62] rocksdb integration for nodeos removed --- libraries/chain/CMakeLists.txt | 7 +- .../backing_store/db_context_rocksdb.cpp | 764 ----------- .../backing_store/db_key_value_any_lookup.cpp | 128 -- .../backing_store/db_key_value_format.cpp | 298 ----- .../chain/backing_store/tests/CMakeLists.txt | 11 - libraries/chain/backing_store/tests/README | 23 - .../backing_store/tests/benchmark_kv.cpp | 599 --------- .../backing_store/tests/benchmark_kv_batch.py | 30 - .../tests/benchmark_kv_single.py | 51 - .../backing_store/tests/test_kv_rocksdb.cpp | 682 ---------- libraries/chain/combined_database.cpp | 686 ---------- libraries/chain/controller.cpp | 14 +- .../include/eosio/chain/backing_store.hpp | 2 - .../chain/backing_store/chain_kv_payer.hpp | 52 - .../eosio/chain/backing_store/db_combined.hpp | 646 ---------- .../eosio/chain/backing_store/db_context.hpp | 4 - .../backing_store/db_key_value_any_lookup.hpp | 78 -- .../backing_store/db_key_value_format.hpp | 581 --------- .../backing_store/db_key_value_iter_store.hpp | 160 --- .../backing_store/db_key_value_sec_lookup.hpp | 638 --------- .../eosio/chain/backing_store/kv_context.hpp | 6 - .../backing_store/kv_context_chainbase.hpp | 1 - .../backing_store/kv_context_rocksdb.hpp | 496 ------- .../chain/include/eosio/chain/config.hpp | 5 - .../chain/include/eosio/chain/controller.hpp | 10 +- .../chain/include/eosio/chain/exceptions.hpp | 4 - ...{combined_database.hpp => kv_database.hpp} | 55 +- .../eosio/chain/transaction_context.hpp | 4 +- libraries/chain/kv_database.cpp | 357 ++++++ .../include/b1/session/rocks_session.hpp | 566 -------- .../chain_kv/include/b1/session/session.hpp | 2 +- .../chain_kv/unit_tests/data_store_tests.hpp | 643 ---------- .../unit_tests/rocks_session_tests.cpp | 113 -- .../chain_kv/unit_tests/session_tests.cpp | 463 ------- .../unit_tests/session_undo_stack_tests.cpp | 96 -- libraries/state_history/create_deltas.cpp | 63 +- .../eosio/state_history/create_deltas.hpp | 4 +- .../include/eosio/state_history/log.hpp | 4 +- .../eosio/state_history/rocksdb_receiver.hpp | 186 --- .../eosio/state_history/serialization.hpp | 49 - libraries/state_history/log.cpp | 2 +- .../testing/backing_store_tester_macros.hpp | 2 - .../testing/include/eosio/testing/tester.hpp | 18 - libraries/testing/tester.cpp | 4 +- plugins/chain_plugin/chain_plugin.cpp | 273 +--- .../eosio/chain_plugin/chain_plugin.hpp | 351 ++--- .../eosio/trace_api/store_provider.hpp | 2 +- tests/amqp_tests.py | 4 +- tests/eosvmoc_tests/codecache_tests.cpp | 11 +- tests/get_table_seckey_tests.cpp | 2 +- tests/get_table_tests.cpp | 2 +- tests/nodeos_read_terminate_at_block_test.py | 2 +- tests/nodeos_run_test.py | 6 +- unittests/api_tests.cpp | 7 +- unittests/db_to_kv_tests.cpp | 1139 ----------------- unittests/kv_tests.cpp | 88 -- unittests/snapshot_tests.cpp | 22 +- unittests/state_history_tests.cpp | 26 +- 58 files changed, 554 insertions(+), 9988 deletions(-) delete mode 100644 libraries/chain/backing_store/db_context_rocksdb.cpp delete mode 100644 libraries/chain/backing_store/db_key_value_any_lookup.cpp delete mode 100644 libraries/chain/backing_store/db_key_value_format.cpp delete mode 100644 libraries/chain/backing_store/tests/CMakeLists.txt delete mode 100644 libraries/chain/backing_store/tests/README delete mode 100644 libraries/chain/backing_store/tests/benchmark_kv.cpp delete mode 100755 libraries/chain/backing_store/tests/benchmark_kv_batch.py delete mode 100755 libraries/chain/backing_store/tests/benchmark_kv_single.py delete mode 100644 libraries/chain/backing_store/tests/test_kv_rocksdb.cpp delete mode 100644 libraries/chain/combined_database.cpp delete mode 100644 libraries/chain/include/eosio/chain/backing_store/chain_kv_payer.hpp delete mode 100644 libraries/chain/include/eosio/chain/backing_store/db_combined.hpp delete mode 100644 libraries/chain/include/eosio/chain/backing_store/db_key_value_any_lookup.hpp delete mode 100644 libraries/chain/include/eosio/chain/backing_store/db_key_value_format.hpp delete mode 100644 libraries/chain/include/eosio/chain/backing_store/db_key_value_iter_store.hpp delete mode 100644 libraries/chain/include/eosio/chain/backing_store/db_key_value_sec_lookup.hpp delete mode 100644 libraries/chain/include/eosio/chain/backing_store/kv_context_rocksdb.hpp rename libraries/chain/include/eosio/chain/{combined_database.hpp => kv_database.hpp} (68%) create mode 100644 libraries/chain/kv_database.cpp delete mode 100644 libraries/chain_kv/include/b1/session/rocks_session.hpp delete mode 100644 libraries/chain_kv/unit_tests/data_store_tests.hpp delete mode 100644 libraries/chain_kv/unit_tests/rocks_session_tests.cpp delete mode 100644 libraries/chain_kv/unit_tests/session_tests.cpp delete mode 100644 libraries/chain_kv/unit_tests/session_undo_stack_tests.cpp delete mode 100644 libraries/state_history/include/eosio/state_history/rocksdb_receiver.hpp delete mode 100644 unittests/db_to_kv_tests.cpp diff --git a/libraries/chain/CMakeLists.txt b/libraries/chain/CMakeLists.txt index ce7bba00cc..79dda80c34 100644 --- a/libraries/chain/CMakeLists.txt +++ b/libraries/chain/CMakeLists.txt @@ -93,7 +93,7 @@ add_library( eosio_chain block_header_state.cpp block_state.cpp fork_database.cpp - combined_database.cpp + kv_database.cpp controller.cpp authorization_manager.cpp resource_limits.cpp @@ -135,9 +135,6 @@ add_library( eosio_chain backing_store/kv_context.cpp backing_store/db_context.cpp backing_store/db_context_chainbase.cpp - backing_store/db_context_rocksdb.cpp - backing_store/db_key_value_format.cpp - backing_store/db_key_value_any_lookup.cpp ${PLATFORM_TIMER_IMPL} ${HEADERS} ) @@ -190,8 +187,6 @@ if(EOSIO_REQUIRE_CHAIN_ID) target_compile_definitions(eosio_chain PUBLIC "EOSIO_REQUIRE_CHAIN_ID=\"${EOSIO_REQUIRE_CHAIN_ID}\"") endif() -add_subdirectory ( backing_store/tests ) - install( TARGETS eosio_chain RUNTIME DESTINATION ${CMAKE_INSTALL_FULL_BINDIR} LIBRARY DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR} diff --git a/libraries/chain/backing_store/db_context_rocksdb.cpp b/libraries/chain/backing_store/db_context_rocksdb.cpp deleted file mode 100644 index e6c04369ee..0000000000 --- a/libraries/chain/backing_store/db_context_rocksdb.cpp +++ /dev/null @@ -1,764 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace eosio { namespace chain { namespace backing_store { - - class db_context_rocksdb : public db_context { - public: - using prim_key_iter_type = secondary_key; - using session_type = eosio::session::session>; - using session_variant_type = eosio::session::session_variant; - using shared_bytes = eosio::session::shared_bytes; - - db_context_rocksdb(apply_context& context, name receiver, session_variant_type session); - - ~db_context_rocksdb() override; - - int32_t db_store_i64(uint64_t scope, uint64_t table, account_name payer, uint64_t id, const char* value , size_t value_size) override; - void db_update_i64(int32_t itr, account_name payer, const char* value , size_t value_size) override; - void db_remove_i64(int32_t itr) override; - int32_t db_get_i64(int32_t itr, char* value , size_t value_size) override; - int32_t db_next_i64(int32_t itr, uint64_t& primary) override; - int32_t db_previous_i64(int32_t itr, uint64_t& primary) override; - int32_t db_find_i64(uint64_t code, uint64_t scope, uint64_t table, uint64_t id) override; - int32_t db_lowerbound_i64(uint64_t code, uint64_t scope, uint64_t table, uint64_t id) override; - int32_t db_upperbound_i64(uint64_t code, uint64_t scope, uint64_t table, uint64_t id) override; - int32_t db_end_i64(uint64_t code, uint64_t scope, uint64_t table) override; - - /** - * interface for uint64_t secondary - */ - int32_t db_idx64_store(uint64_t scope, uint64_t table, account_name payer, uint64_t id, - const uint64_t& secondary) override; - void db_idx64_update(int32_t iterator, account_name payer, const uint64_t& secondary) override; - void db_idx64_remove(int32_t iterator) override; - int32_t db_idx64_find_secondary(uint64_t code, uint64_t scope, uint64_t table, const uint64_t& secondary, - uint64_t& primary) override; - int32_t db_idx64_find_primary(uint64_t code, uint64_t scope, uint64_t table, uint64_t& secondary, - uint64_t primary) override; - int32_t db_idx64_lowerbound(uint64_t code, uint64_t scope, uint64_t table, uint64_t& secondary, - uint64_t& primary) override; - int32_t db_idx64_upperbound(uint64_t code, uint64_t scope, uint64_t table, uint64_t& secondary, - uint64_t& primary) override; - int32_t db_idx64_end(uint64_t code, uint64_t scope, uint64_t table) override; - int32_t db_idx64_next(int32_t iterator, uint64_t& primary) override; - int32_t db_idx64_previous(int32_t iterator, uint64_t& primary) override; - - /** - * interface for uint128_t secondary - */ - int32_t db_idx128_store(uint64_t scope, uint64_t table, account_name payer, uint64_t id, - const uint128_t& secondary) override; - void db_idx128_update(int32_t iterator, account_name payer, const uint128_t& secondary) override; - void db_idx128_remove(int32_t iterator) override; - int32_t db_idx128_find_secondary(uint64_t code, uint64_t scope, uint64_t table, const uint128_t& secondary, - uint64_t& primary) override; - int32_t db_idx128_find_primary(uint64_t code, uint64_t scope, uint64_t table, uint128_t& secondary, - uint64_t primary) override; - int32_t db_idx128_lowerbound(uint64_t code, uint64_t scope, uint64_t table, uint128_t& secondary, - uint64_t& primary) override; - int32_t db_idx128_upperbound(uint64_t code, uint64_t scope, uint64_t table, uint128_t& secondary, - uint64_t& primary) override; - int32_t db_idx128_end(uint64_t code, uint64_t scope, uint64_t table) override; - int32_t db_idx128_next(int32_t iterator, uint64_t& primary) override; - int32_t db_idx128_previous(int32_t iterator, uint64_t& primary) override; - - /** - * interface for 256-bit interger secondary - */ - int32_t db_idx256_store(uint64_t scope, uint64_t table, account_name payer, uint64_t id, - const uint128_t* data) override; - void db_idx256_update(int32_t iterator, account_name payer, const uint128_t* data) override; - - void db_idx256_remove(int32_t iterator) override; - int32_t db_idx256_find_secondary(uint64_t code, uint64_t scope, uint64_t table, const uint128_t* data, - uint64_t& primary) override; - int32_t db_idx256_find_primary(uint64_t code, uint64_t scope, uint64_t table, uint128_t* data, - uint64_t primary) override; - int32_t db_idx256_lowerbound(uint64_t code, uint64_t scope, uint64_t table, uint128_t* data, - uint64_t& primary) override; - int32_t db_idx256_upperbound(uint64_t code, uint64_t scope, uint64_t table, uint128_t* data, - uint64_t& primary) override; - int32_t db_idx256_end(uint64_t code, uint64_t scope, uint64_t table) override; - int32_t db_idx256_next(int32_t iterator, uint64_t& primary) override; - int32_t db_idx256_previous(int32_t iterator, uint64_t& primary) override; - - /** - * interface for double secondary - */ - int32_t db_idx_double_store(uint64_t scope, uint64_t table, account_name payer, uint64_t id, - const float64_t& secondary) override; - void db_idx_double_update(int32_t iterator, account_name payer, const float64_t& secondary) override; - void db_idx_double_remove(int32_t iterator) override; - int32_t db_idx_double_find_secondary(uint64_t code, uint64_t scope, uint64_t table, - const float64_t& secondary, uint64_t& primary) override; - int32_t db_idx_double_find_primary(uint64_t code, uint64_t scope, uint64_t table, float64_t& secondary, - uint64_t primary) override; - int32_t db_idx_double_lowerbound(uint64_t code, uint64_t scope, uint64_t table, float64_t& secondary, - uint64_t& primary) override; - int32_t db_idx_double_upperbound(uint64_t code, uint64_t scope, uint64_t table, float64_t& secondary, - uint64_t& primary) override; - int32_t db_idx_double_end(uint64_t code, uint64_t scope, uint64_t table) override; - int32_t db_idx_double_next(int32_t iterator, uint64_t& primary) override; - int32_t db_idx_double_previous(int32_t iterator, uint64_t& primary) override; - /** - * interface for long double secondary - */ - int32_t db_idx_long_double_store(uint64_t scope, uint64_t table, account_name payer, uint64_t id, - const float128_t& secondary) override; - void db_idx_long_double_update(int32_t iterator, account_name payer, const float128_t& secondary) override; - void db_idx_long_double_remove(int32_t iterator) override; - int32_t db_idx_long_double_find_secondary(uint64_t code, uint64_t scope, uint64_t table, - const float128_t& secondary, uint64_t& primary) override; - int32_t db_idx_long_double_find_primary(uint64_t code, uint64_t scope, uint64_t table, - float128_t& secondary, uint64_t primary) override; - int32_t db_idx_long_double_lowerbound(uint64_t code, uint64_t scope, uint64_t table, float128_t& secondary, - uint64_t& primary) override; - int32_t db_idx_long_double_upperbound(uint64_t code, uint64_t scope, uint64_t table, float128_t& secondary, - uint64_t& primary) override; - int32_t db_idx_long_double_end(uint64_t code, uint64_t scope, uint64_t table) override; - int32_t db_idx_long_double_next(int32_t iterator, uint64_t& primary) override; - int32_t db_idx_long_double_previous(int32_t iterator, uint64_t& primary) override; - - // for primary keys in the iterator store, we don't care about secondary key - static prim_key_iter_type primary_key_iter(int table_ei, uint64_t key, account_name payer); - void swap(int iterator, account_name payer); - - // gets a prefix that allows for only primary key iterators - static prefix_bundle get_primary_slice_in_primaries(name code, name scope, name table, uint64_t id); - pv_bundle get_primary_key_value(name code, name scope, name table, uint64_t id); - void set_value(const shared_bytes& primary_key, const payer_payload& pp); - int32_t find_and_store_primary_key(const session_variant_type::iterator& session_iter, int32_t table_ei, - const shared_bytes& type_prefix, int32_t not_found_return, - const char* calling_func, uint64_t& found_key); - struct exact_iterator { - bool valid = false; - session_variant_type::iterator itr; - shared_bytes type_prefix; - }; - exact_iterator get_exact_iterator(name code, name scope, name table, uint64_t primary); - - enum class comp { equals, gte, gt}; - int32_t find_i64(name code, name scope, name table, uint64_t id, comp comparison); - - using uint128_t = eosio::chain::uint128_t; - using key256_t = eosio::chain::key256_t; - session_variant_type current_session; - db_key_value_iter_store primary_iter_store; - db_key_value_any_lookup primary_lookup; - db_key_value_sec_lookup sec_lookup_i64; - db_key_value_sec_lookup sec_lookup_i128; - db_key_value_sec_lookup sec_lookup_i256; - db_key_value_sec_lookup sec_lookup_double; - db_key_value_sec_lookup sec_lookup_long_double; - static constexpr uint64_t noop_secondary = 0x0; - }; // db_context_rocksdb - - db_context_rocksdb::db_context_rocksdb(apply_context& context, name receiver, session_variant_type session) - : db_context( context, receiver ), current_session{ session }, primary_lookup(*this, session), - sec_lookup_i64(*this, session), sec_lookup_i128(*this, session), sec_lookup_i256(*this, session), - sec_lookup_double(*this, session), sec_lookup_long_double(*this, session) {} - - db_context_rocksdb::~db_context_rocksdb() { - } - - int32_t db_context_rocksdb::db_store_i64(uint64_t scope, uint64_t table, account_name payer, uint64_t id, const char* value , size_t value_size) { - EOS_ASSERT( payer != account_name(), invalid_table_payer, "must specify a valid account to pay for new record" ); - const name scope_name{scope}; - const name table_name{table}; - const auto old_key_value = get_primary_key_value(receiver, scope_name, table_name, id); - - EOS_ASSERT( !old_key_value.value, db_rocksdb_invalid_operation_exception, "db_store_i64 called with pre-existing key"); - - primary_lookup.add_table_if_needed(old_key_value.full_key, payer); - - const payer_payload pp{payer, value, value_size}; - set_value(old_key_value.full_key, pp); - - const int64_t billable_size = static_cast(value_size + db_key_value_any_lookup::overhead); - - std::string event_id; - auto dm_logger = context.control.get_deep_mind_logger(); - if (dm_logger != nullptr) { - event_id = db_context::table_event(receiver, scope_name, table_name, name(id)); - } - - update_db_usage( payer, billable_size, db_context::row_add_trace(context.get_action_id(), std::move(event_id)) ); - - if (dm_logger != nullptr) { - db_context::log_row_insert(*dm_logger, context.get_action_id(), receiver, scope_name, table_name, payer, - name(id), value, value_size); - } - - const unique_table t { receiver, scope_name, table_name }; - const auto table_ei = primary_iter_store.cache_table(t); - return primary_iter_store.add(primary_key_iter(table_ei, id, payer)); - } - - void db_context_rocksdb::db_update_i64(int32_t itr, account_name payer, const char* value , size_t value_size) { - const auto& key_store = primary_iter_store.get(itr); - const auto& table_store = primary_iter_store.get_table(key_store); - EOS_ASSERT( table_store.contract == receiver, table_access_violation, "db access violation" ); - const auto old_key_value = get_primary_key_value(receiver, table_store.scope, table_store.table, key_store.primary); - - EOS_ASSERT( old_key_value.value, db_rocksdb_invalid_operation_exception, - "invariant failure in db_update_i64, iter store found to update but nothing in database"); - - // copy locally, since below the key_store memory will be changed - const auto old_payer = key_store.payer; - if (payer.empty()) { - payer = old_payer; - } - - const payer_payload old_pp{*old_key_value.value}; - const auto old_value_actual_size = old_pp.value_size; - - const payer_payload pp{payer, value, value_size}; - set_value(old_key_value.full_key, pp); - - std::string event_id; - auto dm_logger = context.control.get_deep_mind_logger(); - if (dm_logger != nullptr) { - event_id = db_context::table_event(table_store.contract, table_store.scope, table_store.table, name(key_store.primary)); - } - - const int64_t overhead = db_key_value_any_lookup::overhead; - const int64_t old_size = static_cast(old_value_actual_size + overhead); - const int64_t new_size = static_cast(value_size + overhead); - - if( old_payer != payer ) { - // refund the existing payer - update_db_usage( old_payer, -(old_size), db_context::row_update_rem_trace(context.get_action_id(), std::string(event_id)) ); - // charge the new payer - update_db_usage( payer, (new_size), db_context::row_update_add_trace(context.get_action_id(), std::move(event_id)) ); - - // swap the payer in the iterator store - swap(itr, payer); - } else if(old_size != new_size) { - // charge/refund the existing payer the difference - update_db_usage( old_payer, new_size - old_size, db_context::row_update_trace(context.get_action_id(), std::move(event_id)) ); - } - - if (dm_logger != nullptr) { - db_context::log_row_update(*dm_logger, context.get_action_id(), table_store.contract, table_store.scope, - table_store.table, old_payer, payer, name(key_store.primary), - old_key_value.value->data(), old_key_value.value->size(), value, value_size); - } - } - - void db_context_rocksdb::db_remove_i64(int32_t itr) { - const auto& key_store = primary_iter_store.get(itr); - const auto& table_store = primary_iter_store.get_table(key_store); - EOS_ASSERT( table_store.contract == receiver, table_access_violation, "db access violation" ); - const auto old_key_value = get_primary_key_value(receiver, table_store.scope, table_store.table, key_store.primary); - - EOS_ASSERT( old_key_value.value, db_rocksdb_invalid_operation_exception, - "invariant failure in db_remove_i64, iter store found to update but nothing in database"); - - const auto old_payer = key_store.payer; - - std::string event_id; - auto dm_logger = context.control.get_deep_mind_logger(); - if (dm_logger != nullptr) { - event_id = db_context::table_event(table_store.contract, table_store.scope, table_store.table, name(key_store.primary)); - } - - payer_payload pp(*old_key_value.value); - update_db_usage( old_payer, -(pp.value_size + db_key_value_any_lookup::overhead), db_context::row_rem_trace(context.get_action_id(), std::move(event_id)) ); - - if (dm_logger != nullptr) { - db_context::log_row_remove(*dm_logger, context.get_action_id(), table_store.contract, table_store.scope, - table_store.table, old_payer, name(key_store.primary), pp.value, - pp.value_size); - } - - current_session.erase(old_key_value.full_key); - primary_lookup.remove_table_if_empty(old_key_value.full_key); - - primary_iter_store.remove(itr); // don't use key_store anymore - } - - int32_t db_context_rocksdb::db_get_i64(int32_t itr, char* value , size_t value_size) { - const auto& key_store = primary_iter_store.get(itr); - const auto& table_store = primary_iter_store.get_table(key_store); - const auto old_key_value = get_primary_key_value(table_store.contract, table_store.scope, table_store.table, key_store.primary); - - EOS_ASSERT( old_key_value.value, db_rocksdb_invalid_operation_exception, - "invariant failure in db_get_i64, iter store found to update but nothing in database"); - payer_payload pp {*old_key_value.value}; - const char* const actual_value = pp.value; - const size_t actual_size = pp.value_size; - if (value_size == 0) { - return actual_size; - } - const size_t copy_size = std::min(value_size, actual_size); - memcpy( value, actual_value, copy_size ); - return copy_size; - } - - int32_t db_context_rocksdb::db_next_i64(int32_t itr, uint64_t& primary) { - if (itr < primary_iter_store.invalid_iterator()) return primary_iter_store.invalid_iterator(); // cannot increment past end iterator of table - - const auto& key_store = primary_iter_store.get(itr); - const auto& table_store = primary_iter_store.get_table(key_store); - auto exact = - get_exact_iterator(table_store.contract, table_store.scope, table_store.table, key_store.primary); - EOS_ASSERT( exact.valid, db_rocksdb_invalid_operation_exception, - "invariant failure in db_next_i64, iter store found to update but it does not exist in the database"); - auto& session_iter = exact.itr; - auto& type_prefix = exact.type_prefix; - ++session_iter; - return find_and_store_primary_key(session_iter, key_store.table_ei, type_prefix, - key_store.table_ei, __func__, primary); - } - - int32_t db_context_rocksdb::db_previous_i64(int32_t itr, uint64_t& primary) { - if( itr < primary_iter_store.invalid_iterator() ) { // is end iterator - const backing_store::unique_table* table_store = primary_iter_store.find_table_by_end_iterator(itr); - EOS_ASSERT( table_store, invalid_table_iterator, "not a valid end iterator" ); - if (current_session.begin() == current_session.end()) { - // NOTE: matching chainbase functionality, if iterator store found, but no keys in db - return primary_iter_store.invalid_iterator(); - } - - const auto primary_bounded_key = - get_primary_slice_in_primaries(table_store->contract, table_store->scope, table_store->table, - std::numeric_limits::max()); - auto session_iter = current_session.lower_bound(primary_bounded_key.full_key); - - auto past_end = [&](const auto& iter) { - return !primary_lookup.match_prefix(primary_bounded_key.prefix_key, iter); - }; - // if we are at the end of the db or past the end of this table, then step back one - if (past_end(session_iter)) { - --session_iter; - // either way, the iterator after our known table should have a primary key in this table as the previous iterator - if (past_end(session_iter)) { - // NOTE: matching chainbase functionality, if iterator store found, but no key in db for table - return primary_iter_store.invalid_iterator(); - } - } - - return find_and_store_primary_key(session_iter, primary_iter_store.get_end_iterator_by_table(*table_store), - primary_bounded_key.prefix_key, primary_iter_store.invalid_iterator(), - __func__, primary); - } - - const auto& key_store = primary_iter_store.get(itr); - const backing_store::unique_table& table_store = primary_iter_store.get_table(key_store); - const auto slice_primary_key = get_primary_slice_in_primaries(table_store.contract, table_store.scope, table_store.table, key_store.primary); - auto exact = get_exact_iterator(table_store.contract, table_store.scope, table_store.table, key_store.primary); - auto& session_iter = exact.itr; - // if we didn't find the key, or know that we cannot decrement the iterator, then return the invalid iterator handle - if (!exact.valid || session_iter == current_session.begin()) { - return primary_iter_store.invalid_iterator(); - } - - auto& type_prefix = exact.type_prefix; - --session_iter; - return find_and_store_primary_key(session_iter, key_store.table_ei, type_prefix, - primary_iter_store.invalid_iterator(), __func__, primary); - } - - int32_t db_context_rocksdb::db_find_i64(uint64_t code, uint64_t scope, uint64_t table, uint64_t id) { - return find_i64(name{code}, name{scope}, name{table}, id, comp::equals); - } - - int32_t db_context_rocksdb::db_lowerbound_i64(uint64_t code, uint64_t scope, uint64_t table, uint64_t id) { - return find_i64(name{code}, name{scope}, name{table}, id, comp::gte); - } - - int32_t db_context_rocksdb::db_upperbound_i64(uint64_t code, uint64_t scope, uint64_t table, uint64_t id) { - return find_i64(name{code}, name{scope}, name{table}, id, comp::gt); - } - - int32_t db_context_rocksdb::db_end_i64(uint64_t code, uint64_t scope, uint64_t table) { - return primary_lookup.get_end_iter(name{code}, name{scope}, name{table}, primary_iter_store); - } - - /** - * interface for uint64_t secondary - */ - int32_t db_context_rocksdb::db_idx64_store(uint64_t scope, uint64_t table, account_name payer, uint64_t id, - const uint64_t& secondary) { - return sec_lookup_i64.store(name{scope}, name{table}, payer, id, secondary); - } - - void db_context_rocksdb::db_idx64_update(int32_t iterator, account_name payer, const uint64_t& secondary) { - sec_lookup_i64.update(iterator, payer, secondary); - } - - void db_context_rocksdb::db_idx64_remove(int32_t iterator) { - sec_lookup_i64.remove(iterator); - } - - int32_t db_context_rocksdb::db_idx64_find_secondary(uint64_t code, uint64_t scope, uint64_t table, const uint64_t& secondary, - uint64_t& primary) { - return sec_lookup_i64.find_secondary(name{code}, name{scope}, name{table}, secondary, primary); - } - - int32_t db_context_rocksdb::db_idx64_find_primary(uint64_t code, uint64_t scope, uint64_t table, uint64_t& secondary, - uint64_t primary) { - return sec_lookup_i64.find_primary(name{code}, name{scope}, name{table}, secondary, primary); - } - - int32_t db_context_rocksdb::db_idx64_lowerbound(uint64_t code, uint64_t scope, uint64_t table, uint64_t& secondary, - uint64_t& primary) { - return sec_lookup_i64.lowerbound_secondary(name{code}, name{scope}, name{table}, secondary, primary); - } - - int32_t db_context_rocksdb::db_idx64_upperbound(uint64_t code, uint64_t scope, uint64_t table, uint64_t& secondary, - uint64_t& primary) { - return sec_lookup_i64.upperbound_secondary(name{code}, name{scope}, name{table}, secondary, primary); - } - - int32_t db_context_rocksdb::db_idx64_end(uint64_t code, uint64_t scope, uint64_t table) { - return sec_lookup_i64.end_secondary(name{code}, name{scope}, name{table}); - } - - int32_t db_context_rocksdb::db_idx64_next(int32_t iterator, uint64_t& primary) { - return sec_lookup_i64.next_secondary(iterator, primary); - } - - int32_t db_context_rocksdb::db_idx64_previous(int32_t iterator, uint64_t& primary) { - return sec_lookup_i64.previous_secondary(iterator, primary); - } - - /** - * interface for uint128_t secondary - */ - int32_t db_context_rocksdb::db_idx128_store(uint64_t scope, uint64_t table, account_name payer, uint64_t id, - const uint128_t& secondary) { - return sec_lookup_i128.store(name{scope}, name{table}, payer, id, secondary); - } - - void db_context_rocksdb::db_idx128_update(int32_t iterator, account_name payer, const uint128_t& secondary) { - sec_lookup_i128.update(iterator, payer, secondary); - } - - void db_context_rocksdb::db_idx128_remove(int32_t iterator) { - sec_lookup_i128.remove(iterator); - } - - int32_t db_context_rocksdb::db_idx128_find_secondary(uint64_t code, uint64_t scope, uint64_t table, const uint128_t& secondary, - uint64_t& primary) { - return sec_lookup_i128.find_secondary(name{code}, name{scope}, name{table}, secondary, primary); - } - - int32_t db_context_rocksdb::db_idx128_find_primary(uint64_t code, uint64_t scope, uint64_t table, uint128_t& secondary, - uint64_t primary) { - return sec_lookup_i128.find_primary(name{code}, name{scope}, name{table}, secondary, primary); - } - - int32_t db_context_rocksdb::db_idx128_lowerbound(uint64_t code, uint64_t scope, uint64_t table, uint128_t& secondary, - uint64_t& primary) { - return sec_lookup_i128.lowerbound_secondary(name{code}, name{scope}, name{table}, secondary, primary); - } - - int32_t db_context_rocksdb::db_idx128_upperbound(uint64_t code, uint64_t scope, uint64_t table, uint128_t& secondary, - uint64_t& primary) { - return sec_lookup_i128.upperbound_secondary(name{code}, name{scope}, name{table}, secondary, primary); - } - - int32_t db_context_rocksdb::db_idx128_end(uint64_t code, uint64_t scope, uint64_t table) { - return sec_lookup_i128.end_secondary(name{code}, name{scope}, name{table}); - } - - int32_t db_context_rocksdb::db_idx128_next(int32_t iterator, uint64_t& primary) { - return sec_lookup_i128.next_secondary(iterator, primary); - } - - int32_t db_context_rocksdb::db_idx128_previous(int32_t iterator, uint64_t& primary) { - return sec_lookup_i128.previous_secondary(iterator, primary); - } - - eosio::chain::key256_t convert(const uint128_t* data) { - eosio::chain::key256_t secondary; - std::memcpy(secondary.data(), data, sizeof(secondary)); - return secondary; - } - - void convert_back(uint128_t* data, const eosio::chain::key256_t& secondary) { - std::memcpy(data, secondary.data(), sizeof(secondary)); - } - - /** - * interface for 256-bit interger secondary - */ - int32_t db_context_rocksdb::db_idx256_store(uint64_t scope, uint64_t table, account_name payer, uint64_t id, - const uint128_t* data) { - const auto secondary = convert(data); - return sec_lookup_i256.store(name{scope}, name{table}, payer, id, secondary); - } - - void db_context_rocksdb::db_idx256_update(int32_t iterator, account_name payer, const uint128_t* data) { - const auto secondary = convert(data); - sec_lookup_i256.update(iterator, payer, secondary); - } - - void db_context_rocksdb::db_idx256_remove(int32_t iterator) { - sec_lookup_i256.remove(iterator); - } - - int32_t db_context_rocksdb::db_idx256_find_secondary(uint64_t code, uint64_t scope, uint64_t table, const uint128_t* data, - uint64_t& primary) { - const auto secondary = convert(data); - return sec_lookup_i256.find_secondary(name{code}, name{scope}, name{table}, secondary, primary); - } - - int32_t db_context_rocksdb::db_idx256_find_primary(uint64_t code, uint64_t scope, uint64_t table, uint128_t* data, - uint64_t primary) { - auto secondary = convert(data); - auto ret = sec_lookup_i256.find_primary(name{code}, name{scope}, name{table}, secondary, primary); - convert_back(data, secondary); - return ret; - } - - int32_t db_context_rocksdb::db_idx256_lowerbound(uint64_t code, uint64_t scope, uint64_t table, uint128_t* data, - uint64_t& primary) { - auto secondary = convert(data); - auto ret = sec_lookup_i256.lowerbound_secondary(name{code}, name{scope}, name{table}, secondary, primary); - convert_back(data, secondary); - return ret; - } - - int32_t db_context_rocksdb::db_idx256_upperbound(uint64_t code, uint64_t scope, uint64_t table, uint128_t* data, - uint64_t& primary) { - auto secondary = convert(data); - auto ret = sec_lookup_i256.upperbound_secondary(name{code}, name{scope}, name{table}, secondary, primary); - convert_back(data, secondary); - return ret; - } - - int32_t db_context_rocksdb::db_idx256_end(uint64_t code, uint64_t scope, uint64_t table) { - return sec_lookup_i256.end_secondary(name{code}, name{scope}, name{table}); - } - - int32_t db_context_rocksdb::db_idx256_next(int32_t iterator, uint64_t& primary) { - return sec_lookup_i256.next_secondary(iterator, primary); - } - - int32_t db_context_rocksdb::db_idx256_previous(int32_t iterator, uint64_t& primary) { - return sec_lookup_i256.previous_secondary(iterator, primary); - } - - /** - * interface for double secondary - */ - int32_t db_context_rocksdb::db_idx_double_store(uint64_t scope, uint64_t table, account_name payer, uint64_t id, - const float64_t& secondary) { - return sec_lookup_double.store(name{scope}, name{table}, payer, id, secondary); - } - - void db_context_rocksdb::db_idx_double_update(int32_t iterator, account_name payer, const float64_t& secondary) { - sec_lookup_double.update(iterator, payer, secondary); - } - - void db_context_rocksdb::db_idx_double_remove(int32_t iterator) { - sec_lookup_double.remove(iterator); - } - - int32_t db_context_rocksdb::db_idx_double_find_secondary(uint64_t code, uint64_t scope, uint64_t table, - const float64_t& secondary, uint64_t& primary) { - return sec_lookup_double.find_secondary(name{code}, name{scope}, name{table}, secondary, primary); - } - - int32_t db_context_rocksdb::db_idx_double_find_primary(uint64_t code, uint64_t scope, uint64_t table, float64_t& secondary, - uint64_t primary) { - return sec_lookup_double.find_primary(name{code}, name{scope}, name{table}, secondary, primary); - } - - int32_t db_context_rocksdb::db_idx_double_lowerbound(uint64_t code, uint64_t scope, uint64_t table, float64_t& secondary, - uint64_t& primary) { - return sec_lookup_double.lowerbound_secondary(name{code}, name{scope}, name{table}, secondary, primary); - } - - int32_t db_context_rocksdb::db_idx_double_upperbound(uint64_t code, uint64_t scope, uint64_t table, float64_t& secondary, - uint64_t& primary) { - return sec_lookup_double.upperbound_secondary(name{code}, name{scope}, name{table}, secondary, primary); - } - - int32_t db_context_rocksdb::db_idx_double_end(uint64_t code, uint64_t scope, uint64_t table) { - return sec_lookup_double.end_secondary(name{code}, name{scope}, name{table}); - } - - int32_t db_context_rocksdb::db_idx_double_next(int32_t iterator, uint64_t& primary) { - return sec_lookup_double.next_secondary(iterator, primary); - } - - int32_t db_context_rocksdb::db_idx_double_previous(int32_t iterator, uint64_t& primary) { - return sec_lookup_double.previous_secondary(iterator, primary); - } - - /** - * interface for long double secondary - */ - int32_t db_context_rocksdb::db_idx_long_double_store(uint64_t scope, uint64_t table, account_name payer, uint64_t id, - const float128_t& secondary) { - return sec_lookup_long_double.store(name{scope}, name{table}, payer, id, secondary); - } - - void db_context_rocksdb::db_idx_long_double_update(int32_t iterator, account_name payer, const float128_t& secondary) { - sec_lookup_long_double.update(iterator, payer, secondary); - } - - void db_context_rocksdb::db_idx_long_double_remove(int32_t iterator) { - sec_lookup_long_double.remove(iterator); - } - - int32_t db_context_rocksdb::db_idx_long_double_find_secondary(uint64_t code, uint64_t scope, uint64_t table, - const float128_t& secondary, uint64_t& primary) { - return sec_lookup_long_double.find_secondary(name{code}, name{scope}, name{table}, secondary, primary); - } - - int32_t db_context_rocksdb::db_idx_long_double_find_primary(uint64_t code, uint64_t scope, uint64_t table, - float128_t& secondary, uint64_t primary) { - return sec_lookup_long_double.find_primary(name{code}, name{scope}, name{table}, secondary, primary); - } - - int32_t db_context_rocksdb::db_idx_long_double_lowerbound(uint64_t code, uint64_t scope, uint64_t table, float128_t& secondary, - uint64_t& primary) { - return sec_lookup_long_double.lowerbound_secondary(name{code}, name{scope}, name{table}, secondary, primary); - } - - int32_t db_context_rocksdb::db_idx_long_double_upperbound(uint64_t code, uint64_t scope, uint64_t table, float128_t& secondary, - uint64_t& primary) { - return sec_lookup_long_double.upperbound_secondary(name{code}, name{scope}, name{table}, secondary, primary); - } - - int32_t db_context_rocksdb::db_idx_long_double_end(uint64_t code, uint64_t scope, uint64_t table) { - return sec_lookup_long_double.end_secondary(name{code}, name{scope}, name{table}); - } - - int32_t db_context_rocksdb::db_idx_long_double_next(int32_t iterator, uint64_t& primary) { - return sec_lookup_long_double.next_secondary(iterator, primary); - } - - int32_t db_context_rocksdb::db_idx_long_double_previous(int32_t iterator, uint64_t& primary) { - return sec_lookup_long_double.previous_secondary(iterator, primary); - } - - // for primary keys in the iterator store, we don't care about secondary key - db_context_rocksdb::prim_key_iter_type db_context_rocksdb::primary_key_iter(int table_ei, uint64_t key, account_name payer) { - return prim_key_iter_type { .table_ei = table_ei, .secondary = noop_secondary, .primary = key, .payer = payer}; - } - - void db_context_rocksdb::swap(int iterator, account_name payer) { - primary_iter_store.swap(iterator, noop_secondary, payer); - } - - // gets a prefix that allows for only primary key iterators - prefix_bundle db_context_rocksdb::get_primary_slice_in_primaries(name code, name scope, name table, uint64_t id) { - bytes primary_key = db_key_value_format::create_primary_key(scope, table, id); - return { primary_key, end_of_prefix::at_type, code }; - } - - pv_bundle db_context_rocksdb::get_primary_key_value(name code, name scope, name table, uint64_t id) { - prefix_bundle primary_key = get_primary_slice_in_primaries(code, scope, table, id); - return { primary_key, current_session.read(primary_key.full_key) }; - } - - void db_context_rocksdb::set_value(const shared_bytes& primary_key, const payer_payload& pp) { - current_session.write(primary_key, pp.as_payload()); - } - - int32_t db_context_rocksdb::find_and_store_primary_key(const session_variant_type::iterator& session_iter, - int32_t table_ei, const shared_bytes& type_prefix, - int32_t not_found_return, const char* calling_func, - uint64_t& found_key) { - // if nothing remains in the database, return the passed in value - if (session_iter == current_session.end() || !primary_lookup.match_prefix(type_prefix, (*session_iter).first)) { - return not_found_return; - } - EOS_ASSERT( db_key_value_format::get_primary_key((*session_iter).first, type_prefix, found_key), db_rocksdb_invalid_operation_exception, - "invariant failure in ${func}, iter store found to update but no primary keys in database", - ("func", calling_func)); - - const account_name found_payer = payer_payload(*(*session_iter).second).payer; - - return primary_iter_store.add(primary_key_iter(table_ei, found_key, found_payer)); - } - - // returns the exact iterator and the bounding key (type) - db_context_rocksdb::exact_iterator db_context_rocksdb::get_exact_iterator( - name code, name scope, name table, uint64_t primary) { - auto slice_primary_key = get_primary_slice_in_primaries(code, scope, table, primary); - auto session_iter = current_session.lower_bound(slice_primary_key.full_key); - const bool valid = primary_lookup.match(slice_primary_key.full_key, session_iter); - return { .valid = valid, .itr = std::move(session_iter), .type_prefix = std::move(slice_primary_key.prefix_key) }; - } - - int32_t db_context_rocksdb::find_i64(name code, name scope, name table, uint64_t id, comp comparison) { - // expanding the "in-play" iterator space to include every key type for that table, to ensure we know if - // the key is not found, that there is anything in the table at all (and thus can return an end iterator - // or if an invalid iterator needs to be returned - prefix_bundle primary_and_prefix_keys { db_key_value_format::create_primary_key(scope, table, id), - end_of_prefix::pre_type, code }; - auto session_iter = current_session.lower_bound(primary_and_prefix_keys.full_key); - auto is_in_table = [&prefix_key=primary_and_prefix_keys.prefix_key, - &primary_lookup=this->primary_lookup](const session_variant_type::iterator& iter) { - return primary_lookup.match_prefix(prefix_key, iter); - }; - if (!is_in_table(session_iter)) { - // if no iterator was found in this table, then there is no table entry, so the table is empty - return primary_iter_store.invalid_iterator(); - } - - const unique_table t { code, scope, table }; - const auto table_ei = primary_iter_store.cache_table(t); - - const auto& desired_type_prefix = - db_key_value_format::create_full_key_prefix(primary_and_prefix_keys.full_key, end_of_prefix::at_type); - - std::optional primary_key; - if (!primary_lookup.match(primary_and_prefix_keys.full_key, (*session_iter).first)) { - if (comparison == comp::equals) { - return table_ei; - } - else if (!primary_lookup.match_prefix(desired_type_prefix, (*session_iter).first)) { - return table_ei; - } - } - else if (comparison == comp::gt) { - ++session_iter; - // if we don't have a match, we need to identify if we went past the primary types, and thus are at the end - EOS_ASSERT( is_in_table(session_iter), db_rocksdb_invalid_operation_exception, - "invariant failure in find_i64, primary key found but was not followed by a table key"); - if (!primary_lookup.match_prefix(desired_type_prefix, session_iter)) { - return table_ei; - } - } - else { - // since key is exact, and we didn't advance, it is id - primary_key = id; - } - - const account_name found_payer = payer_payload(*(*session_iter).second).payer; - if (!primary_key) { - uint64_t key = 0; - const auto valid = db_key_value_format::get_primary_key((*session_iter).first, desired_type_prefix, key); - EOS_ASSERT( valid, db_rocksdb_invalid_operation_exception, - "invariant failure in find_i64, primary key already verified but method indicates it is not a primary key"); - primary_key = key; - } - - return primary_iter_store.add(primary_key_iter(table_ei, *primary_key, found_payer)); - } - - std::unique_ptr create_db_rocksdb_context(apply_context& context, name receiver, - db_context_rocksdb::session_variant_type session) - { - static_assert(std::is_convertible::value, "cannot convert"); - static_assert(std::is_convertible, std::default_delete >::value, "cannot convert delete"); - return std::make_unique(context, receiver, std::move(session)); - } - -}}} // namespace eosio::chain::backing_store diff --git a/libraries/chain/backing_store/db_key_value_any_lookup.cpp b/libraries/chain/backing_store/db_key_value_any_lookup.cpp deleted file mode 100644 index f6506376bf..0000000000 --- a/libraries/chain/backing_store/db_key_value_any_lookup.cpp +++ /dev/null @@ -1,128 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include - -namespace eosio { namespace chain { namespace backing_store { - - eosio::session::shared_bytes make_useless_value() { - const char null = '\0'; - return eosio::session::shared_bytes {&null, 1}; - } - - const eosio::session::shared_bytes db_key_value_any_lookup::useless_value = make_useless_value(); - - void db_key_value_any_lookup::add_table_if_needed(const shared_bytes& key, account_name payer) { - auto table_key = db_key_value_format::create_full_key_prefix(key, end_of_prefix::pre_type); - auto session_iter = current_session.lower_bound(table_key); - if (!match_prefix(table_key, session_iter)) { - // need to add the key_type::table to the end - table_key = db_key_value_format::create_table_key(table_key); - const auto legacy_key = db_key_value_format::extract_legacy_key(table_key); - const auto extracted_data = db_key_value_format::get_prefix_thru_key_type(legacy_key); - std::string event_id; - apply_context& context = parent.context; - const auto& scope = std::get<0>(extracted_data); - const auto& table = std::get<1>(extracted_data); - auto dm_logger = context.control.get_deep_mind_logger(); - if (dm_logger != nullptr) { - event_id = db_context::table_event(parent.receiver, scope, table); - } - - context.update_db_usage(payer, table_overhead, db_context::add_table_trace(context.get_action_id(), std::move(event_id))); - - payer_payload pp(payer, nullptr, 0); - current_session.write(table_key, pp.as_payload()); - - if (dm_logger != nullptr) { - db_context::log_insert_table(*dm_logger, context.get_action_id(), parent.receiver, scope, table, payer); - } - } - } - - void db_key_value_any_lookup::remove_table_if_empty(const shared_bytes& key) { - // look for any other entries in the table - auto entire_table_prefix_key = db_key_value_format::create_full_key_prefix(key, end_of_prefix::pre_type); - // since this prefix key is just scope and table, it will include all primary, secondary, and table keys - auto session_itr = current_session.lower_bound(entire_table_prefix_key); - EOS_ASSERT( session_itr != current_session.end(), db_rocksdb_invalid_operation_exception, - "invariant failure in remove_table_if_empty, iter store found and removed, but no table entry was found"); - auto key_value = *session_itr; - EOS_ASSERT( match_prefix(entire_table_prefix_key, key_value.first), db_rocksdb_invalid_operation_exception, - "invariant failure in remove_table_if_empty, iter store found and removed, but no table entry was found"); - // check if the only entry for this contract/scope/table is the table entry - auto legacy_key = db_key_value_format::extract_legacy_key(key_value.first); - if (db_key_value_format::extract_key_type(legacy_key) != backing_store::db_key_value_format::key_type::table) { - return; - } - - const auto extracted_data = db_key_value_format::get_prefix_thru_key_type(legacy_key); - const auto& scope = std::get<0>(extracted_data); - const auto& table = std::get<1>(extracted_data); - - const name payer = payer_payload{*key_value.second}.payer; - std::string event_id; - apply_context& context = parent.context; - auto dm_logger = context.control.get_deep_mind_logger(); - if (dm_logger != nullptr) { - event_id = db_context::table_event(parent.receiver, scope, table); - } - - context.update_db_usage(payer, - table_overhead, db_context::rem_table_trace(context.get_action_id(), std::move(event_id)) ); - - if (dm_logger != nullptr) { - db_context::log_remove_table(*dm_logger, context.get_action_id(), parent.receiver, scope, table, payer); - } - - current_session.erase(key_value.first); - } - - key_bundle db_key_value_any_lookup::get_slice(name code, name scope, name table) { - bytes prefix_key = db_key_value_format::create_prefix_key(scope, table); - return { prefix_key, code }; - } - - key_bundle db_key_value_any_lookup::get_table_end_slice(name code, name scope, name table) { - bytes table_key = db_key_value_format::create_table_key(scope, table); - return { table_key, code }; - } - - bool db_key_value_any_lookup::match_prefix(const shared_bytes& shorter, const shared_bytes& longer) { - if (shorter.size() > longer.size()) { - return false; - } - return memcmp(shorter.data(), longer.data(), shorter.size()) == 0; - } - - bool db_key_value_any_lookup::match_prefix(const shared_bytes& shorter, const session_variant_type::iterator& iter) { - if (iter == current_session.end()) { - return false; - } - return match_prefix(shorter, (*iter).first); - } - - bool db_key_value_any_lookup::match(const shared_bytes& lhs, const shared_bytes& rhs) { - return lhs.size() == rhs.size() && - memcmp(lhs.data(), rhs.data(), lhs.size()) == 0; - } - - bool db_key_value_any_lookup::match(const shared_bytes& lhs, const session_variant_type::iterator& iter) { - if (iter == current_session.end()) { - return false; - } - return match(lhs, (*iter).first); - } - - key_bundle::key_bundle(const b1::chain_kv::bytes& composite_key, name code) - : full_key(db_key_value_format::create_full_key(composite_key, code)){ - } - - prefix_bundle::prefix_bundle(const b1::chain_kv::bytes& composite_key, end_of_prefix prefix_end, name code) - : full_key(db_key_value_format::create_full_key(composite_key, code)), - prefix_key(db_key_value_format::create_full_key_prefix(full_key, prefix_end)) { - } - -}}} // namespace eosio::chain::backing_store diff --git a/libraries/chain/backing_store/db_key_value_format.cpp b/libraries/chain/backing_store/db_key_value_format.cpp deleted file mode 100644 index 2954f45fbd..0000000000 --- a/libraries/chain/backing_store/db_key_value_format.cpp +++ /dev/null @@ -1,298 +0,0 @@ -#include -#include -#include -#include - -namespace eosio { namespace chain { namespace backing_store { namespace db_key_value_format { - namespace detail { - constexpr std::size_t key_size(key_type kt) { - switch(kt) { - case key_type::primary: - return sizeof(uint64_t); - case key_type::sec_i64: - return sizeof(uint64_t); - case key_type::sec_i128: - return sizeof(eosio::chain::uint128_t); - case key_type::sec_i256: - return sizeof(key256_t); - case key_type::sec_double: - return sizeof(float64_t); - case key_type::sec_long_double: - return sizeof(float128_t); - case key_type::table: - return 0; // a table entry ends at the type and has no trailing sub-"keys" - default: - FC_THROW_EXCEPTION(bad_composite_key_exception, - "DB intrinsic key-value store composite key is malformed, key_size should not be " - "called with key_type: ${kt}", ("kt", to_string(kt))); - } - } - - b1::chain_kv::bytes prepare_composite_key_prefix(name scope, name table, std::size_t type_size, std::size_t key_size, std::size_t extension_size) { - constexpr static auto scope_size = sizeof(scope); - constexpr static auto table_size = sizeof(table); - b1::chain_kv::bytes key_storage; - key_storage.reserve(scope_size + table_size + type_size + key_size + extension_size); - b1::chain_kv::append_key(key_storage, scope.to_uint64_t()); - b1::chain_kv::append_key(key_storage, table.to_uint64_t()); - return key_storage; - } - - b1::chain_kv::bytes prepare_composite_key(name scope, name table, std::size_t key_size, key_type kt, std::size_t extension_size) { - constexpr static auto type_size = sizeof(key_type); - static_assert( type_size == 1, "" ); - auto key_storage = prepare_composite_key_prefix(scope, table, type_size, key_size, extension_size); - key_storage.push_back(static_cast(kt)); - return key_storage; - } - - std::string to_string(const key_type& kt) { - switch (kt) { - case key_type::primary: return "primary key of type uint64_t"; - case key_type::primary_to_sec: return "primary key to secondary key type"; - case key_type::sec_i64: return "secondary key of type uint64_t"; - case key_type::sec_i128: return "secondary key of type uint128_t"; - case key_type::sec_i256: return "secondary key of type key256_t"; - case key_type::sec_double: return "secondary key of type float64_t"; - case key_type::sec_long_double: return "secondary key of type float128_t"; - case key_type::table: return "table end key"; - default: - const int kt_as_int = static_cast(kt); - return std::string(""; - } - } - - intermittent_decomposed_prefix_values extract_from_composite_key_prefix(b1::chain_kv::bytes::const_iterator begin, b1::chain_kv::bytes::const_iterator key_end) { - auto key_loc = begin; - uint64_t scope_name = 0; - EOS_ASSERT(b1::chain_kv::extract_key(key_loc, key_end, scope_name), bad_composite_key_exception, - "DB intrinsic key-value store composite key is malformed, it does not contain a scope"); - uint64_t table_name = 0; - EOS_ASSERT(b1::chain_kv::extract_key(key_loc, key_end, table_name), bad_composite_key_exception, - "DB intrinsic key-value store composite key is malformed, it does not contain a table"); - return { name{scope_name}, name{table_name}, key_loc }; - } - - intermittent_decomposed_values extract_from_composite_key(b1::chain_kv::bytes::const_iterator begin, b1::chain_kv::bytes::const_iterator key_end) { - auto intermittent = extract_from_composite_key_prefix(begin, key_end); - auto& key_loc = std::get<2>(intermittent); - EOS_ASSERT(key_loc != key_end, bad_composite_key_exception, - "DB intrinsic key-value store composite key is malformed, it does not contain an indication of the " - "type of the db-key (primary uint64_t or secondary uint64_t/uint128_t/etc)"); - const key_type kt{*key_loc++}; - return { std::get<0>(intermittent), std::get<1>(intermittent), key_loc, kt }; - } - - // NOTE: very limited use till redesign - constexpr uint64_t db_type_and_code_size = detail::prefix_size() - detail::prefix_size(); // 1 (db type) + 8 (contract) - static_assert(db_type_and_code_size == sizeof(char) + sizeof(name), "Some assumptions on formatting have been broken"); - } // namespace detail - - b1::chain_kv::bytes create_primary_key(name scope, name table, uint64_t primary_key) { - const std::size_t zero_extension_size = 0; - b1::chain_kv::bytes composite_key = detail::prepare_composite_key(scope, table, sizeof(uint64_t), key_type::primary, zero_extension_size); - b1::chain_kv::append_key(composite_key, primary_key); - return composite_key; - } - - bool get_primary_key(const b1::chain_kv::bytes& composite_key, name& scope, name& table, uint64_t& primary_key) { - b1::chain_kv::bytes::const_iterator composite_loc; - key_type kt = key_type::sec_i64; - std::tie(scope, table, composite_loc, kt) = detail::extract_from_composite_key(composite_key.cbegin(), composite_key.cend()); - if (kt != key_type::primary) { - return false; - } - EOS_ASSERT(b1::chain_kv::extract_key(composite_loc, composite_key.cend(), primary_key), bad_composite_key_exception, - "DB intrinsic key-value store composite key is malformed, it is supposed to have a primary key"); - EOS_ASSERT(composite_loc == composite_key.cend(), bad_composite_key_exception, - "DB intrinsic key-value store composite key is malformed, it has extra data after the primary key"); - return true; - } - - b1::chain_kv::bytes create_prefix_type_key(name scope, name table, key_type kt) { - static constexpr std::size_t no_key_size = 0; - static constexpr std::size_t no_extension_size = 0; - b1::chain_kv::bytes composite_key = detail::prepare_composite_key(scope, table, no_key_size, kt, no_extension_size); - return composite_key; - } - - b1::chain_kv::bytes create_table_key(name scope, name table) { - // table key ends with the type, so just reuse method - return create_prefix_type_key(scope, table, key_type::table); - } - - eosio::session::shared_bytes create_table_key(const eosio::session::shared_bytes& prefix_key) { - const std::size_t full_prefix_size = db_key_value_format::detail::prefix_size(); - EOS_ASSERT(prefix_key.size() >= full_prefix_size, bad_composite_key_exception, - "DB intrinsic key-value store composite key is malformed, create_table_key expects to be given a full " - "key that has a full prefix leading up to the type"); - const auto new_kt = static_cast(key_type::table); - const char* new_kt_ptr = &new_kt; - // already have everything in char format/order, so just need to assemble it - return detail::assemble<2, eosio::session::shared_bytes>({rocksdb::Slice{prefix_key.data(), full_prefix_size}, - rocksdb::Slice{new_kt_ptr, 1}}); - } - - bool get_table_key(const b1::chain_kv::bytes& composite_key, name& scope, name& table) { - b1::chain_kv::bytes::const_iterator composite_loc; - key_type kt = key_type::primary; - std::tie(scope, table, composite_loc, kt) = detail::extract_from_composite_key(composite_key.cbegin(), composite_key.cend()); - if (kt != key_type::table) { - return false; - } - EOS_ASSERT(composite_loc == composite_key.cend(), bad_composite_key_exception, - "DB intrinsic key-value store composite key is malformed, there should be no trailing primary/secondary key"); - return true; - } - - b1::chain_kv::bytes create_prefix_key(name scope, name table) { - static constexpr std::size_t no_type_size = 0; - static constexpr std::size_t no_key_size = 0; - static constexpr std::size_t no_extension_size = 0; - b1::chain_kv::bytes composite_key = detail::prepare_composite_key_prefix(scope, table, no_type_size, no_key_size, no_extension_size); - return composite_key; - } - - void get_prefix_key(const b1::chain_kv::bytes& composite_key, name& scope, name& table) { - b1::chain_kv::bytes::const_iterator composite_loc; - std::tie(scope, table, composite_loc) = detail::extract_from_composite_key_prefix(composite_key.cbegin(), composite_key.cend()); - } - - bool get_trailing_primary_key(const rocksdb::Slice& full_key, const rocksdb::Slice& secondary_key_prefix, uint64_t& primary_key) { - const auto sec_prefix_size = secondary_key_prefix.size(); - EOS_ASSERT(full_key.size() == sec_prefix_size + sizeof(primary_key), bad_composite_key_exception, - "DB intrinsic key-value get_trailing_primary_key was passed a full key size: ${s1} bytes that was not " - "exactly ${s2} bytes (the size of a primary key) larger than the secondary_key_prefix size: ${s3}", - ("s1", full_key.size())("s2", sizeof(primary_key))("s3", sec_prefix_size)); - const auto comp = memcmp(secondary_key_prefix.data(), full_key.data(), sec_prefix_size); - if (comp != 0) { - return false; - } - auto start_offset = full_key.data() + sec_prefix_size; - const b1::chain_kv::bytes composite_primary_key {start_offset, start_offset + sizeof(primary_key)}; - auto composite_loc = composite_primary_key.cbegin(); - EOS_ASSERT(b1::chain_kv::extract_key(composite_loc, composite_primary_key.cend(), primary_key), bad_composite_key_exception, - "DB intrinsic key-value store invariant has changed, extract_key should only fail if the string size is" - " less than the sizeof(uint64_t)"); - return true; - } - - key_type extract_key_type(const b1::chain_kv::bytes& composite_key) { - key_type kt = key_type::primary; - std::tie(std::ignore, std::ignore, std::ignore, kt) = detail::extract_from_composite_key(composite_key.cbegin(), composite_key.cend()); - return kt; - } - - key_type extract_primary_to_sec_key_type(const b1::chain_kv::bytes& composite_key) { - b1::chain_kv::bytes::const_iterator composite_loc; - key_type kt = key_type::primary; - std::tie(std::ignore, std::ignore, composite_loc, kt) = detail::extract_from_composite_key(composite_key.cbegin(), composite_key.cend()); - EOS_ASSERT(kt == key_type::primary_to_sec, bad_composite_key_exception, - "DB intrinsic extract_primary_to_sec_key_type was passed a key that was not of type: ${type}", ("type", detail::to_string(kt))); - EOS_ASSERT(composite_loc != composite_key.cend(), bad_composite_key_exception, - "DB intrinsic extract_primary_to_sec_key_type was passed a key that was only the type prefix"); - const key_type sec_kt{*composite_loc++}; - return sec_kt; - } - - intermittent_decomposed_values get_prefix_thru_key_type(const b1::chain_kv::bytes& composite_key) { - return detail::extract_from_composite_key(composite_key.cbegin(), composite_key.cend()); - } - - b1::chain_kv::bytes extract_legacy_key(const eosio::session::shared_bytes& complete_key) { - const auto slice = extract_legacy_slice(complete_key); - b1::chain_kv::bytes ret {slice.data(), slice.data() + slice.size()}; - return ret; - } - - rocksdb::Slice extract_legacy_slice(const eosio::session::shared_bytes& full_key) { - // offset into the whole key for the old postfix (after db type and contract) - return {full_key.data() + detail::db_type_and_code_size, - full_key.size() - detail::db_type_and_code_size}; - } - - eosio::session::shared_bytes create_full_key(const b1::chain_kv::bytes& composite_key, name code) { - static const char db_type_prefix = make_rocksdb_contract_db_prefix(); - b1::chain_kv::bytes code_as_bytes; - b1::chain_kv::append_key(code_as_bytes, code.to_uint64_t()); - auto ret = eosio::session::make_shared_bytes({std::string_view{&db_type_prefix, 1}, - std::string_view{code_as_bytes.data(), - code_as_bytes.size()}, - std::string_view{composite_key.data(), - composite_key.size()}}); - return ret; - } - - eosio::session::shared_bytes create_full_key_prefix(const eosio::session::shared_bytes& full_key, end_of_prefix prefix_end) { - auto calc_extension = [](end_of_prefix prefix_end) { - switch (prefix_end) { - case end_of_prefix::pre_type: - return std::size_t(0); - case end_of_prefix::at_type: - return std::size_t(1); - case end_of_prefix::at_prim_to_sec_type: - return std::size_t(2); - case end_of_prefix::at_prim_to_sec_primary_key: - return 2 + sizeof(uint64_t); - } - __builtin_unreachable(); - }; - const std::size_t extension_size = calc_extension(prefix_end); - const std::size_t full_prefix_size = db_key_value_format::detail::prefix_size() + extension_size; - EOS_ASSERT( full_prefix_size < full_key.size(), db_rocksdb_invalid_operation_exception, - "invariant failure in prefix_bundle, the passed in full_key was: ${size1} bytes, but it needs to " - "be at least: ${size2}", ("size1", full_key.size())("size2", full_prefix_size)); - return eosio::session::shared_bytes(full_key.data(), full_prefix_size); - } - - full_key_data parse_full_key(const eosio::session::shared_bytes& full_key) { - static const char db_type_prefix = make_rocksdb_contract_db_prefix(); - EOS_ASSERT( full_key.size() >= 1, db_rocksdb_invalid_operation_exception, - "parse_full_key was passed an empty key."); - full_key_data data; - const char* offset = full_key.data(); - data.db_type = *(offset++); - const std::size_t db_type_size = 1; - if (data.db_type != db_type_prefix || full_key.size() == db_type_size) { - return data; - } - const std::size_t full_prefix_size = db_key_value_format::detail::prefix_size(); - const std::size_t full_prefix_type_size = full_prefix_size + sizeof(key_type); - const std::size_t db_type_and_contract_size = db_type_size + sizeof(name); - EOS_ASSERT( full_key.size() >= db_type_and_contract_size, db_rocksdb_invalid_operation_exception, - "parse_full_key was passed a key with a db type and erroneous data trailing."); - const b1::chain_kv::bytes composite_contract_key {offset, offset + sizeof(name)}; - offset += sizeof(name); - auto composite_loc = composite_contract_key.cbegin(); - uint64_t contract; - EOS_ASSERT(b1::chain_kv::extract_key(composite_loc, composite_contract_key.cend(), contract), bad_composite_key_exception, - "DB intrinsic key-value store invariant has changed, extract_key should only fail if the string size is" - " less than the sizeof(uint64_t), which was verified that it was not"); - EOS_ASSERT( composite_loc == composite_contract_key.cend(), db_rocksdb_invalid_operation_exception, - "bytes constructed to extract contract was constructed incorrectly."); - data.contract = name{contract}; - const auto remaining = full_key.size() - db_type_and_contract_size; - if (!remaining) { - return data; - } - data.legacy_key = b1::chain_kv::bytes {offset, offset + remaining}; - auto prefix_thru_kt = get_prefix_thru_key_type(*data.legacy_key); - data.scope = std::get<0>(prefix_thru_kt); - data.table = std::get<1>(prefix_thru_kt); - data.kt = std::get<3>(prefix_thru_kt); - const std::size_t type_prefix_length = std::distance(data.legacy_key->cbegin(), std::get<2>(prefix_thru_kt)); - data.type_prefix = {data.legacy_key->data(), type_prefix_length}; - - return data; - } - - eosio::session::shared_bytes create_full_primary_key(name code, name scope, name table, uint64_t primary_key) { - bytes composite_key = create_primary_key(scope, table, primary_key); - return create_full_key(composite_key, code); - } - - eosio::session::shared_bytes create_full_prefix_key(name code, name scope, name table, std::optional kt) { - bytes composite_key = kt ? create_prefix_type_key(scope, table, *kt) : create_prefix_key(scope, table); - return create_full_key(composite_key, code); - } -}}}} // namespace eosio::chain::backing_store::db_key_value_format diff --git a/libraries/chain/backing_store/tests/CMakeLists.txt b/libraries/chain/backing_store/tests/CMakeLists.txt deleted file mode 100644 index ea982e3e44..0000000000 --- a/libraries/chain/backing_store/tests/CMakeLists.txt +++ /dev/null @@ -1,11 +0,0 @@ -add_executable( test_kv_rocksdb test_kv_rocksdb.cpp ) -target_link_libraries( test_kv_rocksdb eosio_chain fc chainbase ) -target_include_directories( test_kv_rocksdb PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_SOURCE_DIR}/../include") -add_test(NAME test_kv_rocksdb COMMAND libraries/chain/backing_store/tests/test_kv_rocksdb WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) - -add_executable( benchmark_kv benchmark_kv.cpp ) -target_link_libraries( benchmark_kv eosio_chain fc chainbase Boost::program_options ) -target_include_directories( benchmark_kv PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" "${CMAKE_CURRENT_SOURCE_DIR}/../include") -file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/benchmark_kv_batch.py DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) -file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/benchmark_kv_single.py DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) -file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/README DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) diff --git a/libraries/chain/backing_store/tests/README b/libraries/chain/backing_store/tests/README deleted file mode 100644 index fc80e2696b..0000000000 --- a/libraries/chain/backing_store/tests/README +++ /dev/null @@ -1,23 +0,0 @@ -Microbenchmarking instructions - -1. Run "./benchmark_kv -h" to get all options. -2. To prepare a random key file, - a). use "pwgen -1" to generate - "number-of-keys" keys with "key-length", "-1" indicating - one key per line in a file. - b). use "pwgen" multile times to generate different key sized - files and concatenate them. - c). use "sort -u" to remove duplicate keys. - d). use "shuf" to randomize the final file. -3. To prepare a workset file, use "shuf -n". -4. To facilitate running a suite of benchmarking, "benchmark_kv.py" - is provided. You need to prepare "data" directory which - contains a set of key files and their workset files. - a). run "benchmark_kv.py" will benchmark operations - (get, egt_data, set, create, erase, it_create, it_next, - it_key_value) with all key files on rocksdb and chainkv. - b). a key file must end with ".keys", its workset files must - start with the key file's main part and end with ".ws". - For example, "100k_random.keys" is a key file, - "100k_random_100.ws" and "100k_random_500.ws" are its - workset files. diff --git a/libraries/chain/backing_store/tests/benchmark_kv.cpp b/libraries/chain/backing_store/tests/benchmark_kv.cpp deleted file mode 100644 index 5a7639f297..0000000000 --- a/libraries/chain/backing_store/tests/benchmark_kv.cpp +++ /dev/null @@ -1,599 +0,0 @@ -#include -#include - -#include -#include -#include - -#include -#include - -#include -#include - -using namespace eosio; -using namespace eosio::chain; - -namespace kv_benchmark { - -// Global test data -constexpr account_name receiver = "kvrdb"_n; -constexpr uint64_t contract = receiver.to_uint64_t(); -constexpr account_name payer = "payer"_n; -constexpr uint64_t default_billable_size = 12; - -struct cmd_args { - std::string operation = ""; - std::string backing_store = ""; - std::string key_file = ""; - std::string workset_file = ""; - uint32_t value_size = 1024; - uint64_t num_runs = 1000000; - uint32_t state_size_multiples = 1; // For Chainbase. Multiples of 1GB -}; - -struct measurement_t { - double user_duration_us_avg; - double system_duration_us_avg; - uint64_t actual_num_runs; - uint32_t minor_faults; - uint32_t major_faults; - uint32_t blocks_in; - uint32_t blocks_out; -}; - -struct dummy_control { - fc::logger* get_deep_mind_logger() { - return nullptr; - } - - uint32_t get_action_id() { - return 0; - } -}; - -struct dummy_context { - dummy_control control; - - uint32_t get_action_id() { - return 0; - } -}; - -struct mock_resource_manager { - uint64_t billable_size = default_billable_size; - - int64_t update_table_usage(account_name payer, int64_t delta, const kv_resource_trace& trace) { - return 0; - } - - dummy_context* _context; -}; - -// resource_manager, receiver, limits in kv_context are -// initialzed as a reference. Place them globally to avoid -// going out of scope -constexpr uint32_t max_key_size = 100; -constexpr uint32_t max_value_size = 1020*1024; -constexpr uint32_t max_iterators = 100; -kv_database_config limits { max_key_size, max_value_size, max_iterators }; -mock_resource_manager resource_manager; - -// Main purpose of this program is to microbenchmark individual -// KV API over RocksDB and ChainKV. - -// Loads keys and returns number of keys as an argument. -// This is mainly used to benchmark key creation and erase. -std::vector load_keys(const cmd_args& args, uint32_t& num_keys) { - std::vector keys; - std::ifstream key_file(args.key_file); - - if (!key_file.is_open()) { - std::cerr << "Failed to open key file " << args.key_file << std::endl; - exit(2); - } - - std::string key; - num_keys = 0; - while (getline(key_file, key)) { - keys.push_back(key); - ++num_keys; - } - - key_file.close(); - - return keys; -} - -// Load workset. Benchmarking "get", "get_data", and "set" -// requires it. -std::vector load_workset(const cmd_args& args, uint32_t& workset_size) { - std::ifstream workset_ifs(args.workset_file); - - if (!workset_ifs.is_open()) { - std::cerr << "Failed to open workset file " << args.workset_file << std::endl; - exit(2); - } - - std::string key; - std::vector workset; - workset_size = 0; - while (getline(workset_ifs, key)) { - ++workset_size; - workset.push_back(key); - } - - workset_ifs.close(); - return workset; -} - -// Read keys from key file and create key-values pairs -// on backing store. -void create_key_values(const cmd_args& args, const std::unique_ptr& kv_context_ptr, uint32_t& num_keys) { - std::ifstream key_file(args.key_file); - - if (!key_file.is_open()) { - std::cerr << "Failed to open key file " << args.key_file << std::endl; - exit(2); - } - - std::string value(args.value_size, 'a'); - std::string key; - num_keys = 0; - while (getline(key_file, key)) { - kv_context_ptr->kv_set(contract, key.c_str(), key.size(), value.c_str(), value.size(), payer); - ++num_keys; - } - - key_file.close(); -} - -// Returns the number of loops required to perform total_runs -// for runs_per_loop -uint32_t get_num_loops(const uint64_t total_runs, const uint32_t runs_per_loop) { - if (total_runs <= runs_per_loop) { - return 1; // one loop is sufficient - } else { - return total_runs/runs_per_loop + 1 ; - } -} - -// Returns difference between two timevals -double time_diff_us(const timeval& start, const timeval& end) { - timeval diff; - timersub(&end, &start, &diff); - return (1000000 * diff.tv_sec + diff.tv_usec); -} - -// Returns calculated measurement based on raw data -measurement_t calculated_measurement(const uint64_t actual_num_runs, const rusage& usage_start, const rusage& usage_end) { - measurement_t m; - - m.actual_num_runs = actual_num_runs; - m.user_duration_us_avg = time_diff_us(usage_start.ru_utime, usage_end.ru_utime)/actual_num_runs; - m.system_duration_us_avg = time_diff_us(usage_start.ru_stime, usage_end.ru_stime)/actual_num_runs; - m.minor_faults = uint64_t(usage_end.ru_minflt - usage_start.ru_minflt); - m.major_faults = uint64_t(usage_end.ru_majflt - usage_start.ru_majflt); - m.blocks_in = uint64_t(usage_end.ru_inblock - usage_start.ru_inblock); - m.blocks_out = uint64_t(usage_end.ru_oublock - usage_start.ru_oublock); - - return m; -} - -// Benchmark "get" operation -measurement_t benchmark_get(const cmd_args& args, const std::unique_ptr& kv_context_ptr, const std::vector& workset) { - uint32_t num_loops = get_num_loops(args.num_runs, workset.size()); - rusage usage_start, usage_end; - - getrusage(RUSAGE_SELF, &usage_start); - for (auto i = 0U; i < num_loops; ++i) { - for (auto& key: workset) { - uint32_t actual_value_size; - kv_context_ptr->kv_get(contract, key.c_str(), key.size(),actual_value_size); - } - } - getrusage(RUSAGE_SELF, &usage_end); - - return calculated_measurement(num_loops*workset.size(), usage_start, usage_end); -} - -// Benchmark "get_data" operation -measurement_t benchmark_get_data(const cmd_args& args, const std::unique_ptr& kv_context_ptr, const std::vector& workset) { - char* data = new char[args.value_size]; - uint32_t num_loops = get_num_loops(args.num_runs, workset.size()); - rusage usage_start, usage_end; - - getrusage(RUSAGE_SELF, &usage_start); - for (auto i = 0U; i < num_loops; ++i) { - for (auto& key: workset) { - kv_context_ptr->kv_get_data(0, data, args.value_size); - } - } - getrusage(RUSAGE_SELF, &usage_end); - - return calculated_measurement(num_loops*workset.size(), usage_start, usage_end); -} - -// Benchmark "set" operation -measurement_t benchmark_set(const cmd_args& args, const std::unique_ptr& kv_context_ptr, const std::vector& workset) { - std::string value(args.value_size, 'b'); - uint32_t num_loops = get_num_loops(args.num_runs, workset.size()); - rusage usage_start, usage_end; - - getrusage(RUSAGE_SELF, &usage_start); - for (auto i = 0U; i < num_loops; ++i) { - for (auto& key: workset) { - kv_context_ptr->kv_set(contract, key.c_str(), key.size(), value.c_str(), value.size(), payer); - } - } - getrusage(RUSAGE_SELF, &usage_end); - - return calculated_measurement(num_loops*workset.size(), usage_start, usage_end); -} - -// Benchmark "create" operation -measurement_t benchmark_create(const cmd_args& args, const std::unique_ptr& kv_context_ptr, uint32_t& num_keys) { - // read keys from the key file - std::vector keys = load_keys(args, num_keys); - std::string value(args.value_size, 'a'); - rusage usage_start, usage_end; - - getrusage(RUSAGE_SELF, &usage_start); - for (auto& key: keys) { - kv_context_ptr->kv_set(contract, key.c_str(), key.size(), value.c_str(), value.size(), payer); - } - getrusage(RUSAGE_SELF, &usage_end); - - return calculated_measurement(keys.size(), usage_start, usage_end); -} - -// Benchmark "erase" operation -measurement_t benchmark_erase(const cmd_args& args, const std::unique_ptr& kv_context_ptr, uint32_t& num_keys) { - // create keys in backing store first - std::vector keys = load_keys(args, num_keys); - std::string value(args.value_size, 'a'); - for (auto& key: keys) { - kv_context_ptr->kv_set(contract, key.c_str(), key.size(), value.c_str(), value.size(), payer); - } - - rusage usage_start, usage_end; - getrusage(RUSAGE_SELF, &usage_start); - for (auto& key: keys) { - kv_context_ptr->kv_erase(contract, key.c_str(), key.size()); - } - getrusage(RUSAGE_SELF, &usage_end); - - return calculated_measurement(keys.size(), usage_start, usage_end); -} - -// Benchmark "it_create" operation -measurement_t benchmark_it_create(const cmd_args& args, const std::unique_ptr& kv_context_ptr) { - rusage usage_start, usage_end; - getrusage(RUSAGE_SELF, &usage_start); - auto i = 0U; - std::string prefix = "a"; - while (i < args.num_runs) { - kv_context_ptr->kv_it_create(contract, prefix.c_str(), 0); // kv_it_create creates a unique pointer. Will be destoryed at the end of the scope. - ++i; - } - getrusage(RUSAGE_SELF, &usage_end); - - return calculated_measurement(args.num_runs, usage_start, usage_end); -} - -// Benchmark "it_next" operation -measurement_t benchmark_it_next(const cmd_args& args, const std::unique_ptr& kv_context_ptr, uint32_t num_keys) { - uint32_t found_key_size, found_value_size; - auto it = kv_context_ptr->kv_it_create(contract, "", 0); - it->kv_it_next(&found_key_size, &found_value_size); - - rusage usage_start, usage_end; - getrusage(RUSAGE_SELF, &usage_start); - while (it->kv_it_status() != kv_it_stat::iterator_end) { - it->kv_it_next(&found_key_size, &found_value_size); - } - getrusage(RUSAGE_SELF, &usage_end); - - // As we are iterate the whole set of the keys, the number of runs - // is num_keys. - return calculated_measurement(num_keys, usage_start, usage_end); -} - -// Benchmark "it_key" operation -measurement_t benchmark_it_key(const cmd_args& args, const std::unique_ptr& kv_context_ptr, uint32_t num_keys) { - uint32_t offset = 0; - char* dest = new char[args.value_size]; - uint32_t actual_size; - uint32_t found_key_size, found_value_size; - - auto it = kv_context_ptr->kv_it_create(contract, "", 0); - it->kv_it_next(&found_key_size, &found_value_size); // move to the first position - - rusage usage_start, usage_end; - getrusage(RUSAGE_SELF, &usage_start); - while (it->kv_it_status() != kv_it_stat::iterator_end) { - it->kv_it_key(offset, dest, found_key_size, actual_size); - it->kv_it_next(&found_key_size, &found_value_size); - } - getrusage(RUSAGE_SELF, &usage_end); - - // As we are iterate the whole set of the keys, the number of runs - // is num_keys. - return calculated_measurement(num_keys, usage_start, usage_end); -} - -// Benchmark "it_value" operation -measurement_t benchmark_it_value(const cmd_args& args, const std::unique_ptr& kv_context_ptr, uint32_t num_keys) { - uint32_t offset = 0; - char* dest = new char[args.value_size]; - uint32_t actual_size; - uint32_t found_key_size, found_value_size; - - auto it = kv_context_ptr->kv_it_create(contract, "", 0); - it->kv_it_next(&found_key_size, &found_value_size); // move to the first position - - rusage usage_start, usage_end; - getrusage(RUSAGE_SELF, &usage_start); - while (it->kv_it_status() != kv_it_stat::iterator_end) { - it->kv_it_value(offset, dest, found_value_size, actual_size); - it->kv_it_next(&found_key_size, &found_value_size); - } - getrusage(RUSAGE_SELF, &usage_end); - - // As we are iterate the whole set of the keys, the number of runs - // is num_keys. - return calculated_measurement(num_keys, usage_start, usage_end); -} - -// Print out benchmarking results -void print_results(const cmd_args& args, const uint32_t num_keys, const uint32_t workset_size, const measurement_t& m) { - std::cout - << "backing_store: " << args.backing_store - << ", operation: " << args.operation - << ", key_file: " << args.key_file - << ", num_keys: " << num_keys - << ", workset_file: " << args.workset_file - << ", workset_size: " << workset_size - << ", value_size: " << args.value_size - << ", num_runs: " << m.actual_num_runs - << ", user_cpu_us_avg: " << m.user_duration_us_avg - << ", system_cpu_us_avg: " << m.system_duration_us_avg - << ", minor_faults_total: " << m.minor_faults - << ", major_faults_total: " << m.major_faults - << ", blocks_in_total: " << m.blocks_in - << ", blocks_out_total: " << m.blocks_out - << std::endl; -} - -// Dispatcher to benchmark individual operation -void benchmark_operation(const cmd_args& args, const std::unique_ptr& kv_context_ptr) { - measurement_t m; - uint32_t num_keys {0}, workset_size {0}; - - // Benchmarking "create" and "erase" creates key values - // on their own - if (args.operation != "create" && args.operation != "erase") { - create_key_values(args, kv_context_ptr, num_keys); - } - - if (args.operation == "get" || args.operation == "get_data" || args.operation == "set") { - std::vector workset = load_workset(args, workset_size); - - if (args.operation == "get" ) { - m = benchmark_get(args, std::move(kv_context_ptr), workset); - } else if (args.operation == "get_data" ) { - m = benchmark_get_data(args, std::move(kv_context_ptr), workset); - } else if (args.operation == "set" ) { - m = benchmark_set(args, std::move(kv_context_ptr), workset); - } - } else { - // workset is not used. - if (args.operation == "it_create" ) { - m = benchmark_it_create(args, std::move(kv_context_ptr)); - } else if (args.operation == "it_next" ) { - m = benchmark_it_next(args, std::move(kv_context_ptr), num_keys); - } else if (args.operation == "it_key" ) { - m = benchmark_it_key(args, std::move(kv_context_ptr), num_keys); - } else if (args.operation == "it_value" ) { - m = benchmark_it_value(args, std::move(kv_context_ptr), num_keys); - } else if (args.operation == "create" ) { - m = benchmark_create(args, std::move(kv_context_ptr), num_keys); - } else if (args.operation == "erase" ) { - m = benchmark_erase(args, std::move(kv_context_ptr), num_keys); - } else { - std::cerr << "Unknown operation: " << args.operation << std::endl; - exit(3); - } - }; - - print_results(args, num_keys, workset_size, m); -} - -inline std::shared_ptr make_rocks_db(const std::string& name) { - rocksdb::DB* cache_ptr{ nullptr }; - auto cache = std::shared_ptr{}; - - auto options = rocksdb::Options{}; - options.create_if_missing = true; // Creates a database if it is missing - options.level_compaction_dynamic_level_bytes = true; - options.bytes_per_sync = 1048576; // used to control the write rate of flushes and compactions. - - // By default, RocksDB uses only one background thread - // for flush and compaction. - // Good value for `total_threads` is the number of cores - options.IncreaseParallelism(7); - - options.OptimizeLevelStyleCompaction(512ull << 20); // optimizes level style compaction - - // Number of open files that can be used by the DB. - // Setting it to -1 means files opened are always kept open. - options.max_open_files = -1; - - // Use this option to increase the number of threads - // used to open the files. - options.max_file_opening_threads = 7; // Default should be the # of Cores - - // Write Buffer Size - Sets the size of a single - // memtable. Once memtable exceeds this size, it is - // marked immutable and a new one is created. - // Default should be 128MB - options.write_buffer_size = 128 * 1024 * 1024; - options.max_write_buffer_number = 10; // maximum number of memtables, both active and immutable - options.min_write_buffer_number_to_merge = 2; // minimum number of memtables to be merged before flushing to storage - - // Once level 0 reaches this number of files, L0->L1 compaction is triggered. - options.level0_file_num_compaction_trigger = 2; - - // Size of L0 = write_buffer_size * min_write_buffer_number_to_merge * level0_file_num_compaction_trigger - // therefore: L0 in stable state = 128MB * 2 * 2 = 512MB - // max_bytes_for_level_basei is total size of level 1. - // For optimal performance make this equal to L0 hence 512MB - options.max_bytes_for_level_base = 512 * 1024 * 1024; - - // Files in level 1 will have target_file_size_base - // bytes. It’s recommended setting target_file_size_base - // to be max_bytes_for_level_base / 10, - // so that there are 10 files in level 1.i.e. 512MB/10 - options.target_file_size_base = (512 * 1024 * 1024) / 10; - - // This value represents the maximum number of threads - // that will concurrently perform a compaction job by - // breaking it into multiple, - // smaller ones that are run simultaneously. - options.max_subcompactions = 7; // Default should be the # of CPUs - - // Full and partitioned filters in the block-based table - // use an improved Bloom filter implementation, enabled - // with format_version 5 (or above) because previous - // releases cannot read this filter. This replacement is - // faster and more accurate, especially for high bits - // per key or millions of keys in a single (full) filter. - rocksdb::BlockBasedTableOptions table_options; - table_options.format_version = 5; - table_options.index_block_restart_interval = 16; - - // Sets the bloom filter - Given an arbitrary key, - // this bit array may be used to determine if the key - // may exist or definitely does not exist in the key set. - table_options.filter_policy.reset(rocksdb::NewBloomFilterPolicy(15, false)); - table_options.index_type = rocksdb::BlockBasedTableOptions::kBinarySearch; - - // Incorporates the Table options into options - options.table_factory.reset(NewBlockBasedTableFactory(table_options)); - - auto status = rocksdb::DB::Open(options, name.c_str(), &cache_ptr); - - cache.reset(cache_ptr); - - return cache; -} - -// The driver -void benchmark(const cmd_args& args) { - if (args.backing_store == "rocksdb") { - boost::filesystem::remove_all("kvrdb-tmp"); // Use a clean RocksDB - boost::filesystem::remove_all(chain::config::default_state_dir_name); - - constexpr size_t max_rocks_iterators = 1024; - auto rocks_session = eosio::session::make_session(make_rocks_db("kvrdb-tmp"), max_rocks_iterators); - auto session = eosio::session::session{rocks_session}; - - std::unique_ptr kv_context_ptr = create_kv_rocksdb_context(session, receiver, resource_manager, limits); - benchmark_operation(args, std::move(kv_context_ptr)); // kv_context_ptr must be in the same scope as kv_db and usage_start, since they are references in create_kv_rocksdb_context - } else { - boost::filesystem::remove_all(chain::config::default_state_dir_name); // Use a clean Chainbase - chainbase::database chainbase_db(chain::config::default_state_dir_name, database::read_write, args.state_size_multiples * chain::config::default_state_size); // Default is 1024*1024*1024ll == 1073741824 - chainbase_db.add_index(); - - std::unique_ptr kv_context_ptr = create_kv_chainbase_context(chainbase_db, receiver, resource_manager, limits); - benchmark_operation(args, std::move(kv_context_ptr)); - } -} -} // namespace kv_benchmark - -namespace bpo = boost::program_options; -using bpo::options_description; -using bpo::variables_map; - -int main(int argc, char* argv[]) { - kv_benchmark::cmd_args args; - - variables_map vmap; - options_description cli ("kv_benchmark command line options"); - - std::string gts; - cli.add_options() - ("key-file,k", bpo::value()->required(), "the file storing all the keys, mandatory") - ("workset,w", bpo::value(), "the file storing workset keys, which must be constructed from key-file and be random; the operation is repeatedly run against the workset; mandatory for get, get_data, and set") - ("operation,o", bpo::value()->required(), "operation to be benchmarked: get, get_data, set, create, erase, it_create, it_next, it_key, or it_value, mandatory") - ("backing-store,b", bpo::value()->required(), "the database where kay vlaues are stored, rocksdb or chainbase, mandatory") - ("value-size,v", bpo::value(), "value size for the keys") - ("state-size-multiples,s", bpo::value(), "multiples of 1GB for Chainbase state storage") - ("num-runs,n", bpo::value(), "minimum number of runs of the benchmarked operation") - ("help,h","microbenchmarks KV operations get, get_data, set, create (set to a new key), erase, it_create, it_next, it_key, and it_value against chainbase and rocksdb. Please note: numbers in it_key and it_value include those in it_next"); - - try { - bpo::store(bpo::parse_command_line(argc, argv, cli), vmap); - bpo::notify(vmap); - - if (vmap.count("help") > 0) { - cli.print(std::cerr); - return 0; - } - - if (vmap.count("key-file") > 0) { - args.key_file = vmap["key-file"].as(); - } - if (vmap.count("workset") > 0) { - args.workset_file = vmap["workset"].as(); - } - if (vmap.count("operation") > 0) { - args.operation = vmap["operation"].as(); - if (args.operation != "get" && args.operation != "get_data" && args.operation != "set" && args.operation != "create" && args.operation != "erase" && args.operation != "it_create" && args.operation != "it_next" && args.operation != "it_key" && args.operation != "it_value") { - std::cerr << "\'--operation\' must be get, get_data, set, create, erase, it_create, it_next, it_key, or it_value" << std::endl; - return 1; - } - } - if (vmap.count("value-size") > 0) { - args.value_size = vmap["value-size"].as(); - } - if (vmap.count("state-size-multiples") > 0) { - args.state_size_multiples = vmap["state-size-multiples"].as(); - } - if (vmap.count("num-runs") > 0) { - args.num_runs = vmap["num-runs"].as(); - } - if (vmap.count("backing-store") > 0) { - args.backing_store = vmap["backing-store"].as(); - - if (args.backing_store != "rocksdb" && args.backing_store != "chainbase") { - std::cerr << "\'--backing-store\' must be rocksdb or chainbase" << std::endl; - return 1; - } - } - } catch (boost::program_options::required_option& ex) { - // This exception is thrown whenever required options are not supplied. - // Need to catch it or we will get a crash. - if (vmap.count("help") == 0) { - // Missing required options is not an exception in "--help" case, - // do not report it as an exception - std::cerr << ex.what() << std::endl; - } - cli.print (std::cerr); - return 1; - } catch (bpo::unknown_option &ex) { - std::cerr << ex.what() << std::endl; - cli.print (std::cerr); - return 1; - } - - if ((args.operation == "get" || args.operation == "get_data" || args.operation == "set") && args.workset_file.empty()) { - std::cerr << "\'--workset\' is required for get, get_data, and set" << std::endl; - cli.print(std::cerr); - return 1; - } - - kv_benchmark::benchmark(args); - - return 0; -} diff --git a/libraries/chain/backing_store/tests/benchmark_kv_batch.py b/libraries/chain/backing_store/tests/benchmark_kv_batch.py deleted file mode 100755 index abf0b19649..0000000000 --- a/libraries/chain/backing_store/tests/benchmark_kv_batch.py +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env python3 - -import subprocess -import glob - -from benchmark_kv_single import runBenchMarking - -""" -This script benchmarks all operations with all key files -in "data" directory on rocksdb and chainbase. -You can add new key files and their workset files to -"data" directory. a key file must end with ".keys", -its workset files must start with the key file's main part -and end with ".ws" -A sample data directory would look like this -10m_random_1k.ws 5m_random_1k.ws 5m_random.keys 5m_sorted_5k.ws -10m_random.keys 5m_random_5k.ws 5m_sorted_1k.ws 5m_sorted.keys -""" -def main(): - key_files = glob.glob('data/*.keys') - for key_file in key_files: - dot_pos = key_file.find('.') - name = key_file[:dot_pos] - workset_files = glob.glob(name+"*.ws") - - for workset in workset_files: - runBenchMarking(key_file, workset); - -if __name__ == "__main__": - main() diff --git a/libraries/chain/backing_store/tests/benchmark_kv_single.py b/libraries/chain/backing_store/tests/benchmark_kv_single.py deleted file mode 100755 index 97bd5900cd..0000000000 --- a/libraries/chain/backing_store/tests/benchmark_kv_single.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python3 - -import sys -import subprocess - -""" -This script benchmarks with a single key file and one workset. -It reports both summary and detailed performance numbers -for easy comparison. -""" - -def runBenchMarking(key_file, workset): - output = [] - print("============================================") - print("Benchmarking starts") - print("============================================\n") - - for op in ["create", "get", "get_data", "set", "erase", "it_create", "it_next", "it_key", "it_value"]: - results = [] - for db in ["rocksdb", "chainbase"]: - cmd = "./benchmark_kv -k " + key_file + " -o " + op + " -b " + db + " -v 512 -s 7" - if op=="get" or op=="get_data" or op=="set": - cmd += " -w " + workset - - proc = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) - outs,errs= proc.communicate() - - decoded_outs=outs.decode("utf-8") - print(decoded_outs) - if errs: - print(errs.decode("utf-8")) - results.append(decoded_outs) - - rocksdb = dict(item.split(": ") for item in results[0].split(", ")) - chainbase = dict(item.split(": ") for item in results[1].split(", ")) - out = '{0:<14s}({1:<27} ({2:<}'.format(op, rocksdb["user_cpu_us_avg"] + ", " + rocksdb["system_cpu_us_avg"] + ")", chainbase["user_cpu_us_avg"] + ", " + chainbase["system_cpu_us_avg"] + ")") - output.append(out) - - print("\nSummary:\n") - print("Operation, RocksDB, Chainbase") - print(" (user_cpu_us, sys_cpu_us), (user_cpu_us, sys_cpu_us)\n") - for o in output: - print(o) - -def main(): - key_file = sys.argv[1] - workset = sys.argv[2] - runBenchMarking(key_file, workset); - -if __name__ == "__main__": - main() diff --git a/libraries/chain/backing_store/tests/test_kv_rocksdb.cpp b/libraries/chain/backing_store/tests/test_kv_rocksdb.cpp deleted file mode 100644 index 0f959a9fde..0000000000 --- a/libraries/chain/backing_store/tests/test_kv_rocksdb.cpp +++ /dev/null @@ -1,682 +0,0 @@ -#define BOOST_TEST_MODULE kv_rocksdb_unittests - -#include -#include -#include -#include - -#include - -using namespace eosio; -using namespace eosio::chain; - -// Global test data -constexpr account_name receiver = "kvrdb"_n; -constexpr uint64_t contract = receiver.to_uint64_t(); -constexpr account_name payer = "payernam"_n; -std::string prefix = "prefix"; -constexpr uint32_t it_key_size = 10; -constexpr uint64_t default_billable_size = 12; - -// limits in kv_context is initialzed as a reference. -// Declare it globally to avoid going out of scope -constexpr uint32_t max_key_size = 100; -constexpr uint32_t max_value_size = 1020 * 1024; -constexpr uint32_t max_iterators = 3; -kv_database_config limits{ max_key_size, max_value_size, max_iterators }; - -using namespace eosio::chain::backing_store; - -struct kv_rocksdb_fixture { - - class mock_session { - public: - class iterator final { - public: - iterator(kv_rocksdb_fixture& fixture, bool is_end) : m_fixture{ &fixture }, m_is_end{ is_end } {} - iterator(const iterator& it) = default; - iterator(iterator&&) = default; - - iterator& operator=(const iterator& it) = default; - iterator& operator=(iterator&&) = default; - - iterator& operator++() { return *this; } - iterator operator++(int) { return *this; } - iterator& operator--() { m_is_end = false; return *this; } - iterator operator--(int) { return *this; } - std::pair> operator*() const { - return m_fixture->mock_get_kv(); - } - std::pair> operator->() const { - return m_fixture->mock_get_kv(); - } - bool operator==(const iterator& other) const { return m_is_end == other.m_is_end; } - bool operator!=(const iterator& other) const { return !(*this == other); } - - bool deleted() const { return m_fixture->mock_is_erased(); } - eosio::session::shared_bytes key() { return m_fixture->mock_get_kv().first; } - private: - kv_rocksdb_fixture* m_fixture; - bool m_is_end{ false }; - }; - - public: - mock_session(kv_rocksdb_fixture& fixture) : m_fixture{ &fixture } {} - - std::optional read(const eosio::session::shared_bytes& key) const { - return m_fixture->mock_get(key); - } - void write(const eosio::session::shared_bytes& key, const eosio::session::shared_bytes& value) {} - bool contains(const eosio::session::shared_bytes& key) const { return false; } - void erase(const eosio::session::shared_bytes& key) {} - - iterator begin() { return iterator{ *m_fixture, false }; } - iterator end() { return iterator{ *m_fixture, true }; } - iterator lower_bound(const eosio::session::shared_bytes& key) { - return iterator{ *m_fixture, key != m_fixture->mock_get_key_prefix() || m_fixture->mock_is_end() }; - } - - private: - kv_rocksdb_fixture* m_fixture; - }; - - // Dummy - struct mock_resource_manager { - uint64_t billable_size = default_billable_size; - - struct mock_context { - struct mock_control { - fc::logger* get_deep_mind_logger() { return nullptr; } - }; - uint32_t get_action_id() { return 0; } - mock_control control; - }; - mock_context real; - mock_context* _context = ℜ - - int64_t update_table_usage(account_name payer, int64_t delta, const kv_resource_trace& trace) { return 0; } - }; - - kv_rocksdb_fixture() - : session {*this} { - account_name receiver{ "kvrdb"_n }; - mock_resource_manager resource_manager; - - my_kv_context = std::make_unique>(session, receiver, resource_manager, limits); - } - - mock_session session; - - // mock methods - std::function - mock_kv_set; - std::function(const eosio::session::shared_bytes&)> mock_get; - std::function> const()> mock_get_kv; - std::function mock_is_end = [](){return false;}; - std::function mock_is_valid; - std::function mock_is_erased; - std::function mock_get_key_prefix; - - // object being tested - std::unique_ptr> my_kv_context; - - // test case helpers - void check_get_payer(account_name p) { - char buf[backing_store::payer_in_value_size]; - memcpy(buf, &p, backing_store::payer_in_value_size); // copy payer to buffer - BOOST_CHECK(backing_store::get_payer(buf) == p); // read payer from the buffer and should be equal to the original - } - - struct kv_pair { - std::string key; - std::string value; - }; - - enum class it_dir { NEXT, PREV }; - - enum class it_pos { NOT_END, END }; - - enum class it_state { NOT_ERASED, ERASED }; - - enum class it_keys_equal { YES, NO }; - - enum class it_key_existing { YES, NO }; - - enum class it_exception_expected { YES, NO }; - - void check_set_new_value(const kv_pair& kv) { - uint32_t key_size = kv.key.size(); - uint32_t value_size = kv.value.size(); - int64_t resource_delta = default_billable_size + key_size + value_size; - - BOOST_CHECK(my_kv_context->kv_set(contract, kv.key.c_str(), key_size, kv.value.c_str(), value_size, payer) == - resource_delta); - } - - void check_set_existing_value(const kv_pair& kv, uint32_t old_raw_value_size) { - uint32_t key_size = kv.key.size(); - uint32_t value_size = kv.value.size(); - int64_t resource_delta = value_size - (old_raw_value_size - backing_store::payer_in_value_size); - - BOOST_CHECK(my_kv_context->kv_set(contract, kv.key.c_str(), key_size, kv.value.c_str(), value_size, payer) == - resource_delta); - } - - void check_test_kv_it_status(it_state state, it_pos pos, kv_it_stat expected_status) { - mock_get_kv = []() -> std::pair> { - static const auto expected_key = prefix + std::string{"key"}; - static const auto key = eosio::chain::make_prefix_key(contract, expected_key.c_str(), expected_key.size()); - return std::pair{ key, eosio::session::shared_bytes{"payernamValue", 13} }; - }; - mock_get_key_prefix = [&]() { return eosio::chain::make_prefix_key(contract, prefix.c_str(), prefix.size()); }; - mock_is_erased = []() -> bool { return false; }; - mock_is_end = []() -> bool { return false; }; - std::unique_ptr it = my_kv_context->kv_it_create(contract, prefix.c_str(), prefix.size()); - if (pos != it_pos::END) { - uint32_t found_key_size = 0; - uint32_t found_value_size = 0; - it->kv_it_next(&found_key_size, &found_value_size); - } - mock_is_erased = [state]() -> bool { return state == it_state::ERASED; }; - mock_is_end = [pos]() -> bool { return pos == it_pos::END; }; - BOOST_CHECK(it->kv_it_status() == expected_status); - } - - void check_kv_it_move(it_dir dir, it_state state, it_pos pos, kv_it_stat expected_status) { - mock_is_erased = [state]() -> bool { return state == it_state::ERASED; }; - - mock_is_end = [pos]() -> bool { return pos == it_pos::END; }; - - mock_is_valid = [pos, state]() -> bool { return state == it_state::NOT_ERASED && pos == it_pos::NOT_END; }; - - mock_get_kv = []() -> std::pair> { - return std::pair{ eosio::session::shared_bytes("keykeykeykeykey", 15), - std::optional{eosio::session::shared_bytes("payernamMyvalue", 15)} }; - }; - mock_get_key_prefix = [&]() { return eosio::chain::make_prefix_key(contract, prefix.c_str(), it_key_size); }; - - std::unique_ptr it = my_kv_context->kv_it_create(contract, prefix.c_str(), it_key_size); - uint32_t found_key_size, found_value_size; - if (state == it_state::ERASED) { - if (dir == it_dir::NEXT) { - BOOST_CHECK_THROW(it->kv_it_next(&found_key_size, &found_value_size), kv_bad_iter); - } else { - BOOST_CHECK_THROW(it->kv_it_prev(&found_key_size, &found_value_size), kv_bad_iter); - } - return; - } - - if (dir == it_dir::NEXT) { - BOOST_CHECK(it->kv_it_next(&found_key_size, &found_value_size) == expected_status); - } else { - BOOST_CHECK(it->kv_it_prev(&found_key_size, &found_value_size) == expected_status); - } - - if (state == it_state::NOT_ERASED && pos == it_pos::NOT_END) { // means valid - BOOST_CHECK(found_key_size == 6); // "key" - BOOST_CHECK(found_value_size == 7); // "Myvalue" - } else { - BOOST_CHECK(found_key_size == 0); - BOOST_CHECK(found_value_size == 0); - } - } - - void check_kv_it_key(it_state state, it_key_existing key_existing, kv_it_stat expected_status) { - mock_is_erased = [state]() -> bool { return state == it_state::ERASED; }; - - std::string key = "keykeykeykeykey"; - constexpr uint32_t key_size = 15; - auto sample_key_value = std::pair{ eosio::session::shared_bytes(key.c_str(), key.size()), - std::optional{eosio::session::shared_bytes("valuevaluevalue", 15)} }; - mock_get_kv = [key_existing, sample_key_value]() -> std::pair> { - if (key_existing == it_key_existing::YES) { - return sample_key_value; - } else { - return {eosio::session::shared_bytes{}, std::optional{}}; - } - }; - mock_get_key_prefix = [&]() { return eosio::chain::make_prefix_key(contract, prefix.c_str(), it_key_size); }; - - std::unique_ptr it = my_kv_context->kv_it_create(contract, prefix.c_str(), it_key_size); - uint32_t offset = 0; - char dest[key_size]; - uint32_t size = key_size; - uint32_t actual_size; - - if (state == it_state::ERASED) { - BOOST_CHECK_THROW(it->kv_it_key(offset, dest, key_size, actual_size), kv_bad_iter); - return; - } - - if (expected_status != kv_it_stat::iterator_end) { - uint32_t ignore; - it->kv_it_prev(&ignore, &ignore); - } - BOOST_CHECK(it->kv_it_key(offset, dest, size, actual_size) == expected_status); - - if (expected_status == kv_it_stat::iterator_ok) { - BOOST_CHECK(actual_size == 6); - BOOST_CHECK(memcmp(key.c_str(), dest, 6) == 0); - } else { - BOOST_CHECK(actual_size == 0); - } - } - - void check_kv_it_value(it_state state, it_key_existing key_existing, kv_it_stat expected_status) { - mock_is_erased = [state]() -> bool { return state == it_state::ERASED; }; - - std::string value = "payernamMyvalue"; - constexpr uint32_t value_size = 7; // Myvalue - auto sample_key_value = std::pair{ eosio::session::shared_bytes("keykeykeykeykey", 15), - std::optional{eosio::session::shared_bytes(value.c_str(), value.size())} }; - mock_get_kv = [&]() -> std::pair> { - if (key_existing == it_key_existing::YES) { - return sample_key_value; - } else { - return { eosio::session::shared_bytes{}, std::optional{} }; - } - }; - mock_get_key_prefix = [&]() { return eosio::chain::make_prefix_key(contract, prefix.c_str(), it_key_size); }; - - std::unique_ptr it = my_kv_context->kv_it_create(contract, prefix.c_str(), it_key_size); - uint32_t offset = 0; - char dest[value_size]; - uint32_t actual_size; - - if (state == it_state::ERASED) { - BOOST_CHECK_THROW(it->kv_it_value(offset, dest, value_size, actual_size), kv_bad_iter); - return; - } - - if (expected_status != kv_it_stat::iterator_end) { - uint32_t ignore; - it->kv_it_prev(&ignore, &ignore); - } - BOOST_CHECK(it->kv_it_value(offset, dest, value_size, actual_size) == expected_status); - - if (expected_status == kv_it_stat::iterator_ok) { - BOOST_CHECK(actual_size == value_size); - BOOST_CHECK(memcmp(value.c_str() + backing_store::payer_in_value_size, dest, value_size) == 0); - } else { - BOOST_CHECK(actual_size == 0); - } - } - - void check_kv_it_compare(it_state state, it_exception_expected exception_expected, uint64_t rhs_contract) { - mock_is_erased = [state]() -> bool { return state == it_state::ERASED; }; - mock_get_kv = []() { - return std::pair{ eosio::session::shared_bytes{}, std::optional{} }; - }; - - mock_get_key_prefix = [&]() { return eosio::chain::make_prefix_key(contract, prefix.c_str(), it_key_size); }; - std::unique_ptr it = my_kv_context->kv_it_create(contract, prefix.c_str(), it_key_size); - - uint32_t num_iterators = 5; - uint32_t size = 10; - mock_get_key_prefix = [&]() { return eosio::chain::make_prefix_key(rhs_contract, prefix.c_str(), size); }; - kv_iterator_rocksdb rhs{ num_iterators, *my_kv_context->session, rhs_contract, prefix.c_str(), size }; - - if (exception_expected == it_exception_expected::YES) { - BOOST_CHECK_THROW(it->kv_it_compare(rhs), kv_bad_iter); - } else { - BOOST_CHECK(it->kv_it_compare(rhs)==0); - } - } - - void check_kv_it_key_compare(it_state state, it_keys_equal keys_equal, const std::string& mykey) { - mock_is_erased = [state]() -> bool { return state == it_state::ERASED; }; - - mock_get_kv = []() -> std::pair> { - static const auto expected_key = prefix + std::string{"key"}; - static const auto key = eosio::chain::make_prefix_key(contract, expected_key.c_str(), expected_key.size()); - return std::pair{ key, eosio::session::shared_bytes{"payernamValue", 13} }; - }; - mock_get_key_prefix = [&]() { return eosio::chain::make_prefix_key(contract, prefix.c_str(), prefix.size()); }; - std::unique_ptr it = my_kv_context->kv_it_create(contract, prefix.c_str(), prefix.size()); - if (state == it_state::ERASED) { - BOOST_CHECK_THROW(it->kv_it_key_compare(mykey.c_str(), mykey.size()), kv_bad_iter); - } else { - uint32_t found_key_size = 0; - uint32_t found_value_size = 0; - it->kv_it_next(&found_key_size, &found_value_size); - if (keys_equal == it_keys_equal::YES) { - BOOST_CHECK(it->kv_it_key_compare(mykey.c_str(), mykey.size()) == 0); - } else { - BOOST_CHECK(it->kv_it_key_compare(mykey.c_str(), mykey.size()) != 0); - } - } - } -}; - -// Test cases started -// -BOOST_AUTO_TEST_SUITE(kv_rocksdb_unittests) - -BOOST_AUTO_TEST_CASE(test_actual_value_size) { - BOOST_CHECK(actual_value_size(payer_in_value_size) == 0); - BOOST_CHECK(actual_value_size(payer_in_value_size + 1) == 1); - BOOST_CHECK(actual_value_size(payer_in_value_size + 10) == 10); -} - -BOOST_AUTO_TEST_CASE(test_actual_value_size_small_value) { - // any raw size less than kv_payer_size should throw - BOOST_CHECK_THROW(actual_value_size(payer_in_value_size - 1), kv_rocksdb_bad_value_size_exception); -} - -BOOST_FIXTURE_TEST_CASE(test_get_payer_1_char, kv_rocksdb_fixture) { check_get_payer("a"_n); } - -BOOST_FIXTURE_TEST_CASE(test_get_payer_4_chars, kv_rocksdb_fixture) { check_get_payer("abcd"_n); } - -BOOST_FIXTURE_TEST_CASE(test_get_payer_8_chars, kv_rocksdb_fixture) { check_get_payer("abcdefg"_n); } - -BOOST_AUTO_TEST_CASE(test_actual_value_start) { - char buf[10]; // any size of buffer will work - BOOST_CHECK(actual_value_start(buf) == (buf + payer_in_value_size)); -} - -BOOST_FIXTURE_TEST_CASE(test_kv_erase, kv_rocksdb_fixture) { - std::string v = "111111119876543210"; // "1111111" is payer - mock_get = [v](const eosio::session::shared_bytes&) -> std::optional { - return eosio::session::shared_bytes(v.data(), v.size()); - }; - - std::string key = "key"; - int64_t resource_delta = -(default_billable_size + key.size() + actual_value_size(v.size())); - - BOOST_CHECK(my_kv_context->kv_erase(contract, key.c_str(), key.size()) == resource_delta); -} - -BOOST_FIXTURE_TEST_CASE(test_kv_erase_zero_value, kv_rocksdb_fixture) { - std::string v = "11111111"; // "1111111" is payer, no value existing - mock_get = [v](const eosio::session::shared_bytes&) -> std::optional { - return eosio::session::shared_bytes(v.data(), v.size()); - }; - - std::string key = "key"; - int64_t resource_delta = -(default_billable_size + key.size() + actual_value_size(v.size())); - - BOOST_CHECK(my_kv_context->kv_erase(contract, key.c_str(), key.size()) == resource_delta); -} - -BOOST_FIXTURE_TEST_CASE(test_kv_erase_key_not_exist, kv_rocksdb_fixture) { - mock_get = [](const eosio::session::shared_bytes&) -> std::optional { - return std::optional{}; - }; - - std::string key = "key"; - BOOST_CHECK(my_kv_context->kv_erase(contract, key.c_str(), key.size()) == 0); -} - -BOOST_FIXTURE_TEST_CASE(test_kv_erase_key_contract_not_match, kv_rocksdb_fixture) { - mock_get = [](const eosio::session::shared_bytes&) -> std::optional { - return std::optional{}; - }; - - std::string key = "key"; - BOOST_CHECK_THROW(my_kv_context->kv_erase(contract + 1, key.c_str(), key.size()), table_operation_not_permitted); -} - -BOOST_FIXTURE_TEST_CASE(test_kv_set_new_val, kv_rocksdb_fixture) { - mock_get = [](const eosio::session::shared_bytes&) -> std::optional { - return std::optional{}; // Not found means we need to create new key - }; - - check_set_new_value({ .key = "1", .value = "" }); - check_set_new_value({ .key = "1", .value = "2" }); - check_set_new_value({ .key = "1", .value = "1111222233334444" }); - check_set_new_value({ .key = "1234567890", .value = "1111222233334444" }); - check_set_new_value({ .key = "1234567890", .value = "1111222233334444" }); -} - -BOOST_FIXTURE_TEST_CASE(test_kv_set_bigger_existing_val, kv_rocksdb_fixture) { - std::string v = "111111112345"; - mock_get = [v](const eosio::session::shared_bytes&) -> std::optional { - return eosio::session::shared_bytes(v.data(), v.size()); - }; - - check_set_existing_value({ .key = "1234567890", .value = "23" }, v.size()); -} - -BOOST_FIXTURE_TEST_CASE(test_kv_set_smaller_existing_val, kv_rocksdb_fixture) { - std::string v = "1111111123"; - mock_get = [v](const eosio::session::shared_bytes&) -> std::optional { - return eosio::session::shared_bytes(v.data(), v.size()); - }; - - check_set_existing_value({ .key = "1234567890", .value = "2345678" }, v.size()); -} - -BOOST_FIXTURE_TEST_CASE(test_kv_set_same_existing_val, kv_rocksdb_fixture) { - std::string v = "1111111123"; // First 8 bytes are the payer - mock_get = [v](const eosio::session::shared_bytes&) -> std::optional { - return eosio::session::shared_bytes(v.data(), v.size()); - }; - - check_set_existing_value({ .key = "1234567890", .value = "23" }, v.size()); -} - -BOOST_FIXTURE_TEST_CASE(test_kv_set_key_too_large, kv_rocksdb_fixture) { - std::string key = "key"; - std::string value = "value"; - uint32_t key_size = max_key_size + 1; - uint32_t value_size = value.size(); - - BOOST_CHECK_THROW(my_kv_context->kv_set(contract, key.c_str(), key_size, value.c_str(), value_size, payer), - kv_limit_exceeded); -} - -BOOST_FIXTURE_TEST_CASE(test_kv_set_value_too_large, kv_rocksdb_fixture) { - std::string key = "key"; - std::string value = "value"; - uint32_t key_size = key.size(); - uint32_t value_size = max_value_size + 1; // 1 larger than max - - BOOST_CHECK_THROW(my_kv_context->kv_set(contract, key.c_str(), key_size, value.c_str(), value_size, payer), - kv_limit_exceeded); -} - -BOOST_FIXTURE_TEST_CASE(test_kv_get, kv_rocksdb_fixture) { - std::string v = "1111111123"; // First 8 bytes are the payer - mock_get = [v](const eosio::session::shared_bytes&) -> std::optional { - return eosio::session::shared_bytes(v.data(), v.size()); - }; - - uint32_t value_size; - BOOST_CHECK(my_kv_context->kv_get(contract, "key", 3, value_size)); - BOOST_CHECK(value_size == 2); -} - -BOOST_FIXTURE_TEST_CASE(test_kv_get_not_existing, kv_rocksdb_fixture) { - mock_get = [](const eosio::session::shared_bytes&) -> std::optional { - return std::optional{}; - }; - - uint32_t value_size; - BOOST_CHECK(my_kv_context->kv_get(contract, "key", 3, value_size) == false); - BOOST_CHECK(value_size == 0); -} - -BOOST_FIXTURE_TEST_CASE(test_kv_get_data, kv_rocksdb_fixture) { - std::string v = "111111119876543210"; // First 8 bytes (11111111) are the payer - mock_get = [v](const eosio::session::shared_bytes&) -> std::optional { - return eosio::session::shared_bytes(v.data(), v.size()); - }; - - uint32_t value_size; - constexpr uint32_t data_size = 10; - char data[data_size]; - - // Get the key-value first - BOOST_CHECK(my_kv_context->kv_get(contract, "key", 3, value_size) == true); - - // offset starts at 0 - BOOST_CHECK(my_kv_context->kv_get_data(0, data, data_size) == data_size); - BOOST_CHECK(memcmp(data, v.c_str() + payer_in_value_size, data_size) == 0); - - // offset less than actual data size - BOOST_CHECK(my_kv_context->kv_get_data(5, data, data_size) == data_size); - BOOST_CHECK(memcmp(data, v.c_str() + payer_in_value_size + 5, data_size - 5) == 0); - - // offset greater than actual data size. Data should not be modified - std::string pattern = "mypattern1"; - memcpy(data, pattern.c_str(), data_size); - BOOST_CHECK(my_kv_context->kv_get_data(data_size + 1, data, data_size) == data_size); - BOOST_CHECK(memcmp(data, pattern.c_str(), data_size) == 0); -} - -BOOST_FIXTURE_TEST_CASE(test_kv_get_data_no_key, kv_rocksdb_fixture) { - mock_get = [](const eosio::session::shared_bytes&) -> std::optional { - return std::optional{}; - }; - - uint32_t value_size; - constexpr uint32_t data_size = 10; - char data[data_size]; - std::string pattern = "mypattern1"; - memcpy(data, pattern.c_str(), data_size); - - // Get the key-value first - BOOST_CHECK(my_kv_context->kv_get(contract, "key", 3, value_size) == false); - - BOOST_CHECK(my_kv_context->kv_get_data(0, data, data_size) == 0); - BOOST_CHECK(memcmp(data, pattern.c_str(), data_size) == 0); - BOOST_CHECK(my_kv_context->kv_get_data(100, data, data_size) == 0); // offset too big - BOOST_CHECK(memcmp(data, pattern.c_str(), data_size) == 0); -} - -// Iterators - -BOOST_FIXTURE_TEST_CASE(test_iter_creation_success, kv_rocksdb_fixture) { - mock_get_key_prefix = [&]() { return eosio::chain::make_prefix_key(contract, prefix.c_str(), it_key_size); }; - BOOST_CHECK_NO_THROW(my_kv_context->kv_it_create(contract, prefix.c_str(), it_key_size)); -} - -BOOST_FIXTURE_TEST_CASE(test_iter_creation_too_many, kv_rocksdb_fixture) { - std::unique_ptr its[max_iterators + 1]; // save first max_iterators so they won't be destroyed - mock_get_key_prefix = [&]() { return eosio::chain::make_prefix_key(contract, prefix.c_str(), it_key_size); }; - - // Creating max_iterators iterators - for (auto i = 0U; i < max_iterators; ++i) { - its[i] = my_kv_context->kv_it_create(contract, prefix.c_str(), it_key_size); - } - - // Creating one more - BOOST_CHECK_THROW(my_kv_context->kv_it_create(contract, prefix.c_str(), it_key_size), kv_bad_iter); -} - -BOOST_FIXTURE_TEST_CASE(test_iter_creation_key_size_too_big, kv_rocksdb_fixture) { - mock_get_key_prefix = [&]() { return eosio::chain::make_prefix_key(contract, prefix.c_str(), max_key_size); }; - BOOST_CHECK_NO_THROW(my_kv_context->kv_it_create(contract, prefix.c_str(), max_key_size)); // within the size - BOOST_CHECK_THROW(my_kv_context->kv_it_create(contract, prefix.c_str(), max_key_size + 1), - kv_bad_iter); // outside of the max key size -} - -BOOST_FIXTURE_TEST_CASE(test_is_kv_chainbase_context_iterator, kv_rocksdb_fixture) { - mock_get_key_prefix = [&]() { return eosio::chain::make_prefix_key(contract, prefix.c_str(), it_key_size); }; - std::unique_ptr it = my_kv_context->kv_it_create(contract, prefix.c_str(), it_key_size); - BOOST_CHECK(!it->is_kv_chainbase_context_iterator()); -} - -BOOST_FIXTURE_TEST_CASE(test_is_kv_rocksdb_context_iterator, kv_rocksdb_fixture) { - mock_get_key_prefix = [&]() { return eosio::chain::make_prefix_key(contract, prefix.c_str(), it_key_size); }; - std::unique_ptr it = my_kv_context->kv_it_create(contract, prefix.c_str(), it_key_size); - BOOST_CHECK(it->is_kv_rocksdb_context_iterator()); -} - -BOOST_FIXTURE_TEST_CASE(test_kv_it_compare, kv_rocksdb_fixture) { - check_kv_it_compare(it_state::NOT_ERASED, it_exception_expected::NO, contract); -} - -BOOST_FIXTURE_TEST_CASE(test_kv_it_compare_different_contract, kv_rocksdb_fixture) { - check_kv_it_compare(it_state::NOT_ERASED, it_exception_expected::YES, contract + 1); // +1 to make contract different -} - -BOOST_FIXTURE_TEST_CASE(test_kv_it_compare_erased, kv_rocksdb_fixture) { - check_kv_it_compare(it_state::ERASED, it_exception_expected::YES, contract); -} - -BOOST_FIXTURE_TEST_CASE(test_kv_it_key_compare_equal, kv_rocksdb_fixture) { - static const auto key = prefix + std::string{"key"}; - check_kv_it_key_compare(it_state::NOT_ERASED, it_keys_equal::YES, key.c_str()); -} - -BOOST_FIXTURE_TEST_CASE(test_kv_it_key_compare_non_equal, kv_rocksdb_fixture) { - static const auto key = prefix + std::string{"randomkey"}; - check_kv_it_key_compare(it_state::NOT_ERASED, it_keys_equal::NO, key.c_str()); -} - -BOOST_FIXTURE_TEST_CASE(test_kv_it_key_compare_erased, kv_rocksdb_fixture) { - static const auto key = prefix + std::string{"key"}; - check_kv_it_key_compare(it_state::ERASED, it_keys_equal::YES, key.c_str()); -} - -BOOST_FIXTURE_TEST_CASE(test_kv_it_status_end, kv_rocksdb_fixture) { - check_test_kv_it_status(it_state::ERASED, it_pos::END, kv_it_stat::iterator_end); - check_test_kv_it_status(it_state::NOT_ERASED, it_pos::END, kv_it_stat::iterator_end); -} - -BOOST_FIXTURE_TEST_CASE(test_kv_it_status_erased, kv_rocksdb_fixture) { - check_test_kv_it_status(it_state::ERASED, it_pos::NOT_END, kv_it_stat::iterator_erased); -} - -BOOST_FIXTURE_TEST_CASE(test_kv_it_status_ok, kv_rocksdb_fixture) { - check_test_kv_it_status(it_state::NOT_ERASED, it_pos::NOT_END, kv_it_stat::iterator_ok); -} - -BOOST_FIXTURE_TEST_CASE(test_kv_it_move_to_end, kv_rocksdb_fixture) { - mock_is_erased = []() -> bool { return false; }; - mock_get_key_prefix = [&]() { return eosio::chain::make_prefix_key(contract, prefix.c_str(), it_key_size); }; - std::unique_ptr it = my_kv_context->kv_it_create(contract, prefix.c_str(), it_key_size); - BOOST_CHECK(it->kv_it_move_to_end() == kv_it_stat::iterator_end); -} - -BOOST_FIXTURE_TEST_CASE(test_kv_it_next_erased, kv_rocksdb_fixture) { - check_kv_it_move(it_dir::NEXT, it_state::ERASED, it_pos::NOT_END, kv_it_stat::iterator_ok); -} - -BOOST_FIXTURE_TEST_CASE(test_kv_it_next_noterased_notend, kv_rocksdb_fixture) { - check_kv_it_move(it_dir::NEXT, it_state::NOT_ERASED, it_pos::NOT_END, kv_it_stat::iterator_ok); -} - -BOOST_FIXTURE_TEST_CASE(test_kv_it_next_noterased_end, kv_rocksdb_fixture) { - check_kv_it_move(it_dir::NEXT, it_state::NOT_ERASED, it_pos::END, kv_it_stat::iterator_end); -} - -BOOST_FIXTURE_TEST_CASE(test_kv_it_prev_erased, kv_rocksdb_fixture) { - check_kv_it_move(it_dir::PREV, it_state::ERASED, it_pos::NOT_END, kv_it_stat::iterator_ok); -} - -BOOST_FIXTURE_TEST_CASE(test_kv_it_prev_noterased_notend, kv_rocksdb_fixture) { - check_kv_it_move(it_dir::PREV, it_state::NOT_ERASED, it_pos::NOT_END, kv_it_stat::iterator_ok); -} - -BOOST_FIXTURE_TEST_CASE(test_kv_it_prev_noterased_end, kv_rocksdb_fixture) { - check_kv_it_move(it_dir::PREV, it_state::NOT_ERASED, it_pos::END, kv_it_stat::iterator_end); -} - -BOOST_FIXTURE_TEST_CASE(test_kv_it_key_has_key, kv_rocksdb_fixture) { - check_kv_it_key(it_state::NOT_ERASED, it_key_existing::YES, kv_it_stat::iterator_ok); -} - -BOOST_FIXTURE_TEST_CASE(test_kv_it_key_no_key, kv_rocksdb_fixture) { - check_kv_it_key(it_state::NOT_ERASED, it_key_existing::NO, kv_it_stat::iterator_end); -} - -BOOST_FIXTURE_TEST_CASE(test_kv_it_key_erased, kv_rocksdb_fixture) { - check_kv_it_key(it_state::ERASED, it_key_existing::NO, kv_it_stat::iterator_end); -} - -BOOST_FIXTURE_TEST_CASE(test_kv_it_value_has_key, kv_rocksdb_fixture) { - check_kv_it_value(it_state::NOT_ERASED, it_key_existing::YES, kv_it_stat::iterator_ok); -} - -BOOST_FIXTURE_TEST_CASE(test_kv_it_value_no_key, kv_rocksdb_fixture) { - check_kv_it_value(it_state::NOT_ERASED, it_key_existing::NO, kv_it_stat::iterator_end); -} - -BOOST_FIXTURE_TEST_CASE(test_kv_it_value_erased, kv_rocksdb_fixture) { - check_kv_it_value(it_state::ERASED, it_key_existing::NO, kv_it_stat::iterator_end); -} - -BOOST_AUTO_TEST_SUITE_END() diff --git a/libraries/chain/combined_database.cpp b/libraries/chain/combined_database.cpp deleted file mode 100644 index 4795ac15ed..0000000000 --- a/libraries/chain/combined_database.cpp +++ /dev/null @@ -1,686 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include - -namespace eosio { namespace chain { - combined_session::combined_session(chainbase::database& cb_database, eosio::session::undo_stack* undo_stack) - : kv_undo_stack{ undo_stack } { - cb_session = std::make_unique(cb_database.start_undo_session(true)); - try { - try { - if (kv_undo_stack) { - kv_undo_stack->push(); - } - } - FC_LOG_AND_RETHROW() - } - CATCH_AND_EXIT_DB_FAILURE() - } - - combined_session::combined_session(combined_session&& src) noexcept - : cb_session(std::move(src.cb_session)), kv_undo_stack(src.kv_undo_stack) { - src.kv_undo_stack = nullptr; - } - - void combined_session::push() { - if (cb_session) { - cb_session->push(); - cb_session = nullptr; - - if (kv_undo_stack) { - kv_undo_stack = nullptr; - } - } - } - - void combined_session::squash() { - if (cb_session) { - cb_session->squash(); - cb_session = nullptr; - - if (kv_undo_stack) { - try { - try { - kv_undo_stack->squash(); - kv_undo_stack = nullptr; - } - FC_LOG_AND_RETHROW() - } - CATCH_AND_EXIT_DB_FAILURE() - } - } - } - - void combined_session::undo() { - if (cb_session) { - cb_session->undo(); - cb_session = nullptr; - - if (kv_undo_stack) { - try { - try { - kv_undo_stack->undo(); - kv_undo_stack = nullptr; - } - FC_LOG_AND_RETHROW() - } - CATCH_AND_EXIT_DB_FAILURE() - } - } - } - - template - void walk_index(const Util& utils, const chainbase::database& db, F&& function) { - utils.walk(db, std::forward(function)); - } - - template - void walk_index(const index_utils& utils, const chainbase::database& db, F&& function) {} - - template - void walk_index(const index_utils& utils, const chainbase::database& db, F&& function) {} - - template - void walk_index(const index_utils& utils, const chainbase::database& db, F&& function) {} - - void add_kv_table_to_snapshot(const snapshot_writer_ptr& snapshot, const chainbase::database& db, const kv_undo_stack_ptr& kv_undo_stack) { - snapshot->write_section([&db,&kv_undo_stack](auto& section) { - if (kv_undo_stack && db.get().backing_store == backing_store_type::ROCKSDB) { - using add_database_section_receiver = backing_store::add_database_receiver>; - add_database_section_receiver add_db_receiver(section, db); - backing_store::rocksdb_contract_kv_table_writer writer(add_db_receiver); - const auto begin_key = eosio::session::shared_bytes(&backing_store::rocksdb_contract_kv_prefix, 1); - const auto end_key = begin_key.next(); - backing_store::walk_rocksdb_entries_with_prefix(kv_undo_stack, begin_key, end_key, writer); - } else { - index_utils utils; - utils.walk(db, [&db, §ion](const auto& row) { section.add_row(row, db); }); - } - }); - } - - void read_kv_table_from_snapshot(const snapshot_reader_ptr& snapshot, chainbase::database& db, - const std::unique_ptr& kv_database, uint32_t version, backing_store_type backing_store ) { - if (version < kv_object::minimum_snapshot_version) - return; - if (backing_store == backing_store_type::ROCKSDB) { - auto key_values = std::vector>{}; - constexpr std::size_t batch_size = 500; - key_values.reserve(batch_size); - snapshot->read_section([&key_values, &db, &kv_database](auto& section) { - const std::string_view prefix_key {&backing_store::rocksdb_contract_kv_prefix, 1}; - bool more = !section.empty(); - while (more) { - kv_object_view move_to_rocks; - more = section.read_row(move_to_rocks, db); - b1::chain_kv::bytes contract_as_bytes; - b1::chain_kv::append_key(contract_as_bytes, move_to_rocks.contract.to_uint64_t()); - auto full_key = - eosio::session::make_shared_bytes({prefix_key, - std::string_view{contract_as_bytes.data(), - contract_as_bytes.size()}, - std::string_view{move_to_rocks.kv_key.data.data(), - move_to_rocks.kv_key.data.size()}}); - - // Pack payer and actual key value - auto final_kv_value = backing_store::payer_payload(move_to_rocks.payer, - move_to_rocks.kv_value.data.data(), - move_to_rocks.kv_value.data.size()); - - key_values.emplace_back(full_key, - final_kv_value.as_payload()); - - if (key_values.size() >= batch_size) { - kv_database->write(key_values); - key_values.clear(); - } - } - }); - // write out any remaining key-values - kv_database->write(key_values); - } - else { - snapshot->read_section([&db](auto& section) { - bool more = !section.empty(); - while (more) { - index_utils::create(db, [&db, §ion, &more](auto &row) { more = section.read_row(row, db); }); - } - }); - } - } - - combined_database::combined_database(chainbase::database& chain_db, - uint32_t snapshot_batch_threashold) - : backing_store(backing_store_type::CHAINBASE), db(chain_db), kv_snapshot_batch_threashold(snapshot_batch_threashold * 1024 * 1024) {} - - combined_database::combined_database(chainbase::database& chain_db, - const controller::config& cfg) - : backing_store(backing_store_type::ROCKSDB), db(chain_db), kv_database{ [&]() { - rocksdb::Options options; - - options.create_if_missing = true; // Creates a database if it is missing - options.level_compaction_dynamic_level_bytes = true; - options.bytes_per_sync = cfg.persistent_storage_bytes_per_sync; // used to control the write rate of flushes and compactions. - options.use_adaptive_mutex = true; - - // Number of threads used for flush and compaction. - options.IncreaseParallelism(cfg.persistent_storage_num_threads); - - options.OptimizeLevelStyleCompaction(512ull << 20); // optimizes level style compaction - - // Number of open files that can be used by the DB. - // Setting it to -1 means files opened are always kept open. - options.max_open_files = cfg.persistent_storage_max_num_files; - - // Use this option to increase the number of threads - // used to open the files. - options.max_file_opening_threads = cfg.persistent_storage_num_threads; - - // Write Buffer Size - Sets the size of a single - // memtable. Once memtable exceeds this size, it is - // marked immutable and a new one is created. - // Default should be 128MB - options.write_buffer_size = cfg.persistent_storage_write_buffer_size; - options.max_write_buffer_number = 10; // maximum number of memtables, both active and immutable - options.min_write_buffer_number_to_merge = 2; // minimum number of memtables to be merged before flushing to storage - - // Once level 0 reaches this number of files, L0->L1 compaction is triggered. - options.level0_file_num_compaction_trigger = 2; - - // Size of L0 = write_buffer_size * min_write_buffer_number_to_merge * level0_file_num_compaction_trigger - // For optimal performance make this equal to L0 - options.max_bytes_for_level_base = cfg.persistent_storage_write_buffer_size * options.min_write_buffer_number_to_merge * options.level0_file_num_compaction_trigger; - - // Files in level 1 will have target_file_size_base - // bytes. It’s recommended setting target_file_size_base - // to be max_bytes_for_level_base / 10. - options.target_file_size_base = options.max_bytes_for_level_base / 10; - - // This value represents the maximum number of threads - // that will concurrently perform a compaction job by - // breaking it into multiple, - // smaller ones that are run simultaneously. - options.max_subcompactions = cfg.persistent_storage_num_threads; - - // Full and partitioned filters in the block-based table - // use an improved Bloom filter implementation, enabled - // with format_version 5 (or above) because previous - // releases cannot read this filter. This replacement is - // faster and more accurate, especially for high bits - // per key or millions of keys in a single (full) filter. - rocksdb::BlockBasedTableOptions table_options; - table_options.format_version = 5; - table_options.index_block_restart_interval = 16; - - // Sets the bloom filter - Given an arbitrary key, - // this bit array may be used to determine if the key - // may exist or definitely does not exist in the key set. - table_options.filter_policy.reset(rocksdb::NewBloomFilterPolicy(15, false)); - table_options.index_type = rocksdb::BlockBasedTableOptions::kBinarySearch; - - // Incorporates the Table options into options - options.table_factory.reset(NewBlockBasedTableFactory(table_options)); - - rocksdb::DB* p; - auto status = rocksdb::DB::Open(options, (cfg.state_dir / "chain-kv").string(), &p); - if (!status.ok()) - throw std::runtime_error(std::string{ "database::database: rocksdb::DB::Open: " } + status.ToString()); - auto rdb = std::shared_ptr{ p }; - return std::make_unique(eosio::session::make_session(std::move(rdb), 1024)); - }() }, - kv_undo_stack(std::make_unique>(*kv_database, cfg.state_dir)), - kv_snapshot_batch_threashold(cfg.persistent_storage_mbytes_batch * 1024 * 1024) {} - - void combined_database::check_backing_store_setting(bool clean_startup) { - if (backing_store != db.get().backing_store) { - EOS_ASSERT(clean_startup, database_move_kv_disk_exception, - "Existing state indicates a different backing store is in use; use resync, replay, or restore from snapshot to switch backing store"); - db.modify(db.get(), [this](auto& cfg) { cfg.backing_store = backing_store; }); - } - - if (backing_store == backing_store_type::ROCKSDB) - ilog("using rocksdb for backing store"); - else - ilog("using chainbase for backing store"); - } - - void combined_database::destroy(const fc::path& p) { - if( !fc::is_directory( p ) ) - return; - - fc::remove( p / "shared_memory.bin" ); - fc::remove( p / "shared_memory.meta" ); - - rocks_db_type::destroy((p / "chain-kv").string()); - } - - void combined_database::set_revision(uint64_t revision) { - db.set_revision(revision); - - if (backing_store == backing_store_type::ROCKSDB) { - try { - try { - kv_undo_stack->revision(revision); - } - FC_LOG_AND_RETHROW() - } - CATCH_AND_EXIT_DB_FAILURE() - } - } - - int64_t combined_database::revision() { - if (backing_store == backing_store_type::ROCKSDB) { - try { - try { - return kv_undo_stack->revision(); - } - FC_LOG_AND_RETHROW() - } - CATCH_AND_EXIT_DB_FAILURE() - } else { - return db.revision(); - } - } - - void combined_database::undo() { - db.undo(); - - if (backing_store == backing_store_type::ROCKSDB) { - try { - try { - kv_undo_stack->undo(); - } - FC_LOG_AND_RETHROW() - } - CATCH_AND_EXIT_DB_FAILURE() - } - } - - void combined_database::commit(int64_t revision) { - db.commit(revision); - - if (backing_store == backing_store_type::ROCKSDB) { - try { - try { - kv_undo_stack->commit(revision); - } - FC_LOG_AND_RETHROW() - } - CATCH_AND_EXIT_DB_FAILURE() - } - } - - void combined_database::flush() { - if (backing_store == backing_store_type::ROCKSDB) { - try { - try { - kv_database->flush(); - } - FC_LOG_AND_RETHROW() - } - CATCH_AND_EXIT_DB_FAILURE() - } - } - - std::unique_ptr combined_database::create_kv_context(name receiver, kv_resource_manager resource_manager, - const kv_database_config& limits) const { - switch (backing_store) { - case backing_store_type::ROCKSDB: - return std::visit([&](auto* session){ - return create_kv_rocksdb_context, kv_resource_manager>(*session, receiver, - resource_manager, limits); - }, kv_undo_stack->top().holder()); - case backing_store_type::CHAINBASE: - return create_kv_chainbase_context(db, receiver, resource_manager, limits); - } - EOS_ASSERT(false, action_validate_exception, "Unknown backing store."); - } - - std::unique_ptr combined_database::create_db_context(apply_context& context, name receiver) { - switch (backing_store) { - case backing_store_type::ROCKSDB: - return backing_store::create_db_rocksdb_context(context, receiver, kv_undo_stack->top()); - case backing_store_type::CHAINBASE: - return backing_store::create_db_chainbase_context(context, receiver); - default: - EOS_ASSERT(false, action_validate_exception, "Unknown backing store."); - } - } - - void combined_database::add_to_snapshot( - const eosio::chain::snapshot_writer_ptr& snapshot, const eosio::chain::block_state& head, - const eosio::chain::authorization_manager& authorization, - const eosio::chain::resource_limits::resource_limits_manager& resource_limits) const { - snapshot->write_section( - [this](auto& section) { section.add_row(chain_snapshot_header(), db); }); - - snapshot->write_section( - [this, &head](auto& section) { section.template add_row(head, db); }); - - eosio::chain::controller_index_set::walk_indices([this, &snapshot](auto utils) { - using value_t = typename decltype(utils)::index_t::value_type; - - snapshot->write_section([utils, this](auto& section) { - walk_index(utils, db, [this, §ion](const auto& row) { section.add_row(row, db); }); - }); - }); - - add_kv_table_to_snapshot(snapshot, db, kv_undo_stack); - add_contract_tables_to_snapshot(snapshot); - - authorization.add_to_snapshot(snapshot); - resource_limits.add_to_snapshot(snapshot); - } - - void combined_database::read_from_snapshot(const snapshot_reader_ptr& snapshot, - uint32_t blog_start, - uint32_t blog_end, - eosio::chain::authorization_manager& authorization, - eosio::chain::resource_limits::resource_limits_manager& resource_limits, - eosio::chain::block_state_ptr& head, uint32_t& snapshot_head_block, - const eosio::chain::chain_id_type& chain_id) { - chain_snapshot_header header; - snapshot->read_section([this, &header](auto& section) { - section.read_row(header, db); - header.validate(); - }); - - db.create([](auto&) {}); - check_backing_store_setting(true); - - { /// load and upgrade the block header state - block_header_state head_header_state; - using v2 = legacy::snapshot_block_header_state_v2; - - if (std::clamp(header.version, v2::minimum_version, v2::maximum_version) == header.version) { - snapshot->read_section([this, &head_header_state](auto& section) { - legacy::snapshot_block_header_state_v2 legacy_header_state; - section.read_row(legacy_header_state, db); - head_header_state = block_header_state(std::move(legacy_header_state)); - }); - } else { - snapshot->read_section( - [this, &head_header_state](auto& section) { section.read_row(head_header_state, db); }); - } - - snapshot_head_block = head_header_state.block_num; - EOS_ASSERT(blog_start <= (snapshot_head_block + 1) && snapshot_head_block <= blog_end, block_log_exception, - "Block log is provided with snapshot but does not contain the head block from the snapshot nor a " - "block right after it", - ("snapshot_head_block", snapshot_head_block)("block_log_first_num", - blog_start)("block_log_last_num", blog_end)); - - head = std::make_shared(); - static_cast(*head) = head_header_state; - } - - controller_index_set::walk_indices([this, &snapshot, &header](auto utils) { - using value_t = typename decltype(utils)::index_t::value_type; - - // skip the table_id_object as its inlined with contract tables section - if (std::is_same::value) { - return; - } - - // skip the database_header as it is only relevant to in-memory database - if (std::is_same::value) { - return; - } - - // skip the kv_db_config as it only determines where the kv-database is stored - if (std::is_same_v) { - return; - } - - // special case for in-place upgrade of global_property_object - if (std::is_same::value) { - using v2 = legacy::snapshot_global_property_object_v2; - using v3 = legacy::snapshot_global_property_object_v3; - using v4 = legacy::snapshot_global_property_object_v4; - - if (std::clamp(header.version, v2::minimum_version, v2::maximum_version) == header.version) { - std::optional genesis = extract_legacy_genesis_state(*snapshot, header.version); - EOS_ASSERT(genesis, snapshot_exception, - "Snapshot indicates chain_snapshot_header version 2, but does not contain a genesis_state. " - "It must be corrupted."); - snapshot->read_section( - [&db = this->db, gs_chain_id = genesis->compute_chain_id()](auto& section) { - v2 legacy_global_properties; - section.read_row(legacy_global_properties, db); - - db.create([&legacy_global_properties, &gs_chain_id](auto& gpo) { - gpo.initalize_from(legacy_global_properties, gs_chain_id, kv_database_config{}, - genesis_state::default_initial_wasm_configuration); - }); - }); - return; // early out to avoid default processing - } - - if (std::clamp(header.version, v3::minimum_version, v3::maximum_version) == header.version) { - snapshot->read_section([&db = this->db](auto& section) { - v3 legacy_global_properties; - section.read_row(legacy_global_properties, db); - - db.create([&legacy_global_properties](auto& gpo) { - gpo.initalize_from(legacy_global_properties, kv_database_config{}, - genesis_state::default_initial_wasm_configuration); - }); - }); - return; // early out to avoid default processing - } - - if (std::clamp(header.version, v4::minimum_version, v4::maximum_version) == header.version) { - snapshot->read_section([&db = this->db](auto& section) { - v4 legacy_global_properties; - section.read_row(legacy_global_properties, db); - - db.create([&legacy_global_properties](auto& gpo) { - gpo.initalize_from(legacy_global_properties); - }); - }); - return; // early out to avoid default processing - } - - } - - snapshot->read_section([this](auto& section) { - bool more = !section.empty(); - while (more) { - decltype(utils)::create(db, [this, §ion, &more](auto& row) { more = section.read_row(row, db); }); - } - }); - }); - - read_kv_table_from_snapshot(snapshot, db, kv_database, header.version, backing_store); - read_contract_tables_from_snapshot(snapshot); - - authorization.read_from_snapshot(snapshot); - resource_limits.read_from_snapshot(snapshot, header.version); - - set_revision(head->block_num); - db.create([](const auto& header) { - // nothing to do - }); - - const auto& gpo = db.get(); - EOS_ASSERT(gpo.chain_id == chain_id, chain_id_type_exception, - "chain ID in snapshot (${snapshot_chain_id}) does not match the chain ID that controller was " - "constructed with (${controller_chain_id})", - ("snapshot_chain_id", gpo.chain_id)("controller_chain_id", chain_id)); - } - - template - void chainbase_add_contract_tables_to_snapshot(const chainbase::database& db, Section& section) { - index_utils::walk(db, [&db, §ion](const table_id_object& table_row) { - // add a row for the table - section.add_row(table_row, db); - - // followed by a size row and then N data rows for each type of table - contract_database_index_set::walk_indices([&db, §ion, &table_row](auto utils) { - using utils_t = decltype(utils); - using value_t = typename utils_t::index_t::value_type; - using by_table_id = object_to_table_id_tag_t; - - auto tid_key = std::make_tuple(table_row.id); - auto next_tid_key = std::make_tuple(table_id_object::id_type(table_row.id._id + 1)); - - unsigned_int size = utils_t::template size_range(db, tid_key, next_tid_key); - section.add_row(size, db); - - utils_t::template walk_range(db, tid_key, next_tid_key, [&db, §ion](const auto& row) { - section.add_row(row, db); - }); - }); - }); - } - - void combined_database::add_contract_tables_to_snapshot(const snapshot_writer_ptr& snapshot) const { - snapshot->write_section("contract_tables", [this](auto& section) { - if (kv_undo_stack && db.get().backing_store == backing_store_type::ROCKSDB) { - using add_database_section_receiver = backing_store::add_database_receiver>; - using table_collector = backing_store::rocksdb_whole_db_table_collector; - - add_database_section_receiver add_db_receiver(section, db); - table_collector table_collector_receiver(add_db_receiver); - backing_store::rocksdb_contract_db_table_writer writer(table_collector_receiver); - const auto begin_key = eosio::session::shared_bytes(&backing_store::rocksdb_contract_db_prefix, 1); - const auto end_key = begin_key.next(); - backing_store::walk_rocksdb_entries_with_prefix(kv_undo_stack, begin_key, end_key, writer); - } - else { - chainbase_add_contract_tables_to_snapshot(db, section); - } - }); - } - - template - void chainbase_read_contract_tables_from_snapshot(chainbase::database& db, Section& section) { - bool more = !section.empty(); - while (more) { - // read the row for the table - table_id_object::id_type t_id; - - index_utils::create(db, [&db, §ion, &t_id](auto& row) { - section.read_row(row, db); - t_id = row.id; - }); - - // read the size and data rows for each type of table - contract_database_index_set::walk_indices([&db, §ion, &t_id, &more](auto utils) { - using utils_t = decltype(utils); - - unsigned_int size; - more = section.read_row(size, db); - - for (size_t idx = 0; idx < size.value; ++idx) { - utils_t::create(db, [&db, §ion, &more, &t_id](auto& row) { - row.t_id = t_id; - more = section.read_row(row, db); - }); - } - }); - } - } - - template - void rocksdb_read_contract_tables_from_snapshot(rocks_db_type& kv_database, chainbase::database& db, - Section& section, uint64_t snapshot_batch_threashold) { - std::vector> batch; - bool more = !section.empty(); - auto read_row = [§ion, &more, &db](auto& row) { more = section.read_row(row, db); }; - uint64_t batch_mem_size = 0; - - while (more) { - // read the row for the table - backing_store::table_id_object_view table_obj; - read_row(table_obj); - auto put = [&batch, &table_obj, &batch_mem_size, &kv_database, snapshot_batch_threashold] - (auto&& value, auto create_fun, auto&&... args) { - auto composite_key = create_fun(table_obj.scope, table_obj.table, std::forward(args)...); - batch.emplace_back(backing_store::db_key_value_format::create_full_key(composite_key, table_obj.code), - std::forward(value)); - - const auto& back = batch.back(); - const auto size = back.first.size() + back.second.size(); - if (size >= snapshot_batch_threashold || snapshot_batch_threashold - size < batch_mem_size) { - kv_database.write(batch); - batch_mem_size = 0; - batch.clear(); - } - else { - batch_mem_size += size; - } - }; - - // handle the primary key index - unsigned_int size; - read_row(size); - for (size_t i = 0; i < size.value; ++i) { - backing_store::primary_index_view row; - read_row(row); - backing_store::payer_payload pp{row.payer, row.value.data(), row.value.size()}; - put(pp.as_payload(), backing_store::db_key_value_format::create_primary_key, row.primary_key); - } - - auto write_secondary_index = [&put, &read_row](auto index) { - using index_t = decltype(index); - static const eosio::session::shared_bytes empty_payload; - unsigned_int size; - read_row(size); - for (uint32_t i = 0; i < size.value; ++i) { - backing_store::secondary_index_view row; - read_row(row); - backing_store::payer_payload pp{row.payer, nullptr, 0}; - put(pp.as_payload(), &backing_store::db_key_value_format::create_secondary_key, - row.secondary_key, row.primary_key); - - put(empty_payload, &backing_store::db_key_value_format::create_primary_to_secondary_key, - row.primary_key, row.secondary_key); - } - }; - - // handle secondary key indices - std::tuple indices; - std::apply([&write_secondary_index](auto... index) { (write_secondary_index(index), ...); }, indices); - - backing_store::payer_payload pp{table_obj.payer, nullptr, 0}; - b1::chain_kv::bytes (*create_table_key)(name scope, name table) = backing_store::db_key_value_format::create_table_key; - put(pp.as_payload(), create_table_key); - - } - kv_database.write(batch); - } - - void combined_database::read_contract_tables_from_snapshot(const snapshot_reader_ptr& snapshot) { - snapshot->read_section("contract_tables", [this](auto& section) { - if (kv_undo_stack && db.get().backing_store == backing_store_type::ROCKSDB) - rocksdb_read_contract_tables_from_snapshot(*kv_database, db, section, kv_snapshot_batch_threashold); - else - chainbase_read_contract_tables_from_snapshot(db, section); - }); - } - - std::optional extract_legacy_genesis_state(snapshot_reader& snapshot, - uint32_t version) { - std::optional genesis; - using v2 = legacy::snapshot_global_property_object_v2; - - if (std::clamp(version, v2::minimum_version, v2::maximum_version) == version) { - genesis.emplace(); - snapshot.read_section( - [&genesis = *genesis](auto& section) { section.read_row(genesis); }); - } - return genesis; - } - - // TODO : need to change this method to just return a char - std::vector make_rocksdb_contract_kv_prefix() { return std::vector { backing_store::rocksdb_contract_kv_prefix }; } - char make_rocksdb_contract_db_prefix() { return backing_store::rocksdb_contract_db_prefix; } - -}} // namespace eosio::chain - - diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 2fc3cc41ed..a4b7202511 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -68,7 +68,7 @@ struct completed_block { using block_stage_type = std::variant; struct pending_state { - pending_state( combined_session&& s, const block_header_state& prev, + pending_state( kv_session&& s, const block_header_state& prev, block_timestamp_type when, uint16_t num_prev_blocks_to_confirm, const vector& new_protocol_feature_activations ) @@ -76,7 +76,7 @@ struct pending_state { ,_block_stage( building_block( prev, when, num_prev_blocks_to_confirm, new_protocol_feature_activations ) ) {} - combined_session _db_session; + kv_session _db_session; block_stage_type _block_stage; controller::block_status _block_status = controller::block_status::incomplete; std::optional _producer_block_id; @@ -162,7 +162,7 @@ struct controller_impl { controller& self; std::function shutdown; chainbase::database db; - combined_database kv_db; + kv_database kv_db; block_log blog; std::optional pending; block_state_ptr head; @@ -234,9 +234,7 @@ struct controller_impl { db( cfg.state_dir, cfg.read_only ? database::read_only : database::read_write, cfg.state_size, false, cfg.db_map_mode ), - kv_db(cfg.backing_store == backing_store_type::CHAINBASE - ? combined_database(db, cfg.persistent_storage_mbytes_batch) - : combined_database(db, cfg)), + kv_db(kv_database(db)), blog( cfg.blog ), fork_db( cfg.blog.log_dir / config::reversible_blocks_dir_name ), wasmif( cfg.wasm_runtime, cfg.eosvmoc_tierup, db, cfg.state_dir, cfg.eosvmoc_config, !cfg.profile_accounts.empty() ), @@ -976,7 +974,7 @@ struct controller_impl { const bool validating = !self.is_producing_block(); EOS_ASSERT( !validating || explicit_billed_cpu_time, transaction_exception, "validating requires explicit billing" ); - combined_session undo_session = !self.skip_db_sessions() ? kv_db.make_session() : kv_db.make_no_op_session(); + kv_session undo_session = !self.skip_db_sessions() ? kv_db.make_session() : kv_db.make_no_op_session(); auto gtrx = generated_transaction(gto); @@ -2346,7 +2344,7 @@ const chainbase::database& controller::db()const { return my->db; } chainbase::database& controller::mutable_db()const { return my->db; } const fork_database& controller::fork_db()const { return my->fork_db; } -eosio::chain::combined_database& controller::kv_db() const { return my->kv_db; } +eosio::chain::kv_database& controller::kv_db() const { return my->kv_db; } void controller::preactivate_feature( uint32_t action_id, const digest_type& feature_digest ) { const auto& pfs = my->protocol_features.get_protocol_feature_set(); diff --git a/libraries/chain/include/eosio/chain/backing_store.hpp b/libraries/chain/include/eosio/chain/backing_store.hpp index b1fc9ae373..fafaf712da 100644 --- a/libraries/chain/include/eosio/chain/backing_store.hpp +++ b/libraries/chain/include/eosio/chain/backing_store.hpp @@ -7,7 +7,6 @@ namespace eosio { namespace chain { enum class backing_store_type { CHAINBASE, // A name for regular users. Uses Chainbase. - ROCKSDB }; }} // namespace eosio::chain @@ -21,7 +20,6 @@ template <> inline void from_variant(const fc::variant& v, eosio::chain::backing_store_type& store) { switch (store = (eosio::chain::backing_store_type)v.as_uint64()) { case eosio::chain::backing_store_type::CHAINBASE: - case eosio::chain::backing_store_type::ROCKSDB: return; } throw std::runtime_error("Invalid backing store name: " + v.as_string()); diff --git a/libraries/chain/include/eosio/chain/backing_store/chain_kv_payer.hpp b/libraries/chain/include/eosio/chain/backing_store/chain_kv_payer.hpp deleted file mode 100644 index aae8192c01..0000000000 --- a/libraries/chain/include/eosio/chain/backing_store/chain_kv_payer.hpp +++ /dev/null @@ -1,52 +0,0 @@ -#pragma once - -#include -#include - -namespace eosio { namespace chain { namespace backing_store { - static constexpr auto payer_in_value_size = sizeof(account_name); - - inline static uint32_t actual_value_size(const uint32_t raw_value_size) { - EOS_ASSERT(raw_value_size >= payer_in_value_size, kv_rocksdb_bad_value_size_exception , "The size of value returned from RocksDB is less than payer's size"); - return (raw_value_size - payer_in_value_size); - } - - inline static account_name get_payer(const char* data) { - account_name payer; - memcpy(&payer, data, payer_in_value_size); - return payer; - } - - inline static const char* actual_value_start(const char* data) { - return data + payer_in_value_size; - } - - // used to create a payload of payer and char* buffer, or to take a char* buffer and extract the payload from it - // and a char* to the value portion of the payload. NOTE: this is meant to be a short - struct payer_payload { - payer_payload(const char* data, std::size_t size) - : value(actual_value_start(data)), value_size(actual_value_size(size)), payer(get_payer(data)) {} - - template - payer_payload(const CharCont& data) - : value(actual_value_start(data.data())), value_size(actual_value_size(data.size())), - payer(get_payer(data.data())) {} - - payer_payload(name payer, const char* val, std::size_t val_size) - : value(val), value_size(val_size), payer(payer) {} - - eosio::session::shared_bytes as_payload() const { - char payer_buf[payer_in_value_size]; - memcpy(payer_buf, &payer, payer_in_value_size); - return eosio::session::make_shared_bytes({std::string_view{payer_buf, - payer_in_value_size}, - std::string_view{value, value_size}}); - } - - // pointer to the actual value portion of the payload - const char* value; - // size of the actual value portion of the payload - std::size_t value_size; - account_name payer; - }; -}}} // ns eosio::chain::backing_store diff --git a/libraries/chain/include/eosio/chain/backing_store/db_combined.hpp b/libraries/chain/include/eosio/chain/backing_store/db_combined.hpp deleted file mode 100644 index 571bdca031..0000000000 --- a/libraries/chain/include/eosio/chain/backing_store/db_combined.hpp +++ /dev/null @@ -1,646 +0,0 @@ -#pragma once -#include -#include -#include -#include -#include -#include - -namespace eosio { namespace chain { namespace backing_store { -using rocks_db_type = eosio::session::session; -using session_type = eosio::session::session; -using kv_undo_stack_ptr = std::unique_ptr>; -template -const char* contract_table_type(); - -// chainlib reserves prefixes 0x10 - 0x2F. -static constexpr char rocksdb_contract_kv_prefix = 0x11; // for KV API -static constexpr char rocksdb_contract_db_prefix = 0x12; // for DB API - -namespace detail { - class manage_stack { - public: - manage_stack(const kv_undo_stack_ptr& kv_undo_stack) : kv_undo_stack_(kv_undo_stack), undo_(kv_undo_stack->empty()) { - if (undo_) { - // Get a session to iterate over. - kv_undo_stack->push(); - } - } - - ~manage_stack(){ - if (undo_) { - kv_undo_stack_->undo(); - } - } - private: - const kv_undo_stack_ptr& kv_undo_stack_; - const bool undo_; - }; -} - -template -bool read_rocksdb_entry(const eosio::session::shared_bytes& actual_db_kv_key, - const eosio::session::shared_bytes& value, - F& function) { - uint64_t contract; - constexpr std::size_t type_size = 1; - std::size_t key_prefix_size = type_size + sizeof(contract); - EOS_ASSERT(actual_db_kv_key.size() >= key_prefix_size, database_exception, "Unexpected key in rocksdb"); - - auto key_buffer = std::vector{ actual_db_kv_key.data(), actual_db_kv_key.data() + actual_db_kv_key.size() }; - auto begin = std::begin(key_buffer) + type_size; - auto end = std::begin(key_buffer) + key_prefix_size; - b1::chain_kv::extract_key(begin, end, contract); - - const char* post_contract = actual_db_kv_key.data() + key_prefix_size; - const std::size_t remaining = actual_db_kv_key.size() - key_prefix_size; - return function(contract, post_contract, remaining, value.data(), value.size()); -}; - -struct table_id_object_view { - account_name code; //< code should not be changed within a chainbase modifier lambda - scope_name scope; //< scope should not be changed within a chainbase modifier lambda - table_name table; //< table should not be changed within a chainbase modifier lambda - account_name payer; - uint32_t count = 0; /// the number of elements in the table -}; - -struct blob { - std::string str_data; - std::size_t size() const { return str_data.size(); } - const char* data() const { return str_data.data(); } -}; - -template -inline DataStream &operator<<(DataStream &ds, const blob &b) { - fc::raw::pack(ds, b.str_data); - return ds; -} - -template -inline DataStream &operator>>(DataStream &ds, blob &b) { - fc::raw::unpack(ds, b.str_data); - return ds; -} - -struct primary_index_view { - static primary_index_view create(uint64_t key, const char* value, std::size_t value_size) { - backing_store::payer_payload pp(value, value_size); - return primary_index_view{key, pp.payer, {std::string{pp.value, pp.value + pp.value_size}}}; - } - uint64_t primary_key; - account_name payer; - blob value; -}; - -template -struct secondary_index_view { - uint64_t primary_key; - account_name payer; - SecondaryKeyType secondary_key; -}; - -template -class add_database_receiver { -public: - add_database_receiver(Receiver& receiver, const chainbase::database& db) : receiver_(receiver), db_(db) {} - template - void add_row(const Object& row) { - receiver_.add_row(row, db_); - } - - void add_row(fc::unsigned_int x){ - receiver_.add_row(x, db_); - } - -private: - Receiver& receiver_; - const chainbase::database& db_; -}; - -// used to wrap a rocksdb_contract_db_table_writer to collect all primary and secondary keys and report them after the -// table_id_object_view is reported. This also reports contract table related data in the format expected by snapshots -// (reporting the number of key rows (for primary and secondary key types) before reporting the rows. NOTE: using this -// collector really only makes sense with a key_context of complete, otherwise it is just extra processing with no -// benefit. It will produce invalid output if used with a key_context of complete_reverse -template -class rocksdb_whole_db_table_collector { -public: - rocksdb_whole_db_table_collector(Receiver &r) - : receiver_(r) {} - - void add_row(const chain::backing_store::primary_index_view& row){ - primary_indices_.emplace_back(row); - } - - void add_row(const chain::backing_store::secondary_index_view& row){ - secondary_indices().emplace_back(row); - } - - void add_row(const chain::backing_store::secondary_index_view& row){ - secondary_indices().emplace_back(row); - } - - void add_row(const chain::backing_store::secondary_index_view& row){ - secondary_indices().emplace_back(row); - } - - void add_row(const chain::backing_store::secondary_index_view& row){ - secondary_indices().emplace_back(row); - } - - void add_row(const chain::backing_store::secondary_index_view& row){ - secondary_indices().emplace_back(row); - } - - void add_row(const chain::backing_store::table_id_object_view& row){ - receiver_.add_row(row); - - auto add = [&rec=receiver_](const auto &row) { rec.add_row(row); }; - add(fc::unsigned_int(primary_indices_.size())); - // now that the whole table has been found, now we can report the individual keys - std::for_each(primary_indices_.begin(), primary_indices_.end(), add); - primary_indices_.clear(); - std::tuple secondary_key_types; - std::apply([this](auto... index) { (this->write_secondary_indices(index), ...); }, - secondary_key_types); - } -private: - template - auto& secondary_indices() { - return std::get>>(all_secondary_indices_); - } - - template - void write_secondary_indices(IndexType) { - auto& indices = secondary_indices(); - auto add = [this](const auto& row) { receiver_.add_row(row); }; - add(fc::unsigned_int(indices.size())); - std::for_each(indices.begin(), indices.end(), add); - indices.clear(); - } - - Receiver& receiver_; - std::vector primary_indices_; - template - using secondary_index_views = std::vector>; - std::tuple, - secondary_index_views, - secondary_index_views, - secondary_index_views, - secondary_index_views > - all_secondary_indices_; -}; - -constexpr auto process_all = []() { return true; }; - -// the context in which the table writer should be reporting to receiver_.add_row(...). NOTE: for any context indicating -// that it reports complete/valid table_id_object_views, for it to be valid it must be passed all keys belonging to that -// table, if not, the count portion of the table_id_object_view will not be accurate -enum class key_context { - complete, // report keys (via receiver_.add_row) as they are seen, so table will be after its keys (use rocksdb_whole_db_table_collector to reverse this) - complete_reverse, // report keys (via receiver_.add_row) as they are seen, used when reverse iterating through a table space (do not use with rocksdb_whole_db_table_collector) - standalone, // report an incomplete table (only code/scope/table valid) prior to reporting its keys - standalone_reverse, // report an incomplete table (only code/scope/table valid) prior to reporting its keys - table_only, // report only the table, primary and secondary keys are only processed enough to report a complete/valid table_id_object_view - table_only_reverse // report only the table, primary and secondary keys are only processed enough to report a complete/valid table_id_object_view -}; - -// processes any keys passed to it, reporting them to its receiver as it sees them. Use a key_context to adjust this behavior, -// pass in a keep_processing lambda to indicate to one of the walk_*** methods that processing should stop (like for -// limiting time querying the database for an PRC call) -template> -class rocksdb_contract_db_table_writer { -public: - using key_type = db_key_value_format::key_type; - - rocksdb_contract_db_table_writer(Receiver &r, key_context context, Function keep_processing = process_all) - : receiver_(r), context_(context), keep_processing_(keep_processing) {} - - explicit rocksdb_contract_db_table_writer(Receiver &r) - : receiver_(r), context_(key_context::complete), keep_processing_(process_all) {} - - void extract_primary_index(b1::chain_kv::bytes::const_iterator remaining, - b1::chain_kv::bytes::const_iterator key_end, const char *value, - std::size_t value_size) { - if (!is_table_only()) { - uint64_t primary_key; - EOS_ASSERT(b1::chain_kv::extract_key(remaining, key_end, primary_key), bad_composite_key_exception, - "DB intrinsic key-value store composite key is malformed"); - receiver_.add_row(primary_index_view::create(primary_key, value, value_size)); - } - ++primary_count_; - } - - template - void extract_secondary_index(b1::chain_kv::bytes::const_iterator remaining, - b1::chain_kv::bytes::const_iterator key_end, const char *value, - std::size_t value_size) { - if (!is_table_only()) { - IndexType secondary_key; - uint64_t primary_key; - EOS_ASSERT(b1::chain_kv::extract_key(remaining, key_end, secondary_key), bad_composite_key_exception, - "DB intrinsic key-value store composite key is malformed"); - EOS_ASSERT(b1::chain_kv::extract_key(remaining, key_end, primary_key), bad_composite_key_exception, - "DB intrinsic key-value store composite key is malformed"); - backing_store::payer_payload pp(value, value_size); - receiver_.add_row(secondary_index_view{primary_key, pp.payer, secondary_key}); - } - ++secondary_count(); - } - - // DB Intrinsic Keys - // PREFIX: - // db intrinsic keys are all prefaced with the following information: - // | contract | scope | table | type of key | - // so when iterating through the database, all keys for a given contract/scope/table will be processed together - // - // TYPE OF KEY: - // the "type of key" is an enum which ends up creating the following order for the type of keys: - // primary key - // primary to secondary key (not present in chainbase structure since it can have more than one index on the same structure -- these are ignored for this class's processing) - // uint64_t secondary key - // uint128_t secondary key - // key256_t secondary key - // float64_t secondary key - // float128_t secondary key - // table key (indicates end of table and payer) - // - // KEY PROCESSING (key_context): - // keys may be processed as: - // individual key (standalone) - // -- this can be any key - primary, secondary, or table - // table only (table_only, table_only_reversed) - // -- all keys for a table are passed in and processed only to report a valid table_id_object_view, if the table is - // passed in table_only, the table is determined to be complete when the table key is received, for - // table_only_reversed all of the table_id_object_view data, other than the count, is determined when the - // table key is received and the count is determined when the table is determined to be complete (either by a - // new key being received that is for a different table, or the complete() method is called externally). - // -- only the the table_id_object_view is reported, none of the other key's objects are reported - // complete (complete, complete_reverse) - // -- similar to table only, except all of the objects for the keys in the table are also reported - bool operator()(uint64_t contract, const char *key, std::size_t key_size, - const char *value, std::size_t value_size) { - b1::chain_kv::bytes composite_key(key, key + key_size); - auto[scope, table, remaining, type] = backing_store::db_key_value_format::get_prefix_thru_key_type( - composite_key); - - const name code = name{contract}; - if (!is_same_table(code, scope, table)) { - // since the table is being traversed in reverse order and - // we have identified a key for a new table, need to add - // the key count for the completed table and report it - complete(); - } - - if (!keep_processing_()) { - // performing the check after retrieving the prefix to provide similar results for RPC calls - // (indicating next scope) - stopped_processing_ = true; - table_context_ = table_id_object_view{code, scope, table, name{}, 0}; - return false; - } - - if (type == key_type::table) { - backing_store::payer_payload pp(value, value_size); - - - if (is_reversed()) { - // since table is reversed, we see the table key first, - // so we have the whole table context, except for the - // count of keys contained in the table, that will need - // to be added later once the next table is identified - // (so that we know that this table is - // complete - see complete()) - table_context_ = table_id_object_view{code, scope, table, pp.payer, 0}; - } - else { - table_context_ = table_id_object_view{code, scope, table, pp.payer, total_count()}; - receiver_.add_row(*table_context_); - } - - } else if (type != key_type::primary_to_sec) { - // for individual keys or reversed, need to report the table info for reference - check_context(code, scope, table); - std::invoke(extract_index_member_fun_[static_cast(type)], this, remaining, composite_key.end(), - value, value_size); - } - return true; - } - - std::optional stopped_at() const { - if (stopped_processing_) { - return table_context_; - } - return std::optional(); - } - - bool is_reversed() const { - return context_ == key_context::complete_reverse || context_ == key_context::table_only_reverse || context_ == key_context::standalone_reverse; - }; - - bool is_standalone() const { - return context_ == key_context::standalone || context_ == key_context::standalone_reverse; - }; - - bool is_table_only() const { - return context_ == key_context::table_only || context_ == key_context::table_only_reverse; - }; - - // called to indicate processing is complete (to allow completion of reversed table processing) - void complete() { - if (is_reversed() && table_context_ && !stopped_processing_) { - table_context_->count = total_count(); - receiver_.add_row(*table_context_); - } - } -private: - template - uint32_t& secondary_count() { - constexpr auto index = static_cast(db_key_value_format::derive_secondary_key_type()) - - static_cast(db_key_value_format::key_type::sec_i64); - return all_secondary_indices_count_[index]; - } - - void check_context(const name& code, const name& scope, const name& table) { - // this method only has to do with standalone processing - if (!is_standalone() || is_same_table(code, scope, table)) { - return; - } - table_context_ = table_id_object_view{code, scope, table, name{}, 0}; - receiver_.add_row(*table_context_); - } - - // calculates the total_count of keys, and zeros out the counts - uint32_t total_count() { - uint32_t total = primary_count_; - for (auto& count : all_secondary_indices_count_) { - total += count; - count = 0; - } - primary_count_ = 0; - return total; - } - - bool is_same_table(const name& code, const name& scope, const name& table) const { - return table_context_ && - table_context_->code == code && - table_context_->scope == scope && - table_context_->table == table; - } - - Receiver &receiver_; - const key_context context_; - Function keep_processing_; - std::optional table_context_; - - using extract_index_member_fun_t = void (rocksdb_contract_db_table_writer::*)( - b1::chain_kv::bytes::const_iterator, b1::chain_kv::bytes::const_iterator, const char *, std::size_t); - constexpr static unsigned member_fun_count = static_cast(key_type::sec_long_double) + 1; - const static extract_index_member_fun_t extract_index_member_fun_[member_fun_count]; - uint32_t primary_count_ = 0; - constexpr static unsigned secondary_indices = static_cast(db_key_value_format::key_type::sec_long_double) - - static_cast(db_key_value_format::key_type::sec_i64) + 1; - uint32_t all_secondary_indices_count_[secondary_indices] = {}; - - bool stopped_processing_ = false; -}; - -// array mapping db_key_value_format::key_type values (except table) to functions to break down to component parts -template -const typename rocksdb_contract_db_table_writer::extract_index_member_fun_t - rocksdb_contract_db_table_writer::extract_index_member_fun_[] = { - &rocksdb_contract_db_table_writer::extract_primary_index, - nullptr, // primary_to_sec type is covered by writing the secondary key type - &rocksdb_contract_db_table_writer::extract_secondary_index, - &rocksdb_contract_db_table_writer::extract_secondary_index, - &rocksdb_contract_db_table_writer::extract_secondary_index, - &rocksdb_contract_db_table_writer::extract_secondary_index, - &rocksdb_contract_db_table_writer::extract_secondary_index, -}; - -template> -class rocksdb_contract_kv_table_writer { -public: - rocksdb_contract_kv_table_writer(Receiver& r, const Function& keep_processing = process_all) - : receiver_(r), keep_processing_(keep_processing) {} - - bool operator()(uint64_t contract, const char* key, std::size_t key_size, - const char* value, std::size_t value_size) { - if (!keep_processing_()) { - return false; - } - // In KV RocksDB, payer and actual data are packed together. - // Extract them. - backing_store::payer_payload pp(value, value_size); - kv_object_view row{name(contract), - {{key, key + key_size}}, - {{pp.value, pp.value + pp.value_size}}, - pp.payer}; - receiver_.add_row(row); - return true; - } - -private: - Receiver& receiver_; - const Function& keep_processing_; -}; - -namespace detail { - // used to handle forward and reverse iteration and the limitations of no upper_bound - class iterator_pair { - public: - iterator_pair(const eosio::session::shared_bytes& begin_key, - const eosio::session::shared_bytes& end_key, - bool is_reverse, - kv_undo_stack_ptr::element_type::variant_type& session) : is_reverse_(is_reverse) { - EOS_ASSERT(begin_key < end_key, database_exception, "Invalid iterator_pair request: begin_key was greater than or equal to end_key."); - if (is_reverse_) { - current_ = session.lower_bound(end_key); - end_ = session.lower_bound(begin_key); - // since this is reverse iterating, then need to iterate backward if this is a greater-than iterator, - // to get a greater-than-or-equal reverse iterator - if (current_ == session.end() || (*current_).first > end_key) { - --current_; - EOS_ASSERT(current_ != session.end(), database_exception, "iterator_pair: failed to find lower bound of end_key"); - } - // since this is reverse iterating, then need to iterate backward to get a less-than iterator - --end_; - } - else { - current_ = session.lower_bound(begin_key); - end_ = session.lower_bound(end_key); - } - } - - bool valid() const { - return current_ != end_; - } - - void next() { - if (is_reverse_) - --current_; - else - ++current_; - } - - session_type::iterator::value_type get() const { - return *current_; - } - private: - const bool is_reverse_; - kv_undo_stack_ptr::element_type::variant_type::iterator current_; - kv_undo_stack_ptr::element_type::variant_type::iterator end_; - }; - - template - bool is_reversed(const rocksdb_contract_db_table_writer& writer) { - return writer.is_reversed(); - } - - template - bool is_reversed(const Function& ) { - return false; - } - - template - void complete(rocksdb_contract_db_table_writer& writer) { - writer.complete(); - } - - template - void complete(Function& ) { - } - -} - -template -void walk_rocksdb_entries_with_prefix(const kv_undo_stack_ptr& kv_undo_stack, - const eosio::session::shared_bytes& begin_key, - const eosio::session::shared_bytes& end_key, - F& function) { - auto session = kv_undo_stack->top(); - - detail::iterator_pair iter_pair(begin_key, end_key, detail::is_reversed(function), session); - bool keep_processing = true; - for (; keep_processing && iter_pair.valid(); iter_pair.next()) { - const auto data = iter_pair.get(); - // iterating through the session will always return a valid value - keep_processing = read_rocksdb_entry(data.first, *data.second, function); - } - // indicate processing is done - detail::complete(function); -}; - -// will walk through all entries with the given prefix, so if passed an exact key, it will match that key -// and any keys with that key as a prefix -template > -bool process_rocksdb_entry(Session& session, - const eosio::session::shared_bytes& key, - Receiver& receiver) { - if (!key) { - return false; - } - const auto value = session.read(key); - if (!value) { - return false; - } - const char prefix = key[0]; - if (prefix == rocksdb_contract_kv_prefix) { - rocksdb_contract_kv_table_writer kv_writer(receiver); - read_rocksdb_entry(key, *value, kv_writer); - } - else { - if (prefix != rocksdb_contract_db_prefix) { - char buffer[10]; - const auto len = sprintf(buffer, "%02x", static_cast(prefix)); - buffer[len] = '\0'; - FC_THROW_EXCEPTION(bad_composite_key_exception, - "Passed in key is prefixed with: ${prefix} which is neither the DB or KV prefix", - ("prefix", buffer)); - } - - rocksdb_contract_db_table_writer db_writer(receiver, key_context::standalone); - read_rocksdb_entry(key, *value, db_writer); - } - return true; -} - -template -const char* contract_table_type() { - if constexpr (std::is_same_v) { - return "kv_object_view"; - } - else if constexpr (std::is_same_v) { - return "table_id_object_view"; - } - else if constexpr (std::is_same_v) { - return "primary_index_view"; - } - else if constexpr (std::is_same_v>) { - return "secondary_index_view"; - } - else if constexpr (std::is_same_v>) { - return "secondary_index_view"; - } - else if constexpr (std::is_same_v>) { - return "secondary_index_view"; - } - else if constexpr (std::is_same_v>) { - return "secondary_index_view"; - } - else { - static_assert(std::is_same_v>); - return "secondary_index_view"; - } -} - -template -struct single_type_error_receiver { - template - void add_row(const Object& row) { - if constexpr (std::is_same_v) { - static_cast(this)->add_table_row(row); - } else if constexpr (std::is_same_v) { - static_cast(this)->add_only_row(row); - } else { - FC_THROW_EXCEPTION(Exception, "Invariant failure, should not receive an add_row call of type: ${type}", - ("type", contract_table_type>())); - } - } -}; - -template -struct table_only_error_receiver { - template - void add_row(const Object& row) { - if constexpr (std::is_same_v) { - static_cast(this)->add_table_row(row); - } else { - FC_THROW_EXCEPTION(Exception, "Invariant failure, should not receive an add_row call of type: ${type}", - ("type", contract_table_type>())); - } - } -}; - -}}} - -FC_REFLECT(eosio::chain::backing_store::table_id_object_view, (code)(scope)(table)(payer)(count) ) -FC_REFLECT(eosio::chain::backing_store::primary_index_view, (primary_key)(payer)(value) ) -REFLECT_SECONDARY(eosio::chain::backing_store::secondary_index_view) -REFLECT_SECONDARY(eosio::chain::backing_store::secondary_index_view) -REFLECT_SECONDARY(eosio::chain::backing_store::secondary_index_view) -REFLECT_SECONDARY(eosio::chain::backing_store::secondary_index_view) -REFLECT_SECONDARY(eosio::chain::backing_store::secondary_index_view) - -namespace fc { - inline - void to_variant( const eosio::chain::backing_store::blob& b, variant& v ) { - v = variant(base64_encode(b.str_data.data(), b.str_data.size())); - } - - inline - void from_variant( const variant& v, eosio::chain::backing_store::blob& b ) { - b.str_data = base64_decode(v.as_string()); - } -} diff --git a/libraries/chain/include/eosio/chain/backing_store/db_context.hpp b/libraries/chain/include/eosio/chain/backing_store/db_context.hpp index c171992fb8..c458bb4278 100644 --- a/libraries/chain/include/eosio/chain/backing_store/db_context.hpp +++ b/libraries/chain/include/eosio/chain/backing_store/db_context.hpp @@ -10,7 +10,6 @@ namespace chainbase { namespace eosio { namespace session { - struct rocksdb_t; template class session; @@ -215,7 +214,4 @@ namespace backing_store { }; std::unique_ptr create_db_chainbase_context(apply_context& context, name receiver); - std::unique_ptr create_db_rocksdb_context(apply_context& context, name receiver, - eosio::session::session_variant, eosio::session::session>> session); - }}} // ns eosio::chain::backing_store diff --git a/libraries/chain/include/eosio/chain/backing_store/db_key_value_any_lookup.hpp b/libraries/chain/include/eosio/chain/backing_store/db_key_value_any_lookup.hpp deleted file mode 100644 index 56dbf177cb..0000000000 --- a/libraries/chain/include/eosio/chain/backing_store/db_key_value_any_lookup.hpp +++ /dev/null @@ -1,78 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace eosio { namespace chain { namespace backing_store { - struct db_context; - using end_of_prefix = db_key_value_format::end_of_prefix; - - struct key_bundle { - key_bundle(const b1::chain_kv::bytes& ck, name code); - - eosio::session::shared_bytes full_key; - }; - - struct prefix_bundle { - prefix_bundle(const b1::chain_kv::bytes& composite_key, end_of_prefix prefix_end, name code); - - eosio::session::shared_bytes full_key; - eosio::session::shared_bytes prefix_key; - }; - - template - struct value_bundle : Key { - value_bundle(Key&& key, std::optional&& v) : Key(std::move(key)), value(std::move(v)) {} - value_bundle(const Key& key, std::optional&& v) : Key(key), value(std::move(v)) {} - std::optional value; - }; - - using kv_bundle = value_bundle; - using pv_bundle = value_bundle; - - struct db_key_value_any_lookup { - using session_type = eosio::session::session>; - using session_variant_type = eosio::session::session_variant; - using shared_bytes = eosio::session::shared_bytes; - - db_key_value_any_lookup(db_context& c, session_variant_type session) : parent(c), current_session(session) {} - - static key_bundle get_slice(name code, name scope, name table); - static key_bundle get_table_end_slice(name code, name scope, name table); - void add_table_if_needed(const shared_bytes& key, account_name payer); - void remove_table_if_empty(const shared_bytes& key); - template - int32_t get_end_iter(name code, name scope, name table, IterStore& iter_store) { - const auto table_key = get_table_end_slice(code, scope, table); - auto value = current_session.read(table_key.full_key); - if (!value) { - return iter_store.invalid_iterator(); - } - - const unique_table t { code, scope, table }; - const auto table_ei = iter_store.cache_table(t); - return table_ei; - } - - bool match_prefix(const shared_bytes& shorter, const shared_bytes& longer); - bool match_prefix(const shared_bytes& shorter, const session_variant_type::iterator& iter); - - bool match(const shared_bytes& lhs, const shared_bytes& rhs); - bool match(const shared_bytes& lhs, const session_variant_type::iterator& iter); - - db_context& parent; - session_variant_type current_session; - static constexpr int64_t table_overhead = config::billable_size_v; - static constexpr int64_t overhead = config::billable_size_v; - // this is used for any value that just needs something in it to distinguish it from the invalid value - static const shared_bytes useless_value; - }; - -}}} // ns eosio::chain::backing_store diff --git a/libraries/chain/include/eosio/chain/backing_store/db_key_value_format.hpp b/libraries/chain/include/eosio/chain/backing_store/db_key_value_format.hpp deleted file mode 100644 index 0edcbc143c..0000000000 --- a/libraries/chain/include/eosio/chain/backing_store/db_key_value_format.hpp +++ /dev/null @@ -1,581 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include -#include - -namespace eosio { - namespace session { - class shared_bytes; - } - -namespace chain { namespace backing_store { namespace db_key_value_format { - using key256_t = std::array; - - // NOTE: very limited use till redesign - constexpr uint64_t db_type_and_code_size = sizeof(char) + sizeof(name); // 1 (db type) + 8 (contract) - - b1::chain_kv::bytes create_primary_key(name scope, name table, uint64_t primary_key); - - bool get_primary_key(const b1::chain_kv::bytes& composite_key, name& scope, name& table, uint64_t& primary_key); - - // ordering affects key ordering - enum class key_type : char { - primary = 0, - primary_to_sec = 1, // used to lookup the secondary keys for a specific primary key - sec_i64 = 2, - sec_i128 = 3, - sec_i256 = 4, - sec_double = 5, - sec_long_double = 6, // changes needed in determine_sec_type to accommodate new types - table = std::numeric_limits::max() // require to be highest value for a given scope/table - }; - - using intermittent_decomposed_values = std::tuple; - - namespace detail { - - template - struct determine_sec_type {}; - - template<> - struct determine_sec_type { static constexpr key_type kt = key_type::sec_i64; }; - - template<> - struct determine_sec_type { static constexpr key_type kt = key_type::sec_i128; }; - - template<> - struct determine_sec_type { static constexpr key_type kt = key_type::sec_i256; }; - - template<> - struct determine_sec_type { static constexpr key_type kt = key_type::sec_double; }; - - template<> - struct determine_sec_type { static constexpr key_type kt = key_type::sec_long_double; }; - - template - struct associated; - - template<> - struct associated { - using stored_key = eosio::session::shared_bytes; - }; - - template - struct associated { - using stored_key = b1::chain_kv::bytes; - }; - - template - constexpr std::size_t prefix_size() { - constexpr uint64_t legacy_prefix_size = sizeof(name) * 2; // 8 (scope) + 8 (table) - if constexpr (std::is_same_v) { - constexpr uint64_t db_type_and_code_size = sizeof(char) + sizeof(name); // 1 (db type) + 4 (contract) - return db_type_and_code_size + legacy_prefix_size; - } - else if constexpr (std::is_same_v) { - return legacy_prefix_size; - } - else { - static_assert(std::is_same_v); - return legacy_prefix_size; - } - } - - template - constexpr void consistent_keys() { - if constexpr (std::is_same_v) { - return; - } - else { - static_assert(!std::is_same_v); - static_assert(!std::is_same_v); - } - } - - constexpr std::size_t key_size(key_type kt); - - std::string to_string(const key_type& kt); - - b1::chain_kv::bytes prepare_composite_key_prefix(name scope, name table, std::size_t type_size, std::size_t key_size, std::size_t extension_size); - - b1::chain_kv::bytes prepare_composite_key(name scope, name table, std::size_t key_size, key_type kt, std::size_t extension_size); - - using intermittent_decomposed_prefix_values = std::tuple; - - intermittent_decomposed_prefix_values extract_from_composite_key_prefix(b1::chain_kv::bytes::const_iterator begin, b1::chain_kv::bytes::const_iterator key_end); - - intermittent_decomposed_values extract_from_composite_key(b1::chain_kv::bytes::const_iterator begin, b1::chain_kv::bytes::const_iterator key_end); - - template - void append_identifier_and_key(b1::chain_kv::bytes& composite_key, const Key& some_key, key_type kt = detail::determine_sec_type::kt) { - composite_key.push_back(static_cast(kt)); - b1::chain_kv::append_key(composite_key, some_key); - } - - template - void append_primary_and_secondary_key_types(b1::chain_kv::bytes& composite_key) { - composite_key.push_back(static_cast(key_type::primary_to_sec)); - // NOTE: db_key_value_any_lookup end_of_prefix::at_prim_to_sec_type tied to this layout - const key_type kt = determine_sec_type::kt; - composite_key.push_back(static_cast(kt)); - } - - template - void append_primary_and_secondary_keys(b1::chain_kv::bytes& composite_key, uint64_t primary_key, const Key& sec_key) { - append_primary_and_secondary_key_types(composite_key); - // NOTE: db_key_value_any_lookup end_of_prefix::at_prim_to_sec_primary_key tied to this layout - b1::chain_kv::append_key(composite_key, primary_key); - b1::chain_kv::append_key(composite_key, sec_key); - } - - template - bool validate_primary_to_sec_type_prefix(const CharKey& type_prefix, std::size_t start_prefix_size, key_type expected_sec_kt) { - // possibly add the extra_prefix_size (to cover db_type and contract for a full key) - const auto prefix_minimum_size = start_prefix_size + sizeof(key_type); - const auto both_types_size = prefix_minimum_size + sizeof(key_type); - const auto both_types_and_primary_size = both_types_size + sizeof(uint64_t); - const bool has_sec_type = type_prefix.size() >= both_types_size; - EOS_ASSERT(type_prefix.size() == prefix_minimum_size || type_prefix.size() == both_types_size || - type_prefix.size() == both_types_and_primary_size, bad_composite_key_exception, - "DB intrinsic key-value validate_primary_to_sec_type_prefix was passed a prefix key size: ${s1} " - "bytes that was not either: ${s2}, ${s3}, or ${s4} bytes long", - ("s1", type_prefix.size())("s2", prefix_minimum_size)("s3", both_types_and_primary_size) - ("s4", both_types_and_primary_size)); - const char* prefix_ptr = type_prefix.data() + start_prefix_size; - const key_type prefix_type_kt {*(prefix_ptr++)}; - EOS_ASSERT(key_type::primary_to_sec == prefix_type_kt, bad_composite_key_exception, - "DB intrinsic key-value validate_primary_to_sec_type_prefix was passed a prefix that was the wrong " - "type: ${type}, it should have been of type: ${type2}", - ("type", to_string(prefix_type_kt))("type2", to_string(key_type::primary_to_sec))); - if (has_sec_type) { - const key_type prefix_sec_kt {*(prefix_ptr)}; - EOS_ASSERT(expected_sec_kt == prefix_sec_kt, bad_composite_key_exception, - "DB intrinsic key-value validate_primary_to_sec_type_prefix was passed a prefix that was the " - "wrong secondary type: ${type}, it should have been of type: ${type2}", - ("type", to_string(prefix_sec_kt))("type2", to_string(expected_sec_kt))); - } - return has_sec_type; - } - - template - void validate_primary_to_sec_type_size(const CharKey& full_key, std::size_t start_prefix_size) { - const auto prefix_minimum_size = start_prefix_size + sizeof(key_type); - const auto both_types_size = prefix_minimum_size + sizeof(key_type); - const auto full_size = both_types_size + sizeof(uint64_t) + sizeof(Key); - EOS_ASSERT(full_key.size() == full_size, bad_composite_key_exception, - "DB intrinsic key-value validate_primary_to_sec_type_size was passed a full key size: ${s1} bytes that was not at least " - "larger than the size that would include the secondary type: ${s2}", - ("s1", full_key.size())("s2", full_size)); - } - - template - bool verify_primary_to_sec_type(const CharKey1& key, const CharKey2& type_prefix) { - detail::consistent_keys(); - - const std::size_t start_prefix_size = prefix_size(); - // >> validate that the type prefix is actually of primary_to_sec key type - const key_type expected_sec_kt = determine_sec_type::kt; - const auto check_sec_type = !validate_primary_to_sec_type_prefix(type_prefix, start_prefix_size, expected_sec_kt); - - // check if key is prefixed with type_prefix - if (memcmp(type_prefix.data(), key.data(), type_prefix.size()) != 0) { - return false; - } - if (check_sec_type) { - const key_type prefix_sec_kt {*(key.data() + start_prefix_size + sizeof(key_type))}; - // key was not the right sec type - if (expected_sec_kt != prefix_sec_kt) { - return false; - } - } - - validate_primary_to_sec_type_size(key, start_prefix_size); - return true; - } - - template - std::pair locate_trailing_primary_and_secondary_keys(const CharKey& key) { - // this method just points to the data, it expects validation to be done prior to calling - const auto both_type_size = prefix_size() + sizeof(key_type) + sizeof(key_type); - const auto primary_offset = key.data() + both_type_size; - return { rocksdb::Slice{primary_offset, sizeof(uint64_t)}, - rocksdb::Slice{primary_offset + sizeof(uint64_t), sizeof(Key)}}; - } - - template - void extract_trailing_primary_and_secondary_keys(const CharKey& key, uint64_t& primary_key, Key& sec_key) { - const auto key_offsets = locate_trailing_primary_and_secondary_keys(key); - const auto keys_sizes = key_offsets.first.size() + key_offsets.second.size(); - const auto start_offset = key_offsets.first.data(); - const b1::chain_kv::bytes composite_trailing_prim_sec_key {start_offset, start_offset + keys_sizes}; - auto composite_loc = composite_trailing_prim_sec_key.cbegin(); - EOS_ASSERT(b1::chain_kv::extract_key(composite_loc, composite_trailing_prim_sec_key.cend(), primary_key), bad_composite_key_exception, - "DB intrinsic key-value store invariant has changed, extract_key should only fail if the remaining " - "string size is less than the sizeof(uint64_t)"); - EOS_ASSERT(b1::chain_kv::extract_key(composite_loc, composite_trailing_prim_sec_key.cend(), sec_key), bad_composite_key_exception, - "DB intrinsic key-value store invariant has changed, extract_key should only fail if the remaining " - "string size is less than: ${s} bytes, which was already verified", ("s", key_offsets.second.size())); - EOS_ASSERT(composite_loc == composite_trailing_prim_sec_key.cend(), bad_composite_key_exception, - "DB intrinsic key-value store invariant has changed, calls to extract the pimary and secondary " - "keys should have resulted in the passed in buffer being consumed fully"); - } - - template - auto assemble(std::array&& data) { - if constexpr (std::is_same_v) { - return session::make_shared_bytes(std::move(data)); - } - else { - b1::chain_kv::bytes key_data; - const std::size_t init = 0; - const std::size_t length = std::accumulate(data.begin(), data.end(), init, - [](std::size_t a, const rocksdb::Slice& b) { return a + b.size(); }); - key_data.reserve(length); - for (const auto& d : data) { - key_data.insert(key_data.end(), d.data(), d.data() + d.size()); - } - return key_data; - } - } - } - - template - constexpr key_type derive_secondary_key_type() { - return detail::determine_sec_type::kt; - } - - template - b1::chain_kv::bytes create_secondary_key(name scope, name table, const Key& sec_key, uint64_t primary_key) { - const key_type kt = detail::determine_sec_type::kt; - b1::chain_kv::bytes composite_key = detail::prepare_composite_key(scope, table, sizeof(Key), kt, sizeof(primary_key)); - b1::chain_kv::append_key(composite_key, sec_key); - b1::chain_kv::append_key(composite_key, primary_key); - return composite_key; - } - - b1::chain_kv::bytes create_prefix_type_key(name scope, name table, key_type kt); - - template - b1::chain_kv::bytes create_prefix_secondary_key(name scope, name table, const Key& sec_key) { - const key_type kt = detail::determine_sec_type::kt; - b1::chain_kv::bytes composite_key = detail::prepare_composite_key(scope, table, sizeof(Key), kt, 0); - b1::chain_kv::append_key(composite_key, sec_key); - return composite_key; - } - - template - bool get_secondary_key(const b1::chain_kv::bytes& composite_key, name& scope, name& table, Key& sec_key, uint64_t& primary_key) { - b1::chain_kv::bytes::const_iterator composite_loc; - key_type kt = key_type::sec_i64; - std::tie(scope, table, composite_loc, kt) = detail::extract_from_composite_key(composite_key.cbegin(), composite_key.cend()); - constexpr static auto expected_kt = detail::determine_sec_type::kt; - if (kt != expected_kt) { - return false; - } - EOS_ASSERT(b1::chain_kv::extract_key(composite_loc, composite_key.cend(), sec_key), bad_composite_key_exception, - "DB intrinsic key-value store composite key is malformed, it is supposed to have an underlying: " + - detail::to_string(kt)); - EOS_ASSERT(b1::chain_kv::extract_key(composite_loc, composite_key.cend(), primary_key), bad_composite_key_exception, - "DB intrinsic key-value store composite key is malformed, it is supposed to have a trailing primary key"); - return true; - } - - struct secondary_key_pair { - b1::chain_kv::bytes secondary_key; - b1::chain_kv::bytes primary_to_secondary_key; - }; - - template - secondary_key_pair create_secondary_key_pair(name scope, name table, const Key& sec_key, uint64_t primary_key) { - b1::chain_kv::bytes secondary_key = create_secondary_key(scope, table, sec_key, primary_key); - b1::chain_kv::bytes primary_to_secondary_key(secondary_key.begin(), secondary_key.begin() + detail::prefix_size()); - detail::append_primary_and_secondary_keys(primary_to_secondary_key, primary_key, sec_key); - return { std::move(secondary_key), std::move(primary_to_secondary_key) }; - } - - template - CharKey create_secondary_key_from_primary_to_sec_key(const CharKey& primary_to_sec_key) { - // checking size first, to validate that the key is the correct size before it is split - const auto start_prefix_size = detail::prefix_size(); - detail::validate_primary_to_sec_type_size(primary_to_sec_key, start_prefix_size); - const auto both_types_and_primary_size = detail::prefix_size() + sizeof(key_type) + sizeof(key_type) + sizeof(uint64_t); - const rocksdb::Slice prefix {primary_to_sec_key.data(), both_types_and_primary_size}; - // now validating the front end of the key - const key_type expected_sec_kt = detail::determine_sec_type::kt; - // NOTE: using Slice, but it has the new db type and contract prefix - const auto sec_kt_checked = detail::validate_primary_to_sec_type_prefix(prefix, start_prefix_size, expected_sec_kt); - EOS_ASSERT(sec_kt_checked, bad_composite_key_exception, - "DB intrinsic invariant failure, key-value create_secondary_key_from_primary_to_sec_key was passed a " - "key that was verified as long enough but validate_primary_to_sec_type_prefix indicated that it wasn't " - "long enough to verify the secondary type"); - const auto key_offsets = detail::locate_trailing_primary_and_secondary_keys(primary_to_sec_key); - using return_type = typename detail::associated::stored_key; - const auto new_kt = static_cast(expected_sec_kt); - const char* new_kt_ptr = &new_kt; - // already have everything in char format/order, so just need to assemble it - return detail::assemble<4, return_type>({rocksdb::Slice{primary_to_sec_key.data(), start_prefix_size}, - rocksdb::Slice{new_kt_ptr, 1}, - key_offsets.second, - key_offsets.first}); - } - - template - bool get_trailing_sec_prim_keys(const CharKey1& full_key, const CharKey2& secondary_type_prefix, Key& sec_key, uint64_t& primary_key) { - detail::consistent_keys(); - - // >> verify that secondary_type_prefix is really a valid secondary type prefix - - // possibly add the extra_prefix_size (to cover db_type and contract for a full key) - const auto sec_type_prefix_size = secondary_type_prefix.size(); - const auto prefix_type_size = detail::prefix_size() + sizeof(key_type); - EOS_ASSERT(sec_type_prefix_size == prefix_type_size, bad_composite_key_exception, - "DB intrinsic key-value get_trailing_sec_prim_keys was passed a secondary_type_prefix with key size: ${s1} bytes " - "which is not equal to the expected size: ${s2}", ("s1", sec_type_prefix_size)("s2", prefix_type_size)); - const key_type expected_kt = detail::determine_sec_type::kt; - const key_type actual_kt {*(secondary_type_prefix.data() + sec_type_prefix_size - 1)}; - EOS_ASSERT(expected_kt == actual_kt, bad_composite_key_exception, - "DB intrinsic key-value get_trailing_sec_prim_keys was passed a key that was the wrong type: ${type}," - " it should have been of type: ${type2}", - ("type", detail::to_string(actual_kt))("type2", detail::to_string(expected_kt))); - const auto keys_size = sizeof(sec_key) + sizeof(primary_key); - EOS_ASSERT(full_key.size() == sec_type_prefix_size + keys_size, bad_composite_key_exception, - "DB intrinsic key-value get_trailing_sec_prim_keys was passed a full key size: ${s1} bytes that was not exactly " - "${s2} bytes (the size of the secondary key and a primary key) larger than the secondary_type_prefix " - "size: ${s3}", ("s1", full_key.size())("s2", keys_size)("s3", sec_type_prefix_size)); - - // >> identify if the passed in key matches the prefix - - const auto comp = memcmp(secondary_type_prefix.data(), full_key.data(), sec_type_prefix_size); - if (comp != 0) { - return false; - } - auto start_offset = full_key.data() + sec_type_prefix_size; - const b1::chain_kv::bytes composite_trailing_sec_prim_key {start_offset, start_offset + keys_size}; - auto composite_loc = composite_trailing_sec_prim_key.cbegin(); - EOS_ASSERT(b1::chain_kv::extract_key(composite_loc, composite_trailing_sec_prim_key.cend(), sec_key), bad_composite_key_exception, - "DB intrinsic key-value store invariant has changed, extract_key should only fail if the remaining " - "string size is less than: ${s} bytes, which was already verified", ("s", keys_size)); - EOS_ASSERT(b1::chain_kv::extract_key(composite_loc, composite_trailing_sec_prim_key.cend(), primary_key), bad_composite_key_exception, - "DB intrinsic key-value store invariant has changed, extract_key should only fail if the remaining " - "string size is less than the sizeof(uint64_t)"); - return true; - } - - bool get_trailing_primary_key(const rocksdb::Slice& full_key, const rocksdb::Slice& secondary_key_prefix, uint64_t& primary_key); - - template - b1::chain_kv::bytes create_primary_to_secondary_key(name scope, name table, uint64_t primary_key, const Key& sec_key) { - constexpr static auto type_size = sizeof(key_type); - auto composite_key = detail::prepare_composite_key_prefix(scope, table, type_size * 2, sizeof(primary_key), sizeof(sec_key)); - detail::append_primary_and_secondary_keys(composite_key, primary_key, sec_key); - return composite_key; - } - - template - b1::chain_kv::bytes create_prefix_primary_to_secondary_key(name scope, name table, uint64_t primary_key) { - constexpr static auto type_size = sizeof(key_type); - constexpr static auto zero_size = 0; - auto composite_key = detail::prepare_composite_key_prefix(scope, table, type_size * 2, sizeof(primary_key), zero_size); - detail::append_primary_and_secondary_key_types(composite_key); - b1::chain_kv::append_key(composite_key, primary_key); - return composite_key; - } - - template - b1::chain_kv::bytes create_prefix_primary_to_secondary_key(name scope, name table) { - constexpr static auto type_size = sizeof(key_type); - constexpr static auto zero_size = 0; - auto composite_key = detail::prepare_composite_key_prefix(scope, table, type_size * 2, zero_size, zero_size); - detail::append_primary_and_secondary_key_types(composite_key); - return composite_key; - } - - enum class prim_to_sec_type_result { invalid_type, wrong_sec_type, valid_type }; - template - prim_to_sec_type_result get_primary_to_secondary_keys(const b1::chain_kv::bytes& composite_key, name& scope, name& table, uint64_t& primary_key, Key& sec_key) { - b1::chain_kv::bytes::const_iterator composite_loc; - key_type kt = key_type::sec_i64; - std::tie(scope, table, composite_loc, kt) = detail::extract_from_composite_key(composite_key.cbegin(), composite_key.cend()); - if (kt != key_type::primary_to_sec) { - return prim_to_sec_type_result::invalid_type; - } - EOS_ASSERT(composite_loc != composite_key.cend(), bad_composite_key_exception, - "DB intrinsic key-value store composite key is malformed, it does not contain an indication of the " - "type of the secondary-key"); - const key_type sec_kt{*composite_loc++}; - constexpr static auto expected_kt = detail::determine_sec_type::kt; - if (sec_kt != expected_kt) { - return prim_to_sec_type_result::wrong_sec_type; - } - EOS_ASSERT(b1::chain_kv::extract_key(composite_loc, composite_key.cend(), primary_key), bad_composite_key_exception, - "DB intrinsic key-value store composite key is malformed, it is supposed to have a trailing primary key"); - - EOS_ASSERT(b1::chain_kv::extract_key(composite_loc, composite_key.cend(), sec_key), bad_composite_key_exception, - "DB intrinsic key-value store composite key is malformed, it is supposed to have an underlying: " + - detail::to_string(sec_kt)); - return prim_to_sec_type_result::valid_type; - } - - template - bool get_trailing_prim_sec_keys(const CharKey1& full_key, const CharKey2& primary_to_sec_type_prefix, - uint64_t& primary_key, Key& sec_key) { - detail::consistent_keys(); - - // >> verify that secondary_type_prefix is really a valid secondary type prefix - - // possibly add the extra_prefix_size (to cover db_type and contract for a full key) - if (!detail::verify_primary_to_sec_type(full_key, primary_to_sec_type_prefix)) { - return false; - } - - detail::extract_trailing_primary_and_secondary_keys(full_key, primary_key, sec_key); - return true; - } - - template - bool get_trailing_secondary_key(const CharKey1& full_key, const CharKey2& sec_type_trailing_prefix, Key& sec_key) { - detail::consistent_keys(); - // possibly add the extra_prefix_size (to cover db_type and contract for a full key) - const auto sec_type_trailing_prefix_size = sec_type_trailing_prefix.size(); - const auto prefix_with_primary_key_size = detail::prefix_size() + sizeof(key_type) * 2 + sizeof(uint64_t); - EOS_ASSERT(sec_type_trailing_prefix_size == prefix_with_primary_key_size, bad_composite_key_exception, - "DB intrinsic key-value get_trailing_secondary_key was passed a sec_type_trailing_prefix with key size: ${s1} bytes " - "which is not equal to the expected size: ${s2}", ("s1", sec_type_trailing_prefix_size)("s2", prefix_with_primary_key_size)); - if (!detail::verify_primary_to_sec_type(full_key, sec_type_trailing_prefix)) { - return false; - } - const auto key_size = sizeof(Key); - auto start_offset = full_key.data() + sec_type_trailing_prefix_size; - const b1::chain_kv::bytes composite_trailing_prim_sec_key {start_offset, start_offset + key_size}; - auto composite_loc = composite_trailing_prim_sec_key.cbegin(); - EOS_ASSERT(b1::chain_kv::extract_key(composite_loc, composite_trailing_prim_sec_key.cend(), sec_key), bad_composite_key_exception, - "DB intrinsic key-value store invariant has changed, extract_key should only fail if the remaining " - "string size is less than: ${s} bytes, which was already verified", ("s", key_size)); - return true; - } - - b1::chain_kv::bytes create_table_key(name scope, name table); - - eosio::session::shared_bytes create_table_key(const eosio::session::shared_bytes& prefix_key); - - bool get_table_key(const b1::chain_kv::bytes& composite_key, name& scope, name& table); - - b1::chain_kv::bytes create_prefix_key(name scope, name table); - - void get_prefix_key(const b1::chain_kv::bytes& composite_key, name& scope, name& table); - - template - rocksdb::Slice prefix_slice(const CharKey& composite_key) { - detail::consistent_keys(); - const auto prefix_size = detail::prefix_size(); - EOS_ASSERT(composite_key.size() >= prefix_size, bad_composite_key_exception, - "Cannot extract a prefix from a key that is not big enough to contain a prefix"); - return rocksdb::Slice(composite_key.data(), prefix_size); - } - - template - rocksdb::Slice prefix_type_slice(const CharKey& composite_key) { - detail::consistent_keys(); - const auto prefix_type_size = detail::prefix_size() + sizeof(key_type); - EOS_ASSERT(composite_key.size() >= prefix_type_size, bad_composite_key_exception, - "Cannot extract a prefix from a key that is not big enough to contain a prefix"); - return rocksdb::Slice(composite_key.data(), prefix_type_size); - } - - template - rocksdb::Slice prefix_primary_to_secondary_slice(const CharKey& composite_key, bool with_primary_key) { - detail::consistent_keys(); - // possibly add the extra_prefix_size (to cover db_type and contract for a full key) - static constexpr auto prefix_size = detail::prefix_size(); - static constexpr auto prefix_type_size = prefix_size + sizeof(key_type); - auto expected_size = prefix_type_size + sizeof(key_type) + (with_primary_key ? sizeof(uint64_t) : 0); - EOS_ASSERT(composite_key.size() >= expected_size, bad_composite_key_exception, - "Cannot extract a primary-to-secondary key prefix from a key that is not big enough to contain a one"); - const char* offset = composite_key.data() + prefix_size; - EOS_ASSERT(key_type{*offset++} == key_type::primary_to_sec, bad_composite_key_exception, - "Cannot extract a primary-to-secondary key prefix from a key that is not of that type"); - const key_type sec_type { *offset }; - EOS_ASSERT(sec_type >= key_type::sec_i64 && sec_type <= key_type::sec_long_double, - bad_composite_key_exception, - "Invariant failure, composite_key is of the correct type, but the format of the key after that is incorrect"); - return rocksdb::Slice(composite_key.data(), expected_size); - } - - key_type extract_key_type(const b1::chain_kv::bytes& composite_key); - - key_type extract_primary_to_sec_key_type(const b1::chain_kv::bytes& composite_key); - - intermittent_decomposed_values get_prefix_thru_key_type(const b1::chain_kv::bytes& composite_key); - - b1::chain_kv::bytes extract_legacy_key(const eosio::session::shared_bytes& complete_key); - - rocksdb::Slice extract_legacy_slice(const eosio::session::shared_bytes& complete_key); - - eosio::session::shared_bytes create_full_key(const b1::chain_kv::bytes& ck, name code); - enum class end_of_prefix { pre_type, at_type, at_prim_to_sec_type, at_prim_to_sec_primary_key }; - eosio::session::shared_bytes create_full_key_prefix(const eosio::session::shared_bytes& ck, end_of_prefix prefix_end); - - struct full_key_data { - char db_type; - std::optional contract; - std::optional legacy_key; - std::optional scope; - std::optional table; - std::optional kt; - rocksdb::Slice type_prefix; - }; - full_key_data parse_full_key(const eosio::session::shared_bytes& full_key); - - template - bool get_primary_key(const CharKey1& full_key, const CharKey2& type_prefix, uint64_t& primary_key) { - detail::consistent_keys(); - // possibly add the extra_prefix_size (to cover db_type and contract for a full key) - const auto type_prefix_size = type_prefix.size(); - const auto prefix_type_size = detail::prefix_size() + sizeof(key_type); - EOS_ASSERT(type_prefix_size == prefix_type_size, bad_composite_key_exception, - "DB intrinsic key-value get_primary_key was passed a type_prefix with key size: ${s1} bytes " - "which is not equal to the expected size: ${s2}", ("s1", type_prefix_size)("s2", prefix_type_size)); - const auto key_size = sizeof(primary_key); - EOS_ASSERT(full_key.size() == type_prefix_size + key_size, bad_composite_key_exception, - "DB intrinsic key-value get_primary_key was passed a full key size: ${s1} bytes that was not exactly " - "${s2} bytes (the size of a primary key) larger than the type_prefix size: ${s3}", - ("s1", full_key.size())("s2", key_size)("s3", type_prefix_size)); - const auto comp = memcmp(type_prefix.data(), full_key.data(), type_prefix_size); - if (comp != 0) { - return false; - } - const key_type actual_kt {*(type_prefix.data() + type_prefix_size - 1)}; - EOS_ASSERT(key_type::primary == actual_kt, bad_composite_key_exception, - "DB intrinsic key-value get_primary_key was passed a key that was the wrong type: ${type}," - " it should have been of type: ${type2}", - ("type", detail::to_string(actual_kt))("type2", detail::to_string(key_type::primary))); - auto start_offset = full_key.data() + type_prefix_size; - const b1::chain_kv::bytes composite_trailing_prim_key {start_offset, start_offset + key_size}; - auto composite_loc = composite_trailing_prim_key.cbegin(); - EOS_ASSERT(b1::chain_kv::extract_key(composite_loc, composite_trailing_prim_key.cend(), primary_key), bad_composite_key_exception, - "DB intrinsic key-value get_primary_key invariant has changed, extract_key should only fail if the " - "remaining string size is less than the sizeof(uint64_t)"); - return true; - } - - eosio::session::shared_bytes create_full_primary_key(name code, name scope, name table, uint64_t primary_key); - eosio::session::shared_bytes create_full_prefix_key(name code, name scope, name table, std::optional kt = std::optional{}); - - template - eosio::session::shared_bytes create_full_secondary_key(name code, name scope, name table, const Key& sec_key, uint64_t primary_key) { - bytes composite_key = create_secondary_key(scope, table, sec_key, primary_key); - return create_full_key(composite_key, code); - } - - template - eosio::session::shared_bytes create_full_prefix_secondary_key(name code, name scope, name table, const Key& sec_key) { - bytes composite_key = create_prefix_secondary_key(scope, table, sec_key); - return create_full_key(composite_key, code); - } - -}}}} // ns eosio::chain::backing_store::db_key_value_format diff --git a/libraries/chain/include/eosio/chain/backing_store/db_key_value_iter_store.hpp b/libraries/chain/include/eosio/chain/backing_store/db_key_value_iter_store.hpp deleted file mode 100644 index d9f921325f..0000000000 --- a/libraries/chain/include/eosio/chain/backing_store/db_key_value_iter_store.hpp +++ /dev/null @@ -1,160 +0,0 @@ -#pragma once -#include -#include -#include -#include - -namespace eosio { namespace chain { namespace backing_store { -struct unique_table { - name contract; - name scope; - name table; -}; - -template -struct secondary_key { - int table_ei; - T secondary; - uint64_t primary; - account_name payer; -}; - -inline bool operator<(const unique_table& lhs, const unique_table& rhs) { - return std::tie(lhs.contract, lhs.scope, lhs.table) < std::tie(rhs.contract, rhs.scope, rhs.table); -} - -template -bool operator<(const secondary_key& lhs, const secondary_key& rhs) { - // checking primary second to optimize the search since a given primary key - // cannot have more than one secondary key with the same value (but keeping - // remainder of algorithm for consistency) - return std::tie(lhs.table_ei, lhs.primary, lhs.secondary) < std::tie(rhs.table_ei, rhs.primary, rhs.secondary); - // ignoring payer, it does not contribute to uniqueness -} - -template -class db_key_value_iter_store { - public: - using secondary_obj_type = secondary_key; - using secondary_key_type = SecondaryKey; - - db_key_value_iter_store(){ - _end_iterator_to_table.reserve(8); - _iterator_to_object.reserve(32); - } - - constexpr int invalid_iterator() const { return -1; } - - /// Returns end iterator of the table. - int cache_table( const unique_table& tobj ) { - auto itr = _table_cache.find(tobj); - if( itr != _table_cache.end() ) - return itr->second; - - auto ei = index_to_end_iterator(_end_iterator_to_table.size()); - _end_iterator_to_table.push_back( tobj ); - _table_cache.emplace( tobj, ei ); - return ei; - } - - int get_end_iterator_by_table( const unique_table& tobj )const { - auto itr = _table_cache.find(tobj); - EOS_ASSERT( itr != _table_cache.end(), table_not_in_cache, "an invariant was broken, table should be in cache" ); - return itr->second; - } - - const unique_table* find_table_by_end_iterator( int ei )const { - EOS_ASSERT( ei < invalid_iterator(), invalid_table_iterator, "not an end iterator" ); - auto indx = end_iterator_to_index(ei); - if( indx >= _end_iterator_to_table.size() ) return nullptr; - return &_end_iterator_to_table[indx]; - } - - const unique_table& get_table( const secondary_obj_type& obj )const { - EOS_ASSERT( obj.table_ei < invalid_iterator(), invalid_table_iterator, "not an end iterator" ); - auto indx = end_iterator_to_index(obj.table_ei); - EOS_ASSERT( indx < _end_iterator_to_table.size(), table_not_in_cache, - "an invariant was broken, secondary object references table that is not in cache" ); - return _end_iterator_to_table[indx]; - } - - const secondary_obj_type& get( int iterator ) const { - validate_object_iterator(iterator, "dereference of end iterator"); - // grab a const ref, to ensure that we are returning a reference to the actual object in the vector - const auto& result = _iterator_to_object[iterator]; - EOS_ASSERT( result, table_operation_not_permitted, "dereference of deleted object" ); - return *result; - } - - void remove( int iterator ) { - validate_object_iterator(iterator, "cannot call remove on end iterators"); - - auto optional_obj = _iterator_to_object[iterator]; - if( !optional_obj ) return; - _object_to_iterator.erase( *optional_obj ); - - _iterator_to_object[iterator].reset(); - } - - void swap( int iterator, const secondary_key_type& secondary, account_name payer ) { - validate_object_iterator(iterator, "cannot call swap on end iterators"); - - auto& optional_obj = _iterator_to_object[iterator]; - if( !optional_obj ) return; - // check to see if the object will actually change - if(optional_obj->payer == payer && optional_obj->secondary == secondary) return; - const bool map_key_change = optional_obj->secondary != secondary; - if (map_key_change) { - // edit object and swap out the map entry - _object_to_iterator.erase( *optional_obj ); - optional_obj->secondary = secondary; - } - optional_obj->payer = payer; - - if (map_key_change) { - _object_to_iterator.insert({ *optional_obj, iterator }); - } - } - - int find( const secondary_obj_type& obj ) const { - auto itr = _object_to_iterator.find( obj ); - if( itr != _object_to_iterator.end() ) - return itr->second; - - return invalid_iterator(); - } - - int add( const secondary_obj_type& obj ) { - auto itr = find( obj ); - if( itr != invalid_iterator() ) - return itr; - - EOS_ASSERT( obj.table_ei < invalid_iterator(), invalid_table_iterator, "not an end iterator" ); - const auto indx = end_iterator_to_index(obj.table_ei); - EOS_ASSERT( indx < _end_iterator_to_table.size(), invalid_table_iterator, "an invariant was broken, table should be in cache" ); - _object_to_iterator.insert({ obj, _iterator_to_object.size() }); - _iterator_to_object.emplace_back( obj ); - - return _iterator_to_object.size() - 1; - } - - private: - void validate_object_iterator(int iterator, const char* explanation_for_no_end_iterators) const { - EOS_ASSERT( iterator != invalid_iterator(), invalid_table_iterator, "invalid iterator" ); - EOS_ASSERT( iterator >= 0, table_operation_not_permitted, explanation_for_no_end_iterators ); - EOS_ASSERT( (size_t)iterator < _iterator_to_object.size(), invalid_table_iterator, "iterator out of range" ); - } - - map _table_cache; - vector _end_iterator_to_table; - vector> _iterator_to_object; - map _object_to_iterator; - - /// Precondition: std::numeric_limits::min() < ei < -1 - /// Iterator of -1 is reserved for invalid iterators (i.e. when the appropriate table has not yet been created). - inline size_t end_iterator_to_index( int ei )const { return (-ei - 2); } - /// Precondition: indx < _end_iterator_to_table.size() <= std::numeric_limits::max() - inline int index_to_end_iterator( size_t indx )const { return -(indx + 2); } -}; /// class db_key_value_iter_store - -}}} // namespace eosio::chain::backing_store diff --git a/libraries/chain/include/eosio/chain/backing_store/db_key_value_sec_lookup.hpp b/libraries/chain/include/eosio/chain/backing_store/db_key_value_sec_lookup.hpp deleted file mode 100644 index f6da7986ea..0000000000 --- a/libraries/chain/include/eosio/chain/backing_store/db_key_value_sec_lookup.hpp +++ /dev/null @@ -1,638 +0,0 @@ -#pragma once - -#include -#include - -namespace eosio { namespace chain { namespace backing_store { - - template - struct secondary_helper; - - template<> - struct secondary_helper { - const char* desc() { return "idx64"; } - eosio::session::shared_bytes value(const uint64_t& sec_key, name payer) { - return payer_payload(payer, nullptr, 0).as_payload(); - } - void extract(const eosio::session::shared_bytes& payload, uint64_t& sec_key, name& payer) { - payer_payload pp(payload); - payer = pp.payer; - EOS_ASSERT( pp.value_size == 0, db_rocksdb_invalid_operation_exception, - "Payload should not have anything more than the payer"); - } - constexpr inline uint64_t overhead() { return config::billable_size_v;} - }; - - template<> - struct secondary_helper { - const char* desc() { return "idx128"; } - eosio::session::shared_bytes value(const eosio::chain::uint128_t& sec_key, name payer) { - return payer_payload(payer, nullptr, 0).as_payload(); - } - void extract(const eosio::session::shared_bytes& payload, eosio::chain::uint128_t& sec_key, name& payer) { - payer_payload pp(payload); - payer = pp.payer; - EOS_ASSERT( pp.value_size == 0, db_rocksdb_invalid_operation_exception, - "Payload should not have anything more than the payer"); - } - constexpr inline uint64_t overhead() { return config::billable_size_v;} - }; - - template<> - struct secondary_helper { - const char* desc() { return "idx256"; } - eosio::session::shared_bytes value(const eosio::chain::key256_t& sec_key, name payer) { - return payer_payload(payer, nullptr, 0).as_payload(); - } - void extract(const eosio::session::shared_bytes& payload, eosio::chain::key256_t& sec_key, name& payer) { - payer_payload pp(payload); - payer = pp.payer; - EOS_ASSERT( pp.value_size == 0, db_rocksdb_invalid_operation_exception, - "Payload should not have anything more than the payer"); - } - constexpr inline uint64_t overhead() { return config::billable_size_v;} - }; - - template<> - struct secondary_helper { - const char* desc() { return "idx_double"; } - eosio::session::shared_bytes value(const float64_t& sec_key, name payer) { - constexpr static auto value_size = sizeof(sec_key); - char value_buf[value_size]; - memcpy(value_buf, &sec_key, value_size); - // float64_t stores off the original bit pattern since the key pattern has to lose the differentiation - // between +0.0 and -0.0 - return payer_payload(payer, value_buf, value_size).as_payload(); - } - void extract(const eosio::session::shared_bytes& payload, float64_t& sec_key, name& payer) { - payer_payload pp(payload); - EOS_ASSERT( pp.value_size == sizeof(sec_key), db_rocksdb_invalid_operation_exception, - "Payload does not contain the expected float64_t exact representation"); - payer = pp.payer; - // see note on value(...) method above - memcpy(&sec_key, pp.value, pp.value_size); - } - constexpr inline uint64_t overhead() { return config::billable_size_v;} - }; - - template<> - struct secondary_helper { - const char* desc() { return "idx_long_double"; } - eosio::session::shared_bytes value(const float128_t& sec_key, name payer) { - constexpr static auto value_size = sizeof(sec_key); - // NOTE: this is not written in a way that sorting on value would work, but it is just used for storage - char value_buf[value_size]; - memcpy(value_buf, &sec_key, value_size); - // float128_t stores off the original bit pattern since the key pattern has to lose the differentiation - // between +0.0 and -0.0 - return payer_payload(payer, value_buf, value_size).as_payload(); - } - void extract(const eosio::session::shared_bytes& payload, float128_t& sec_key, name& payer) { - payer_payload pp(payload); - EOS_ASSERT( pp.value_size == sizeof(sec_key), db_rocksdb_invalid_operation_exception, - "Payload does not contain the expected float128_t exact representation"); - payer = pp.payer; - // see note on value(...) method above - memcpy(&sec_key, pp.value, pp.value_size); - } - constexpr inline uint64_t overhead() { return config::billable_size_v;} - }; - - template - class db_key_value_sec_lookup : public db_key_value_any_lookup - { - public: - db_key_value_sec_lookup( db_context& p, session_variant_type& s ) : db_key_value_any_lookup(p, s) {} - - int store( name scope, name table, const account_name& payer, - uint64_t id, const SecondaryKey& secondary ) { - EOS_ASSERT( payer != account_name(), invalid_table_payer, "must specify a valid account to pay for new record" ); - - const sec_pair_bundle secondary_key = get_secondary_slices_in_secondaries(parent.receiver, scope, table, secondary, id); - - add_table_if_needed(secondary_key.full_secondary_key, payer); - - auto old_value = current_session.read(secondary_key.full_secondary_key); - - EOS_ASSERT( !old_value, db_rocksdb_invalid_operation_exception, "db_${d}_store called with pre-existing key", ("d", helper.desc())); - - // identify if this primary key already has a secondary key of this type - auto session_iter = current_session.lower_bound(secondary_key.prefix_primary_to_sec_key); - EOS_ASSERT( !match_prefix(secondary_key.prefix_primary_to_sec_key, session_iter), db_rocksdb_invalid_operation_exception, - "db_${d}_store called, but primary key: ${primary} already has a secondary key of this type", - ("d", helper.desc())("primary", id)); - - set_value(secondary_key.full_secondary_key, helper.value(secondary, payer)); - set_value(secondary_key.full_primary_to_sec_key, useless_value); - - std::string event_id; - if (parent.context.control.get_deep_mind_logger() != nullptr) { - event_id = db_context::table_event(parent.receiver, scope, table, name(id)); - } - - parent.context.update_db_usage( payer, helper.overhead(), backing_store::db_context::secondary_add_trace(parent.context.get_action_id(), std::move(event_id)) ); - - const unique_table t { parent.receiver, scope, table }; - const auto table_ei = iter_store.cache_table(t); - return iter_store.add({ .table_ei = table_ei, .secondary = secondary, .primary = id, .payer = payer }); - } - - void remove( int iterator ) { - const iter_obj& key_store = iter_store.get(iterator); - const unique_table& table = iter_store.get_table(key_store); - EOS_ASSERT( table.contract == parent.receiver, table_access_violation, "db access violation" ); - - const sec_pair_bundle secondary_key = get_secondary_slices_in_secondaries(parent.receiver, table.scope, table.table, key_store.secondary, key_store.primary); - auto old_value = current_session.read(secondary_key.full_secondary_key); - - EOS_ASSERT( old_value, db_rocksdb_invalid_operation_exception, - "invariant failure in db_${d}_remove, iter store found to update but nothing in database", ("d", helper.desc())); - - auto session_iter = current_session.lower_bound(secondary_key.prefix_primary_to_sec_key); - EOS_ASSERT( match_prefix(secondary_key.full_primary_to_sec_key, session_iter), db_rocksdb_invalid_operation_exception, - "db_${d}_remove called, but primary key: ${primary} didn't have a primary to secondary key", - ("d", helper.desc())("primary", key_store.primary)); - - std::string event_id; - if (parent.context.control.get_deep_mind_logger() != nullptr) { - event_id = db_context::table_event(table.contract, table.scope, table.table, name(key_store.primary)); - } - - parent.context.update_db_usage( key_store.payer, -( helper.overhead() ), db_context::secondary_rem_trace(parent.context.get_action_id(), std::move(event_id)) ); - - current_session.erase(secondary_key.full_secondary_key); - current_session.erase(secondary_key.full_primary_to_sec_key); - remove_table_if_empty(secondary_key.full_secondary_key); - iter_store.remove(iterator); - } - - void update( int iterator, account_name payer, const SecondaryKey& secondary ) { - const iter_obj& key_store = iter_store.get(iterator); - const unique_table& table = iter_store.get_table(key_store); - EOS_ASSERT( table.contract == parent.receiver, table_access_violation, "db access violation" ); - - const sec_pair_bundle secondary_key = get_secondary_slices_in_secondaries(parent.receiver, table.scope, table.table, key_store.secondary, key_store.primary); - auto old_value = current_session.read(secondary_key.full_secondary_key); - - secondary_helper helper; - EOS_ASSERT( old_value, db_rocksdb_invalid_operation_exception, - "invariant failure in db_${d}_remove, iter store found to update but nothing in database", ("d", helper.desc())); - - // identify if this primary key already has a secondary key of this type - auto primary_sec_uesless_value = current_session.read(secondary_key.full_primary_to_sec_key); - EOS_ASSERT( primary_sec_uesless_value, db_rocksdb_invalid_operation_exception, - "invariant failure in db_${d}_update, the secondary to primary lookup key was found, but not the " - "primary to secondary lookup key", ("d", helper.desc())); - - if( payer == account_name() ) payer = key_store.payer; - - auto& context = parent.context; - std::string event_id; - if (context.control.get_deep_mind_logger() != nullptr) { - event_id = backing_store::db_context::table_event(table.contract, table.scope, table.table, name(key_store.primary)); - } - - if( key_store.payer != payer ) { - context.update_db_usage( key_store.payer, -helper.overhead(), backing_store::db_context::secondary_update_rem_trace(context.get_action_id(), std::string(event_id)) ); - context.update_db_usage( payer, helper.overhead(), backing_store::db_context::secondary_update_add_trace(context.get_action_id(), std::move(event_id)) ); - } - - // if the secondary value is different, remove the old key and add the new key - if (secondary != key_store.secondary) { - // remove the secondary (to primary) key and the primary to secondary key - current_session.erase(secondary_key.full_secondary_key); - current_session.erase(secondary_key.full_primary_to_sec_key); - - // store the new secondary (to primary) key - const auto new_secondary_keys = db_key_value_format::create_secondary_key_pair(table.scope, table.table, secondary, key_store.primary); - set_value(db_key_value_format::create_full_key(new_secondary_keys.secondary_key, parent.receiver), helper.value(secondary, payer)); - - // store the new primary to secondary key - set_value(db_key_value_format::create_full_key(new_secondary_keys.primary_to_secondary_key, parent.receiver), useless_value); - } - else { - // no key values have changed, so can use the old key (and no change to secondary to primary key) - set_value(secondary_key.full_primary_to_sec_key, helper.value(secondary, payer)); - } - iter_store.swap(iterator, secondary, payer); - } - - int find_secondary( name code, name scope, name table, const SecondaryKey& secondary, uint64_t& primary ) { - prefix_bundle secondary_key = get_secondary_slice_in_table(code, scope, table, secondary); - auto session_iter = current_session.lower_bound(secondary_key.full_key); - - // if we don't get any match for this table, then return the invalid iterator - if (!match_prefix(secondary_key.prefix_key, session_iter)) { - // no keys for this entire table - return iter_store.invalid_iterator(); - } - - const unique_table t { code, scope, table }; - const auto table_ei = iter_store.cache_table(t); - // verify if the key that was found contains this secondary key - if (!match_prefix(secondary_key.full_key, session_iter)) { - return table_ei; - } - - const auto full_slice = db_key_value_format::extract_legacy_slice((*session_iter).first); - const auto prefix_secondary_slice = db_key_value_format::extract_legacy_slice(secondary_key.full_key); - const bool valid_key = db_key_value_format::get_trailing_primary_key(full_slice, prefix_secondary_slice, primary); - EOS_ASSERT( valid_key, db_rocksdb_invalid_operation_exception, - "invariant failure in db_${d}_find_secondary, key portions were identical but " - "get_trailing_primary_key indicated that it couldn't extract", ("d", helper.desc())); - - return iter_store.add({ .table_ei = table_ei, .secondary = secondary, .primary = primary, - .payer = payer_payload(*(*session_iter).second).payer }); - } - - int lowerbound_secondary( name code, name scope, name table, SecondaryKey& secondary, uint64_t& primary ) { - return bound_secondary(code, scope, table, bound_type::lower, secondary, primary); - } - - int upperbound_secondary( name code, name scope, name table, SecondaryKey& secondary, uint64_t& primary ) { - return bound_secondary(code, scope, table, bound_type::upper, secondary, primary); - } - - int end_secondary( name code, name scope, name table ) { - return get_end_iter(name{code}, name{scope}, name{table}, iter_store); - } - - int next_secondary( int iterator, uint64_t& primary ) { - if( iterator < iter_store.invalid_iterator() ) return iter_store.invalid_iterator(); // cannot increment past end iterator of index - - const iter_obj& key_store = iter_store.get(iterator); - const unique_table& table = iter_store.get_table(key_store); - - prefix_bundle secondary_key = get_secondary_slice_in_secondaries(table.contract, table.scope, table.table, key_store.secondary, key_store.primary); - auto session_iter = current_session.lower_bound(secondary_key.full_key); - EOS_ASSERT( match(secondary_key.full_key, session_iter), db_rocksdb_invalid_operation_exception, - "invariant failure in db_${d}_next_secondary, found iterator in iter store but didn't find " - "any entry in the database (no secondary keys of this specific type)", ("d", helper.desc())); - ++session_iter; - if (!match_prefix(secondary_key.prefix_key, session_iter)) { - return key_store.table_ei; - } - - SecondaryKey secondary; - const bool valid_key = db_key_value_format::get_trailing_sec_prim_keys((*session_iter).first, secondary_key.prefix_key, secondary, primary); - EOS_ASSERT( valid_key, db_rocksdb_invalid_operation_exception, - "invariant failure in db_${d}_next_secondary, since the type slice matched, the trailing " - "primary key should have been extracted", ("d", helper.desc())); - - return iter_store.add({ .table_ei = key_store.table_ei, .secondary = secondary, .primary = primary, - .payer = payer_payload(*(*session_iter).second).payer }); - } - - int previous_secondary( int iterator, uint64_t& primary ) { - if( iterator < iter_store.invalid_iterator() ) { - const backing_store::unique_table* table = iter_store.find_table_by_end_iterator(iterator); - EOS_ASSERT( table, invalid_table_iterator, "not a valid end iterator" ); - constexpr static auto kt = db_key_value_format::derive_secondary_key_type(); - const bytes legacy_type_key = db_key_value_format::create_prefix_type_key(table->scope, table->table, kt); - const shared_bytes type_prefix = db_key_value_format::create_full_key(legacy_type_key, table->contract); - const shared_bytes next_type_prefix = type_prefix.next(); - auto session_iter = current_session.lower_bound(next_type_prefix); - --session_iter; // move back to the last secondary key of this type (if there is one) - // check if there is any secondary key of this type - if (!match_prefix(type_prefix, session_iter)) { - return iter_store.invalid_iterator(); - } - - auto found_keys = find_matching_sec_prim(*session_iter, type_prefix); - if (!found_keys) { - return iter_store.invalid_iterator(); - } - primary = std::get(*found_keys); - return iter_store.add({ .table_ei = iterator, .secondary = std::move(std::get(*found_keys)), - .primary = std::move(std::get(*found_keys)), - .payer = std::move(std::get(*found_keys))}); - } - - const iter_obj& key_store = iter_store.get(iterator); - const unique_table& table = iter_store.get_table(key_store); - - prefix_bundle secondary_key = get_secondary_slice_in_secondaries(table.contract, table.scope, table.table, key_store.secondary, key_store.primary); - auto session_iter = current_session.lower_bound(secondary_key.full_key); - EOS_ASSERT( match(secondary_key.full_key, session_iter), db_rocksdb_invalid_operation_exception, - "invariant failure in db_${d}_next_secondary, found iterator in iter store but didn't find its " - "matching entry in the database", ("d", helper.desc())); - --session_iter; - if (!match_prefix(secondary_key.prefix_key, session_iter)) { - return key_store.table_ei; - } - - auto found_keys = find_matching_sec_prim(*session_iter, secondary_key.prefix_key); - EOS_ASSERT( found_keys, db_rocksdb_invalid_operation_exception, - "invariant failure in db_${d}_previous_secondary, since the type slice matched, the trailing " - "primary key should have been extracted", ("d", helper.desc())); - primary = std::get(*found_keys); - - return iter_store.add({ .table_ei = key_store.table_ei, .secondary = std::move(std::get(*found_keys)), - .primary = primary, .payer = std::move(std::get(*found_keys)) }); - } - - int find_primary( name code, name scope, name table, SecondaryKey& secondary, uint64_t primary ) { - const bytes legacy_prim_to_sec_key = db_key_value_format::create_prefix_primary_to_secondary_key(scope, table, primary); - const shared_bytes key = db_key_value_format::create_full_key(legacy_prim_to_sec_key, code); - const shared_bytes prefix = db_key_value_format::create_full_key_prefix(key, end_of_prefix::pre_type); - auto session_iter = current_session.lower_bound(key); - // check if nothing remains in the primary table space - if (!match_prefix(prefix, session_iter)) { - return iter_store.invalid_iterator(); - } - - const unique_table t { code, scope, table }; - const auto table_ei = iter_store.cache_table(t); - - // if this is not of the same type and primary key - if (!backing_store::db_key_value_format::get_trailing_secondary_key((*session_iter).first, key, secondary)) { - // since this is not the desired secondary key entry, reject it - return table_ei; - } - - const bytes legacy_secondary_key = db_key_value_format::create_secondary_key(scope, table, secondary, primary); - const shared_bytes secondary_key = db_key_value_format::create_full_key(legacy_secondary_key, code); - const auto old_value = current_session.read(secondary_key); - EOS_ASSERT( old_value, db_rocksdb_invalid_operation_exception, - "invariant failure in db_${d}_previous_secondary, found primary to secondary key in database but " - "not the secondary to primary key", ("d", helper.desc())); - EOS_ASSERT( old_value->data() != nullptr, db_rocksdb_invalid_operation_exception, - "invariant failure in db_${d}_previous_secondary, empty value", ("d", helper.desc())); - EOS_ASSERT( old_value->size(), db_rocksdb_invalid_operation_exception, - "invariant failure in db_${d}_previous_secondary, value has zero size", ("d", helper.desc())); - payer_payload pp{*old_value}; - - return iter_store.add({ .table_ei = table_ei, .secondary = secondary, .primary = primary, - .payer = pp.payer}); - } - - int lowerbound_primary( name code, name scope, name table, uint64_t primary ) { - return bound_primary(code, scope, table, primary, bound_type::lower); - } - - int upperbound_primary( name code, name scope, name table, uint64_t primary ) { - return bound_primary(code, scope, table, primary, bound_type::upper); - } - - int next_primary( int iterator, uint64_t& primary ) { - if( iterator < iter_store.invalid_iterator() ) return iter_store.invalid_iterator(); // cannot increment past end iterator of index - - const iter_obj& key_store = iter_store.get(iterator); - const unique_table& table = iter_store.get_table(key_store); - - const bytes prim_to_sec_key = - db_key_value_format::create_primary_to_secondary_key(table.scope, table.table, key_store.primary, key_store.secondary); - const shared_bytes key = db_key_value_format::create_full_key(prim_to_sec_key, table.contract); - const shared_bytes sec_type_prefix = db_key_value_format::create_full_key_prefix(key, end_of_prefix::at_prim_to_sec_type); - auto session_iter = current_session.lower_bound(key); - // This key has to exist, since it is cached in our iterator store - EOS_ASSERT( match(key, session_iter), db_rocksdb_invalid_operation_exception, - "invariant failure in db_${d}_next_primary, found iterator in iter store but didn't find its " - "matching entry in the database", ("d", helper.desc())); - ++session_iter; - if (!match_prefix(sec_type_prefix, session_iter)) { - return key_store.table_ei; - } - - auto found_keys = find_matching_prim_sec(*session_iter, sec_type_prefix); - EOS_ASSERT( found_keys, db_rocksdb_invalid_operation_exception, - "invariant failure in db_${d}_next_primary, since the type slice matched, the trailing " - "keys should have been extracted", ("d", helper.desc())); - primary = std::get(*found_keys); - return iter_store.add({ .table_ei = key_store.table_ei, .secondary = std::move(std::get(*found_keys)), - .primary = std::move(std::get(*found_keys)), - .payer = std::move(std::get(*found_keys))}); - } - - int previous_primary( int iterator, uint64_t& primary ) { - if( iterator < iter_store.invalid_iterator() ) { - const backing_store::unique_table* table = iter_store.find_table_by_end_iterator(iterator); - EOS_ASSERT( table, invalid_table_iterator, "not a valid end iterator" ); - const bytes types_key = db_key_value_format::create_prefix_primary_to_secondary_key(table->scope, table->table); - - // create the key pointing to the start of the primary to secondary keys of this type, and then increment it to be past the end - const shared_bytes type_prefix = db_key_value_format::create_full_key(types_key, table->contract); - const shared_bytes next_type_prefix = type_prefix.next(); - auto session_iter = current_session.lower_bound(next_type_prefix); //upperbound of this primary_to_sec key type - --session_iter; // move back to the last primary key of this type (if there is one) - // check if no keys exist for this type - if (!match_prefix(type_prefix, session_iter)) { - return iter_store.invalid_iterator(); - } - auto found_keys = find_matching_prim_sec(*session_iter, type_prefix); - if (!found_keys) { - return iter_store.invalid_iterator(); - } - primary = std::move(std::get(*found_keys)); - return iter_store.add({ .table_ei = iterator, .secondary = std::move(std::get(*found_keys)), - .primary = std::move(std::get(*found_keys)), - .payer = std::move(std::get(*found_keys))}); - } - - const iter_obj& key_store = iter_store.get(iterator); - const unique_table& table = iter_store.get_table(key_store); - - const bytes prim_to_sec_key = - db_key_value_format::create_primary_to_secondary_key(table.scope, table.table, key_store.primary, key_store.secondary); - const shared_bytes key = db_key_value_format::create_full_key(prim_to_sec_key, table.contract); - const shared_bytes sec_type_prefix = db_key_value_format::create_full_key_prefix(key, end_of_prefix::at_prim_to_sec_type); - auto session_iter = current_session.lower_bound(key); - EOS_ASSERT( match(key, session_iter), db_rocksdb_invalid_operation_exception, - "invariant failure in db_${d}_previous_primary, found iterator in iter store but didn't find its " - "matching entry in the database", ("d", helper.desc())); - --session_iter; - // determine if we are still in the range of this primary_to_sec key type - if (!match_prefix(sec_type_prefix, session_iter)) { - return iter_store.invalid_iterator(); - } - - auto found_keys = find_matching_prim_sec(*session_iter, sec_type_prefix); - if (!found_keys) { - return iter_store.invalid_iterator(); - } - primary = std::get(*found_keys); - return iter_store.add({ .table_ei = key_store.table_ei, .secondary = std::move(std::get(*found_keys)), - .primary = std::move(std::get(*found_keys)), - .payer = std::move(std::get(*found_keys))}); - } - - void get( int iterator, uint64_t& primary, SecondaryKey& secondary ) { - const iter_obj& key_store = iter_store.get(iterator); - - primary = key_store.primary; - secondary = key_store.secondary; - } - - struct sec_pair_bundle { - sec_pair_bundle(const b1::chain_kv::bytes& composite_sec_key, const b1::chain_kv::bytes& composite_prim_to_sec_key, name code) - : full_secondary_key(db_key_value_format::create_full_key(composite_sec_key, code)), - full_primary_to_sec_key(db_key_value_format::create_full_key(composite_prim_to_sec_key, code)), - prefix_secondary_key(db_key_value_format::create_full_key_prefix(full_secondary_key, end_of_prefix::at_type)), - prefix_primary_to_sec_key(db_key_value_format::create_full_key_prefix(full_primary_to_sec_key, end_of_prefix::at_prim_to_sec_primary_key)) { - - } - - shared_bytes full_secondary_key; - shared_bytes full_primary_to_sec_key; - shared_bytes prefix_secondary_key; // points to the type prefix for composite_key - shared_bytes prefix_primary_to_sec_key; // points to the primary key in full_primary_to_sec_key (all except the secondary key) - }; - - // gets a prefix that allows for only this secondary key iterators, and the primary to secondary key, with a prefix key to the secondary key type - static sec_pair_bundle get_secondary_slices_in_secondaries(name code, name scope, name table, const SecondaryKey& secondary, uint64_t primary) { - auto secondary_keys = db_key_value_format::create_secondary_key_pair(scope, table, secondary, primary); - return { secondary_keys.secondary_key, secondary_keys.primary_to_secondary_key, code }; - } - - // gets a specific secondary key without the trailing primary key, and will allow for only secondaries of this type (prefix) - static prefix_bundle get_secondary_slice_in_secondaries(name code, name scope, name table, const SecondaryKey& secondary) { - bytes secondary_key = db_key_value_format::create_prefix_secondary_key(scope, table, secondary); - return { secondary_key, end_of_prefix::at_type, code }; - } - - // gets a specific secondary and primary keys, and will allow for only secondaries of this type (prefix) - static prefix_bundle get_secondary_slice_in_secondaries(name code, name scope, name table, const SecondaryKey& secondary, uint64_t primary) { - bytes secondary_key = db_key_value_format::create_secondary_key(scope, table, secondary, primary); - return { secondary_key, end_of_prefix::at_type, code }; - } - - // gets a prefix that allows for a specific secondary key without the trailing primary key, but will allow for all iterators in the table - static prefix_bundle get_secondary_slice_in_table(name code, name scope, name table, const SecondaryKey& secondary) { - bytes secondary_key = db_key_value_format::create_prefix_secondary_key(scope, table, secondary); - return { secondary_key, end_of_prefix::pre_type, code }; - } - - void set_value(const shared_bytes& full_key, const shared_bytes& payload) { - current_session.write(full_key, payload); - } - - private: - using found_keys = std::tuple; - struct fk_index { - static constexpr std::size_t secondary = 0; - static constexpr std::size_t primary = 1; - static constexpr std::size_t payer = 2; - }; - - template - std::optional find_matching_sec_prim(const std::pair >& found_kv, - const CharCont& prefix) { - SecondaryKey secondary_key; - uint64_t primary_key; - if (!backing_store::db_key_value_format::get_trailing_sec_prim_keys(found_kv.first, prefix, secondary_key, primary_key)) { - // since this is not the desired secondary key entry, reject it - return {}; - } - - EOS_ASSERT( found_kv.second, db_rocksdb_invalid_operation_exception, - "invariant failure in db_${d}_previous_secondary, the secondary key was found but it had no value " - "(the payer) stored for it", ("d", helper.desc())); - account_name payer = payer_payload(*found_kv.second).payer; - return found_keys{ std::move(secondary_key), std::move(primary_key), std::move(payer) }; - } - - template - std::optional find_matching_prim_sec(const std::pair>& found_kv, - const shared_bytes& prefix) { - SecondaryKey secondary_key; - uint64_t primary_key; - if (!backing_store::db_key_value_format::get_trailing_prim_sec_keys(found_kv.first, prefix, primary_key, secondary_key)) { - // since this is not the desired secondary key entry, reject it - return {}; - } - - auto full_secondary_key = backing_store::db_key_value_format::create_secondary_key_from_primary_to_sec_key(found_kv.first); - auto session_iter = current_session.read(full_secondary_key); - EOS_ASSERT( match(found_kv.first, session_iter), db_rocksdb_invalid_operation_exception, - "invariant failure in find_matching_prim_sec, the primary to sec key was found in the database, " - "but no secondary to primary key"); - EOS_ASSERT( (*session_iter)->second, db_rocksdb_invalid_operation_exception, - "invariant failure in find_matching_prim_sec, the secondary key was found but it had no value " - "(the payer) stored for it"); - account_name payer = payer_payload(*(*session_iter)->second).payer; - return found_keys{ std::move(secondary_key), std::move(primary_key), std::move(payer) }; - } - - enum class bound_type { lower, upper }; - const char* as_string(bound_type bt) { return (bt == bound_type::upper) ? "upper" : "lower"; } - int bound_secondary( name code, name scope, name table, bound_type bt, SecondaryKey& secondary, uint64_t& primary ) { - prefix_bundle secondary_key = get_secondary_slice_in_table(code, scope, table, secondary); - auto session_iter = current_session.lower_bound(secondary_key.full_key); - // setting the "key space" to be the whole table, so that we either get a match or another key for this table - if (!match_prefix(secondary_key.prefix_key, session_iter)) { - // no keys for this entire table - return iter_store.invalid_iterator(); - } - - const unique_table t { code, scope, table }; - const auto table_ei = iter_store.cache_table(t); - const auto type_prefix = db_key_value_format::create_full_key_prefix(secondary_key.full_key, end_of_prefix::at_type); - // verify if the key that was found contains a secondary key of this type - if (!match_prefix(type_prefix, session_iter)) { - return table_ei; - } - else if (bt == bound_type::upper) { - // need to advance till we get to a different secondary key - while (match_prefix(secondary_key.full_key, session_iter)) { - ++session_iter; - if (!match_prefix(type_prefix, session_iter)) { - // no more secondary keys of this type - return table_ei; - } - } - } - - SecondaryKey secondary_ret {}; - uint64_t primary_ret = 0; - const bool valid_key = db_key_value_format::get_trailing_sec_prim_keys( - (*session_iter).first, type_prefix, secondary_ret, primary_ret); - EOS_ASSERT( valid_key, db_rocksdb_invalid_operation_exception, - "invariant failure in db_${d}_${bt_str}bound_secondary, since the type slice matched, the trailing " - "keys should have been extracted", ("d", helper.desc())("bt_str",as_string(bt))); - - // since this method cannot control that the input for primary and secondary having overlapping memory - // setting primary 1st and secondary 2nd to maintain the same results as chainbase - primary = primary_ret; - secondary = secondary_ret; - - return iter_store.add({ .table_ei = table_ei, .secondary = secondary_ret, .primary = primary_ret, - .payer = payer_payload(*(*session_iter).second).payer }); - } - - int bound_primary( name code, name scope, name table, uint64_t primary, bound_type bt ){ - const bytes prim_to_sec_key = db_key_value_format::create_prefix_primary_to_secondary_key(scope, table, primary); - const shared_bytes key = db_key_value_format::create_full_key(prim_to_sec_key, code); - // use the primary to secondary key type to only retrieve secondary keys of SecondaryKey type - const shared_bytes sec_type_prefix = db_key_value_format::create_full_key_prefix(key, end_of_prefix::at_prim_to_sec_type); - auto session_iter = current_session.lower_bound(key); - // check if nothing remains in the table database - if (!match_prefix(sec_type_prefix, session_iter)) { - return iter_store.invalid_iterator(); - } - - const unique_table t { code, scope, table }; - const auto table_ei = iter_store.cache_table(t); - - if (bt == bound_type::upper && match_prefix(key, session_iter)) { - ++session_iter; - if (!match_prefix(sec_type_prefix, session_iter)) { - return table_ei; - } - } - - auto found_keys = find_matching_prim_sec(*session_iter, sec_type_prefix); - EOS_ASSERT( found_keys, db_rocksdb_invalid_operation_exception, - "invariant failure in db_${d}_${bt_str}bound_secondary, since the type slice matched, the trailing " - "keys should have been extracted", ("d", helper.desc())("bt_str",as_string(bt))); - - return iter_store.add({ .table_ei = table_ei, .secondary = std::move(std::get(*found_keys)), - .primary = std::move(std::get(*found_keys)), - .payer = std::move(std::get(*found_keys))}); - } - - db_key_value_iter_store iter_store; - using iter_obj = typename db_key_value_iter_store::secondary_obj_type; - secondary_helper helper; - }; - -}}} // ns eosio::chain::backing_store diff --git a/libraries/chain/include/eosio/chain/backing_store/kv_context.hpp b/libraries/chain/include/eosio/chain/backing_store/kv_context.hpp index 65d758a447..dacb83d087 100644 --- a/libraries/chain/include/eosio/chain/backing_store/kv_context.hpp +++ b/libraries/chain/include/eosio/chain/backing_store/kv_context.hpp @@ -10,11 +10,6 @@ namespace chainbase { class database; } -namespace b1::chain_kv { - struct database; - class undo_stack; -} - namespace eosio { namespace chain { class apply_context; @@ -31,7 +26,6 @@ namespace eosio { namespace chain { virtual ~kv_iterator() {} virtual bool is_kv_chainbase_context_iterator() const = 0; - virtual bool is_kv_rocksdb_context_iterator() const = 0; virtual kv_it_stat kv_it_status() = 0; virtual int32_t kv_it_compare(const kv_iterator& rhs) = 0; virtual int32_t kv_it_key_compare(const char* key, uint32_t size) = 0; diff --git a/libraries/chain/include/eosio/chain/backing_store/kv_context_chainbase.hpp b/libraries/chain/include/eosio/chain/backing_store/kv_context_chainbase.hpp index 8397caa105..2e35f5fe03 100644 --- a/libraries/chain/include/eosio/chain/backing_store/kv_context_chainbase.hpp +++ b/libraries/chain/include/eosio/chain/backing_store/kv_context_chainbase.hpp @@ -26,7 +26,6 @@ namespace eosio { namespace chain { ~kv_iterator_chainbase() override { --itr_count; } bool is_kv_chainbase_context_iterator() const override { return true; } - bool is_kv_rocksdb_context_iterator() const override { return false; } template kv_it_stat move_to(const It& it, uint32_t* found_key_size, uint32_t* found_value_size) { diff --git a/libraries/chain/include/eosio/chain/backing_store/kv_context_rocksdb.hpp b/libraries/chain/include/eosio/chain/backing_store/kv_context_rocksdb.hpp deleted file mode 100644 index 9b2c290ef9..0000000000 --- a/libraries/chain/include/eosio/chain/backing_store/kv_context_rocksdb.hpp +++ /dev/null @@ -1,496 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#include - -namespace eosio { namespace chain { - - static inline uint32_t actual_key_size(const uint32_t raw_key_size) { - static auto rocks_prefix = make_rocksdb_contract_kv_prefix(); - static auto prefix_size = rocks_prefix.size() + sizeof(uint64_t); // kv.db prefix + contract size. - EOS_ASSERT(raw_key_size >= prefix_size, kv_rocksdb_bad_value_size_exception, - "The size of key returned from RocksDB is less than prefix size"); - return raw_key_size - prefix_size; - } - - - static inline const char* actual_key_start(const char* key) { - static auto rocks_prefix = make_rocksdb_contract_kv_prefix(); - static auto prefix_size = rocks_prefix.size() + sizeof(uint64_t); - return key + prefix_size; - } - - // Need to store payer so that this account is properly - // credited when storage is removed or changed - // to another payer - inline static eosio::session::shared_bytes build_value(const char* value, uint32_t value_size, const account_name& payer) { - const uint32_t final_value_size = backing_store::payer_in_value_size + value_size; - auto result = eosio::session::shared_bytes(final_value_size); - - char buf[backing_store::payer_in_value_size]; - memcpy(buf, &payer, backing_store::payer_in_value_size); - std::memcpy(result.data(), buf, backing_store::payer_in_value_size); - std::memcpy(result.data() + backing_store::payer_in_value_size, value, value_size); - return result; - } - - - static inline eosio::session::shared_bytes make_prefix_key(uint64_t contract, const char* user_prefix, uint32_t user_prefix_size) { - static auto rocks_prefix = make_rocksdb_contract_kv_prefix(); - - auto result = eosio::session::shared_bytes(rocks_prefix.size() + sizeof(contract) + user_prefix_size); - std::memcpy(result.data(), rocks_prefix.data(), rocks_prefix.size()); - b1::chain_kv::insert_key(result, rocks_prefix.size(), contract); - std::memcpy(result.data() + rocks_prefix.size() + sizeof(contract), user_prefix, user_prefix_size); - return result; - } - - static inline eosio::session::shared_bytes - make_composite_key(uint64_t contract, const char* user_prefix, uint32_t user_prefix_size, const char* key, uint32_t key_size) { - static auto rocks_prefix = make_rocksdb_contract_kv_prefix(); - - auto result = eosio::session::shared_bytes(rocks_prefix.size() + sizeof(contract) + user_prefix_size + key_size); - std::memcpy(result.data(), rocks_prefix.data(), rocks_prefix.size()); - b1::chain_kv::insert_key(result, rocks_prefix.size(), contract); - std::memcpy(result.data() + rocks_prefix.size() + sizeof(contract), user_prefix, user_prefix_size); - std::memcpy(result.data() + rocks_prefix.size() + sizeof(contract) + user_prefix_size, key, key_size); - return result; - } - - static inline eosio::session::shared_bytes make_composite_key(const eosio::session::shared_bytes& prefix, - const char* key, uint32_t key_size) { - auto result = eosio::session::shared_bytes(prefix.size() + key_size); - std::memcpy(result.data(), prefix.data(), prefix.size()); - std::memcpy(result.data() + prefix.size(), key, key_size); - return result; - } - - static inline int32_t compare_bytes(const eosio::session::shared_bytes& left, const eosio::session::shared_bytes& right) { - if (left < right) { - return -1; - } - - if (left > right) { - return 1; - } - - return 0; - }; - - template - struct kv_iterator_rocksdb : kv_iterator { - using session_type = std::remove_pointer_t; - - uint32_t& num_iterators; - uint64_t kv_contract{0}; - eosio::session::shared_bytes kv_prefix; // Format: [contract, prefix] - session_type* kv_session{nullptr}; - typename session_type::iterator kv_begin; - typename session_type::iterator kv_end; - typename session_type::iterator kv_current; - - kv_iterator_rocksdb(uint32_t& num_iterators, session_type& session, uint64_t contract, const char* user_prefix, - uint32_t user_prefix_size) - : num_iterators{ num_iterators }, kv_contract{ contract }, - kv_prefix{ make_prefix_key(contract, user_prefix, user_prefix_size) }, - kv_session{ &session }, kv_begin{ kv_session->lower_bound(kv_prefix) }, - kv_end{ [&](){ auto kv_next_prefix = kv_prefix.next(); return kv_session->lower_bound(kv_next_prefix); }() }, - kv_current{ kv_end } { - ++num_iterators; - } - - ~kv_iterator_rocksdb() override { --num_iterators; } - - bool is_kv_chainbase_context_iterator() const override { return false; } - bool is_kv_rocksdb_context_iterator() const override { return true; } - - kv_it_stat kv_it_status() override { - if (kv_current == kv_end) - return kv_it_stat::iterator_end; - else if (kv_current.deleted()) - return kv_it_stat::iterator_erased; - else - return kv_it_stat::iterator_ok; - } - - int32_t kv_it_compare(const kv_iterator& rhs) override { - EOS_ASSERT(rhs.is_kv_rocksdb_context_iterator(), kv_bad_iter, "Incompatible key-value iterators"); - auto& r = const_cast(static_cast(rhs)); - EOS_ASSERT(kv_session == r.kv_session && kv_prefix == r.kv_prefix, kv_bad_iter, - "Incompatible key-value iterators"); - EOS_ASSERT(!kv_current.deleted(), kv_bad_iter, "Iterator to erased element"); - EOS_ASSERT(!r.kv_current.deleted(), kv_bad_iter, "Iterator to erased element"); - try { - try { - auto left_status = kv_it_status(); - auto right_status = r.kv_it_status(); - - if (left_status == kv_it_stat::iterator_end && right_status == kv_it_stat::iterator_end) { - return 0; - } - if (left_status == kv_it_stat::iterator_end) { - return 1; - } - if (right_status == kv_it_stat::iterator_end) { - return -1; - } - - auto result = compare_bytes(kv_current.key(), r.kv_current.key()); - if (result) { - return result; - } - auto left = (*kv_current).second; - auto right = (*r.kv_current).second; - if (left && right) { - return compare_bytes(*left, *right); - } - return left.has_value() == right.has_value(); - } - FC_LOG_AND_RETHROW() - } - CATCH_AND_EXIT_DB_FAILURE() - } - - int32_t kv_it_key_compare(const char* key, uint32_t size) override { - EOS_ASSERT(!kv_current.deleted(), kv_bad_iter, "Iterator to erased element"); - try { - try { - auto current_key = eosio::session::shared_bytes{}; - if (kv_it_status() == kv_it_stat::iterator_ok) { - current_key = kv_current.key(); - } else { - return 1; - } - return compare_bytes(current_key, make_composite_key(kv_contract, nullptr, 0, key, size)); - } - FC_LOG_AND_RETHROW() - } - CATCH_AND_EXIT_DB_FAILURE() - } - - kv_it_stat kv_it_move_to_end() override { - try { - try { - kv_current = kv_end; - return kv_it_stat::iterator_end; - } - FC_LOG_AND_RETHROW() - } - CATCH_AND_EXIT_DB_FAILURE() - } - - kv_it_stat kv_it_next(uint32_t* found_key_size, uint32_t* found_value_size) override { - EOS_ASSERT(!kv_current.deleted(), kv_bad_iter, "Iterator to erased element"); - kv_it_stat status; - try { - try { - if (kv_current == kv_end) { - kv_current = kv_begin; - } else { - ++kv_current; - } - status = get_current_key_value_sizes(found_key_size, found_value_size); - } - FC_LOG_AND_RETHROW() - } - CATCH_AND_EXIT_DB_FAILURE() - - return status; - } - - kv_it_stat kv_it_prev(uint32_t* found_key_size, uint32_t* found_value_size) override { - EOS_ASSERT(!kv_current.deleted(), kv_bad_iter, "Iterator to erased element"); - kv_it_stat status; - try { - try { - if (kv_current == kv_begin) { - kv_current = kv_end; - } else { - --kv_current; - } - status = get_current_key_value_sizes(found_key_size, found_value_size); - } - FC_LOG_AND_RETHROW() - } - CATCH_AND_EXIT_DB_FAILURE() - - return status; - } - - kv_it_stat kv_it_lower_bound(const char* key, uint32_t size, uint32_t* found_key_size, - uint32_t* found_value_size) override { - kv_it_stat status; - try { - try { - auto key_bytes = make_composite_key(kv_contract, nullptr, 0, key, size); - if (key_bytes < kv_prefix) { - key_bytes = kv_prefix; - } - - kv_current = kv_session->lower_bound(key_bytes); - status = get_current_key_value_sizes(found_key_size, found_value_size); - } - FC_LOG_AND_RETHROW() - } - CATCH_AND_EXIT_DB_FAILURE() - - return status; - } - - kv_it_stat kv_it_key(uint32_t offset, char* dest, uint32_t size, uint32_t& actual_size) override { - EOS_ASSERT(!kv_current.deleted(), kv_bad_iter, "Iterator to erased element"); - - auto key = eosio::session::shared_bytes{}; - try { - try { - if (kv_it_status() == kv_it_stat::iterator_ok) { - key = kv_current.key(); - } - } - FC_LOG_AND_RETHROW() - } - CATCH_AND_EXIT_DB_FAILURE() - - if (key) { - actual_size = actual_key_size(key.size()); - if (offset < actual_size) { - std::memcpy(dest, actual_key_start(key.data()) + offset, std::min(size, actual_size - offset)); - } - return kv_it_stat::iterator_ok; - } else { - actual_size = 0; - return kv_it_stat::iterator_end; - } - } - - kv_it_stat kv_it_value(uint32_t offset, char* dest, uint32_t size, uint32_t& actual_size) override { - EOS_ASSERT(!kv_current.deleted(), kv_bad_iter, "Iterator to erased element"); - - auto kv = std::pair{ eosio::session::shared_bytes{}, std::optional{} }; - try { - try { - if (kv_it_status() == kv_it_stat::iterator_ok) { - kv = *kv_current; - } - } - FC_LOG_AND_RETHROW() - } - CATCH_AND_EXIT_DB_FAILURE() - - auto& value = kv.second; - if (value) { - actual_size = backing_store::actual_value_size(value->size()); - if (offset < actual_size) { - std::memcpy(dest, backing_store::actual_value_start(value->data()) + offset, std::min(size, actual_size - offset)); - } - return kv_it_stat::iterator_ok; - } else { - actual_size = 0; - return kv_it_stat::iterator_end; - } - } - - std::optional kv_it_payer() override { - EOS_ASSERT(!kv_current.deleted(), kv_bad_iter, "Iterator to erased element"); - if (kv_it_status() == kv_it_stat::iterator_ok) { - const auto& value = (*kv_current).second; - if (value) { - return backing_store::payer_payload(*value).payer; - } - } - return {}; - } - - private: - kv_it_stat get_current_key_value_sizes(uint32_t* found_key_size, uint32_t* found_value_size) { - auto status = kv_it_status(); - if (status == kv_it_stat::iterator_ok) { - // Not end or at an erased kv pair - auto kv = *kv_current; - *found_value_size = backing_store::actual_value_size( - kv.second->size()); // This must be before *found_key_size in case actual_value_size throws - *found_key_size = actual_key_size(kv.first.size()); - } else { - *found_key_size = 0; - *found_value_size = 0; - } - return status; - } - }; // kv_iterator_rocksdb - - template - struct kv_context_rocksdb : kv_context { - using session_type = std::remove_pointer_t; - - session_type* session; - name receiver; - Resource_manager resource_manager; - const kv_database_config& limits; - uint32_t num_iterators = 0; - eosio::session::shared_bytes current_key; - std::optional current_value; - - kv_context_rocksdb(session_type& the_session, name receiver, Resource_manager /*kv_resource_manager*/ resource_manager, - const kv_database_config& limits) - : session{ &the_session }, receiver{ receiver }, - resource_manager{ resource_manager }, limits{ limits } {} - - int64_t kv_erase(uint64_t contract, const char* key, uint32_t key_size) override { - const name contract_name {contract}; - EOS_ASSERT(contract_name == receiver, table_operation_not_permitted, "Can not write to this key"); - - current_value.reset(); - current_key = eosio::session::shared_bytes{}; - auto old_value = std::optional{}; - std::optional pp; - try { - try { - auto composite_key = make_composite_key(contract, nullptr, 0, key, key_size); - const auto old_value = session->read(composite_key); - if (!old_value) - return 0; - pp = backing_store::payer_payload(*old_value); - session->erase(composite_key); - } - FC_LOG_AND_RETHROW() - } - CATCH_AND_EXIT_DB_FAILURE() - - const int64_t resource_delta = erase_table_usage(resource_manager, pp->payer, key, key_size, pp->value_size); - - if (auto dm_logger = resource_manager._context->control.get_deep_mind_logger()) { - fc_dlog(*dm_logger, "KV_OP REM ${action_id} ${db} ${payer} ${key} ${odata}", - ("action_id", resource_manager._context->get_action_id()) - ("contract", contract_name) - ("payer", pp->payer) - ("key", fc::to_hex(key, key_size)) - ("odata", fc::to_hex(pp->value, pp->value_size)) - ); - } - return resource_delta; - } - - int64_t kv_set(uint64_t contract, const char* key, uint32_t key_size, const char* value, uint32_t value_size, - account_name payer) override { - const name contract_name {contract}; - EOS_ASSERT(contract_name == receiver, table_operation_not_permitted, "Can not write to this key"); - EOS_ASSERT(key_size <= limits.max_key_size, kv_limit_exceeded, "Key too large"); - EOS_ASSERT(value_size <= limits.max_value_size, kv_limit_exceeded, "Value too large"); - - current_value.reset(); - current_key = eosio::session::shared_bytes{}; - auto old_value = std::optional{}; - std::optional old_pp; - try { - try { - auto composite_key = make_composite_key(contract, nullptr, 0, key, key_size); - old_value = session->read(composite_key); - if (old_value) { - old_pp = backing_store::payer_payload(*old_value); - } - - backing_store::payer_payload new_value (payer, value, value_size); - session->write(composite_key, new_value.as_payload()); - } - FC_LOG_AND_RETHROW() - } - CATCH_AND_EXIT_DB_FAILURE() - - auto resource_delta = int64_t{ 0 }; - if (old_value) { - resource_delta = - update_table_usage(resource_manager, old_pp->payer, payer, key, key_size, old_pp->value_size, value_size); - - if (auto dm_logger = resource_manager._context->control.get_deep_mind_logger()) { - fc_dlog(*dm_logger, "KV_OP UPD ${action_id} ${db} ${payer} ${key} ${odata}:${ndata}", - ("action_id", resource_manager._context->get_action_id()) - ("contract", contract_name) - ("payer", payer) - ("key", fc::to_hex(key, key_size)) - ("odata", fc::to_hex(old_pp->value, old_pp->value_size)) - ("ndata", fc::to_hex(value, value_size)) - ); - } - } else { - resource_delta = create_table_usage(resource_manager, payer, key, key_size, value_size); - - if (auto dm_logger = resource_manager._context->control.get_deep_mind_logger()) { - fc_dlog(*dm_logger, "KV_OP INS ${action_id} ${db} ${payer} ${key} ${ndata}", - ("action_id", resource_manager._context->get_action_id()) - ("contract", contract_name) - ("payer", payer) - ("key", fc::to_hex(key, key_size)) - ("ndata", fc::to_hex(value, value_size)) - ); - } - } - - return resource_delta; - } - - bool kv_get(uint64_t contract, const char* key, uint32_t key_size, uint32_t& value_size) override { - current_key = eosio::session::shared_bytes{}; - current_value.reset(); - try { - try { - current_key = make_composite_key(contract, nullptr, 0, key, key_size); - current_value = session->read(current_key); - } - FC_LOG_AND_RETHROW() - } - CATCH_AND_EXIT_DB_FAILURE() - - if (current_value) { - value_size = backing_store::actual_value_size(current_value->size()); - return true; - } else { - value_size = 0; - return false; - } - } - - uint32_t kv_get_data(uint32_t offset, char* data, uint32_t data_size) override { - uint32_t temp_size = 0; - if (current_value) { - temp_size = backing_store::actual_value_size(current_value->size()); - } - if (offset < temp_size) { - const char* temp = backing_store::actual_value_start(current_value->data()); - std::memcpy(data, temp + offset, std::min(data_size, temp_size - offset)); - } - return temp_size; - } - - std::unique_ptr kv_it_create(uint64_t contract, const char* prefix, uint32_t size) override { - EOS_ASSERT(num_iterators < limits.max_iterators, kv_bad_iter, "Too many iterators"); - EOS_ASSERT(size <= limits.max_key_size, kv_bad_iter, "Prefix too large"); - try { - try { - return std::make_unique>(num_iterators, *session, contract, prefix, - size); - } - FC_LOG_AND_RETHROW() - } - CATCH_AND_EXIT_DB_FAILURE() - } - }; // kv_context_rocksdb - - template - std::unique_ptr create_kv_rocksdb_context(std::remove_pointer_t& session, name receiver, - Resource_manager resource_manager, - const kv_database_config& limits) { - try { - try { - return std::make_unique>(session, receiver, - resource_manager, limits); - } - FC_LOG_AND_RETHROW() - } - CATCH_AND_EXIT_DB_FAILURE() - } - -}} // namespace eosio::chain diff --git a/libraries/chain/include/eosio/chain/config.hpp b/libraries/chain/include/eosio/chain/config.hpp index 5cccf71d89..e8e1a75129 100644 --- a/libraries/chain/include/eosio/chain/config.hpp +++ b/libraries/chain/include/eosio/chain/config.hpp @@ -87,11 +87,6 @@ const static uint16_t default_controller_thread_pool_size = 2; const static uint32_t default_max_variable_signature_length = 16384u; const static uint32_t default_max_nonprivileged_inline_action_size = 4 * 1024; // 4 KB const static uint32_t default_max_action_return_value_size = 256; -const static uint16_t default_persistent_storage_num_threads = 1; -const static int default_persistent_storage_max_num_files = -1; -const static uint64_t default_persistent_storage_write_buffer_size = 128 * 1024 * 1024; -const static uint64_t default_persistent_storage_bytes_per_sync = 1 * 1024 * 1024; -const static uint32_t default_persistent_storage_mbytes_batch = 50; static_assert(MAX_SIZE_OF_BYTE_ARRAYS == 20*1024*1024, "Changing MAX_SIZE_OF_BYTE_ARRAYS breaks consensus. Make sure this is expected"); diff --git a/libraries/chain/include/eosio/chain/controller.hpp b/libraries/chain/include/eosio/chain/controller.hpp index d0665f9c71..f691f1ac34 100644 --- a/libraries/chain/include/eosio/chain/controller.hpp +++ b/libraries/chain/include/eosio/chain/controller.hpp @@ -33,7 +33,7 @@ namespace eosio { namespace chain { class resource_limits_manager; }; - class combined_database; + class kv_database; struct controller_impl; using chainbase::database; @@ -82,12 +82,8 @@ namespace eosio { namespace chain { uint16_t thread_pool_size = chain::config::default_controller_thread_pool_size; uint16_t max_retained_block_files = chain::config::default_max_retained_block_files; uint64_t blocks_log_stride = chain::config::default_blocks_log_stride; + /* backing_store is deprecated and remains for backward compatibility only */ backing_store_type backing_store = backing_store_type::CHAINBASE; - uint16_t persistent_storage_num_threads = 0; // Will be set to number of cores dynamically or by user configuration; - int persistent_storage_max_num_files = chain::config::default_persistent_storage_max_num_files; - uint64_t persistent_storage_write_buffer_size = chain::config::default_persistent_storage_write_buffer_size; - uint64_t persistent_storage_bytes_per_sync = chain::config::default_persistent_storage_bytes_per_sync; - uint32_t persistent_storage_mbytes_batch = chain::config::default_persistent_storage_mbytes_batch; fc::microseconds abi_serializer_max_time_us = fc::microseconds(chain::config::default_abi_serializer_max_time_us); uint32_t max_nonprivileged_inline_action_size = chain::config::default_max_nonprivileged_inline_action_size; bool read_only = false; @@ -395,7 +391,7 @@ namespace eosio { namespace chain { void replace_producer_keys( const public_key_type& key ); void replace_account_keys( name account, name permission, const public_key_type& key ); - eosio::chain::combined_database& kv_db()const; + eosio::chain::kv_database& kv_db()const; private: friend class apply_context; diff --git a/libraries/chain/include/eosio/chain/exceptions.hpp b/libraries/chain/include/eosio/chain/exceptions.hpp index 6fd9c575fe..5053924d2c 100644 --- a/libraries/chain/include/eosio/chain/exceptions.hpp +++ b/libraries/chain/include/eosio/chain/exceptions.hpp @@ -332,12 +332,8 @@ namespace eosio { namespace chain { 3060006, "Chainbase and chain-kv databases are at different revisions" ) FC_DECLARE_DERIVED_EXCEPTION( database_move_kv_disk_exception, database_exception, 3060007, "Cannot change backing store when existing state has already stored data in a different backing store; use resync, replay, or snapshot to move these to the new backing store" ) - FC_DECLARE_DERIVED_EXCEPTION( kv_rocksdb_bad_value_size_exception, database_exception, - 3060008, "The size of value returned from RocksDB is less than payer's size" ) FC_DECLARE_DERIVED_EXCEPTION( bad_composite_key_exception, database_exception, 3060009, "Retrieved composite key from key/value store that was formatted incorrectly" ) - FC_DECLARE_DERIVED_EXCEPTION( db_rocksdb_invalid_operation_exception, database_exception, - 3060010, "Requested operation not valid for database state." ) FC_DECLARE_DERIVED_EXCEPTION( guard_exception, database_exception, 3060100, "Guard Exception" ) diff --git a/libraries/chain/include/eosio/chain/combined_database.hpp b/libraries/chain/include/eosio/chain/kv_database.hpp similarity index 68% rename from libraries/chain/include/eosio/chain/combined_database.hpp rename to libraries/chain/include/eosio/chain/kv_database.hpp index 47d27492b0..2af7a6f115 100644 --- a/libraries/chain/include/eosio/chain/combined_database.hpp +++ b/libraries/chain/include/eosio/chain/kv_database.hpp @@ -23,9 +23,7 @@ #include #include -#include #include -#include // It's a fatal condition if chainbase and chain_kv get out of sync with each // other due to exceptions. @@ -36,9 +34,6 @@ } namespace eosio { namespace chain { - using rocks_db_type = eosio::session::session; - using session_type = eosio::session::session; - using kv_undo_stack_ptr = std::unique_ptr>; using controller_index_set = index_set* undo_stack); + kv_session(chainbase::database& cb_database); + kv_session(kv_session&& src) noexcept; - combined_session(combined_session&& src) noexcept; + ~kv_session() { undo(); } - ~combined_session() { undo(); } - - combined_session& operator=(const combined_session& src) = delete; + kv_session& operator=(const kv_session& src) = delete; void push(); @@ -76,29 +70,23 @@ namespace eosio { namespace chain { private: std::unique_ptr cb_session = {}; - eosio::session::undo_stack* kv_undo_stack = nullptr; }; - class combined_database { + class kv_database { public: - explicit combined_database(chainbase::database& chain_db, - uint32_t snapshot_batch_threashold); - - combined_database(chainbase::database& chain_db, - const controller::config& cfg); + explicit kv_database(chainbase::database& chain_db); - combined_database(const combined_database& copy) = delete; - combined_database& operator=(const combined_database& copy) = delete; + kv_database(const kv_database& copy) = delete; + kv_database& operator=(const kv_database& copy) = delete; - // Save the backing_store setting to the chainbase in order to detect - // when this setting is switched between chainbase and rocksdb. - // If existing state is not clean, switching is not allowed. + // Save the backing_store setting to the chainbase + // Currently deprecated, left just for backward compatibility void check_backing_store_setting(bool clean_startup); - static combined_session make_no_op_session() { return combined_session(); } - - combined_session make_session() { - return combined_session(db, kv_undo_stack.get()); + static kv_session make_no_op_session() { return kv_session(); } + + kv_session make_session() { + return kv_session(db); } void set_revision(uint64_t revision); @@ -129,23 +117,14 @@ namespace eosio { namespace chain { const eosio::chain::chain_id_type& chain_id); auto &get_db(void) const { return db; } - auto &get_kv_undo_stack(void) const { return kv_undo_stack; } - backing_store_type get_backing_store() const { return backing_store; } - + private: void add_contract_tables_to_snapshot(const snapshot_writer_ptr& snapshot) const; void read_contract_tables_from_snapshot(const snapshot_reader_ptr& snapshot); backing_store_type backing_store; chainbase::database& db; - std::unique_ptr kv_database; - kv_undo_stack_ptr kv_undo_stack; - const uint64_t kv_snapshot_batch_threashold; }; std::optional extract_legacy_genesis_state(snapshot_reader& snapshot, uint32_t version); - - std::vector make_rocksdb_contract_kv_prefix(); - char make_rocksdb_contract_db_prefix(); - }} // namespace eosio::chain diff --git a/libraries/chain/include/eosio/chain/transaction_context.hpp b/libraries/chain/include/eosio/chain/transaction_context.hpp index 74e7c27422..d97da3e881 100644 --- a/libraries/chain/include/eosio/chain/transaction_context.hpp +++ b/libraries/chain/include/eosio/chain/transaction_context.hpp @@ -1,5 +1,5 @@ #pragma once -#include +#include #include #include #include @@ -157,7 +157,7 @@ namespace eosio { namespace chain { controller& control; const packed_transaction& packed_trx; - combined_session undo_session; + kv_session undo_session; transaction_trace_ptr trace; fc::time_point start; diff --git a/libraries/chain/kv_database.cpp b/libraries/chain/kv_database.cpp new file mode 100644 index 0000000000..0851352c31 --- /dev/null +++ b/libraries/chain/kv_database.cpp @@ -0,0 +1,357 @@ +#include +#include +#include +#include + +namespace eosio { namespace chain { + kv_session::kv_session(chainbase::database& cb_database) + { + cb_session = std::make_unique(cb_database.start_undo_session(true)); + } + kv_session::kv_session(kv_session&& src) noexcept + : cb_session(std::move(src.cb_session)) { + } + + void kv_session::push() { + if (cb_session) { + cb_session->push(); + cb_session = nullptr; + } + } + + void kv_session::squash() { + if (cb_session) { + cb_session->squash(); + cb_session = nullptr; + } + } + + void kv_session::undo() { + if (cb_session) { + cb_session->undo(); + cb_session = nullptr; + } + } + + template + void walk_index(const Util& utils, const chainbase::database& db, F&& function) { + utils.walk(db, std::forward(function)); + } + + template + void walk_index(const index_utils& utils, const chainbase::database& db, F&& function) {} + + template + void walk_index(const index_utils& utils, const chainbase::database& db, F&& function) {} + + template + void walk_index(const index_utils& utils, const chainbase::database& db, F&& function) {} + + void add_kv_table_to_snapshot(const snapshot_writer_ptr& snapshot, const chainbase::database& db) { + snapshot->write_section([&db](auto& section) { + index_utils utils; + utils.walk(db, [&db, §ion](const auto& row) { section.add_row(row, db); }); + }); + } + + void read_kv_table_from_snapshot(const snapshot_reader_ptr& snapshot, chainbase::database& db, uint32_t version) { + if (version < kv_object::minimum_snapshot_version) + return; + snapshot->read_section([&db](auto& section) { + bool more = !section.empty(); + while (more) { + index_utils::create(db, [&db, §ion, &more](auto &row) { more = section.read_row(row, db); }); + } + }); + } + + kv_database::kv_database(chainbase::database& chain_db) + : backing_store(backing_store_type::CHAINBASE), db(chain_db) {} + + void kv_database::check_backing_store_setting(bool clean_startup) { + if (backing_store != db.get().backing_store) { + EOS_ASSERT(clean_startup, database_move_kv_disk_exception, + "Existing state indicates a different backing store is in use; use resync, replay, or restore from snapshot to switch backing store"); + db.modify(db.get(), [this](auto& cfg) { cfg.backing_store = backing_store; }); + } + } + + void kv_database::destroy(const fc::path& p) { + if( !fc::is_directory( p ) ) + return; + + fc::remove( p / "shared_memory.bin" ); + fc::remove( p / "shared_memory.meta" ); + } + + void kv_database::set_revision(uint64_t revision) { + db.set_revision(revision); + } + + int64_t kv_database::revision() { + return db.revision(); + } + + void kv_database::undo() { + db.undo(); + } + + void kv_database::commit(int64_t revision) { + db.commit(revision); + } + + void kv_database::flush() { + } + + std::unique_ptr kv_database::create_kv_context(name receiver, kv_resource_manager resource_manager, + const kv_database_config& limits) const { + return create_kv_chainbase_context(db, receiver, resource_manager, limits); + } + + std::unique_ptr kv_database::create_db_context(apply_context& context, name receiver) { + return backing_store::create_db_chainbase_context(context, receiver); + } + + void kv_database::add_to_snapshot( + const eosio::chain::snapshot_writer_ptr& snapshot, const eosio::chain::block_state& head, + const eosio::chain::authorization_manager& authorization, + const eosio::chain::resource_limits::resource_limits_manager& resource_limits) const { + snapshot->write_section( + [this](auto& section) { section.add_row(chain_snapshot_header(), db); }); + + snapshot->write_section( + [this, &head](auto& section) { section.template add_row(head, db); }); + + eosio::chain::controller_index_set::walk_indices([this, &snapshot](auto utils) { + using value_t = typename decltype(utils)::index_t::value_type; + + snapshot->write_section([utils, this](auto& section) { + walk_index(utils, db, [this, §ion](const auto& row) { section.add_row(row, db); }); + }); + }); + + add_kv_table_to_snapshot(snapshot, db); + add_contract_tables_to_snapshot(snapshot); + + authorization.add_to_snapshot(snapshot); + resource_limits.add_to_snapshot(snapshot); + } + + void kv_database::read_from_snapshot(const snapshot_reader_ptr& snapshot, + uint32_t blog_start, + uint32_t blog_end, + eosio::chain::authorization_manager& authorization, + eosio::chain::resource_limits::resource_limits_manager& resource_limits, + eosio::chain::block_state_ptr& head, uint32_t& snapshot_head_block, + const eosio::chain::chain_id_type& chain_id) { + chain_snapshot_header header; + snapshot->read_section([this, &header](auto& section) { + section.read_row(header, db); + header.validate(); + }); + + db.create([](auto&) {}); + check_backing_store_setting(true); + + { /// load and upgrade the block header state + block_header_state head_header_state; + using v2 = legacy::snapshot_block_header_state_v2; + + if (std::clamp(header.version, v2::minimum_version, v2::maximum_version) == header.version) { + snapshot->read_section([this, &head_header_state](auto& section) { + legacy::snapshot_block_header_state_v2 legacy_header_state; + section.read_row(legacy_header_state, db); + head_header_state = block_header_state(std::move(legacy_header_state)); + }); + } else { + snapshot->read_section( + [this, &head_header_state](auto& section) { section.read_row(head_header_state, db); }); + } + + snapshot_head_block = head_header_state.block_num; + EOS_ASSERT(blog_start <= (snapshot_head_block + 1) && snapshot_head_block <= blog_end, block_log_exception, + "Block log is provided with snapshot but does not contain the head block from the snapshot nor a " + "block right after it", + ("snapshot_head_block", snapshot_head_block)("block_log_first_num", + blog_start)("block_log_last_num", blog_end)); + + head = std::make_shared(); + static_cast(*head) = head_header_state; + } + + controller_index_set::walk_indices([this, &snapshot, &header](auto utils) { + using value_t = typename decltype(utils)::index_t::value_type; + + // skip the table_id_object as its inlined with contract tables section + if (std::is_same::value) { + return; + } + + // skip the database_header as it is only relevant to in-memory database + if (std::is_same::value) { + return; + } + + // skip the kv_db_config as it only determines where the kv-database is stored + if (std::is_same_v) { + return; + } + + // special case for in-place upgrade of global_property_object + if (std::is_same::value) { + using v2 = legacy::snapshot_global_property_object_v2; + using v3 = legacy::snapshot_global_property_object_v3; + using v4 = legacy::snapshot_global_property_object_v4; + + if (std::clamp(header.version, v2::minimum_version, v2::maximum_version) == header.version) { + std::optional genesis = extract_legacy_genesis_state(*snapshot, header.version); + EOS_ASSERT(genesis, snapshot_exception, + "Snapshot indicates chain_snapshot_header version 2, but does not contain a genesis_state. " + "It must be corrupted."); + snapshot->read_section( + [&db = this->db, gs_chain_id = genesis->compute_chain_id()](auto& section) { + v2 legacy_global_properties; + section.read_row(legacy_global_properties, db); + + db.create([&legacy_global_properties, &gs_chain_id](auto& gpo) { + gpo.initalize_from(legacy_global_properties, gs_chain_id, kv_database_config{}, + genesis_state::default_initial_wasm_configuration); + }); + }); + return; // early out to avoid default processing + } + + if (std::clamp(header.version, v3::minimum_version, v3::maximum_version) == header.version) { + snapshot->read_section([&db = this->db](auto& section) { + v3 legacy_global_properties; + section.read_row(legacy_global_properties, db); + + db.create([&legacy_global_properties](auto& gpo) { + gpo.initalize_from(legacy_global_properties, kv_database_config{}, + genesis_state::default_initial_wasm_configuration); + }); + }); + return; // early out to avoid default processing + } + + if (std::clamp(header.version, v4::minimum_version, v4::maximum_version) == header.version) { + snapshot->read_section([&db = this->db](auto& section) { + v4 legacy_global_properties; + section.read_row(legacy_global_properties, db); + + db.create([&legacy_global_properties](auto& gpo) { + gpo.initalize_from(legacy_global_properties); + }); + }); + return; // early out to avoid default processing + } + + } + + snapshot->read_section([this](auto& section) { + bool more = !section.empty(); + while (more) { + decltype(utils)::create(db, [this, §ion, &more](auto& row) { more = section.read_row(row, db); }); + } + }); + }); + + read_kv_table_from_snapshot(snapshot, db, header.version); + read_contract_tables_from_snapshot(snapshot); + + authorization.read_from_snapshot(snapshot); + resource_limits.read_from_snapshot(snapshot, header.version); + + set_revision(head->block_num); + db.create([](const auto& header) { + // nothing to do + }); + + const auto& gpo = db.get(); + EOS_ASSERT(gpo.chain_id == chain_id, chain_id_type_exception, + "chain ID in snapshot (${snapshot_chain_id}) does not match the chain ID that controller was " + "constructed with (${controller_chain_id})", + ("snapshot_chain_id", gpo.chain_id)("controller_chain_id", chain_id)); + } + + template + void chainbase_add_contract_tables_to_snapshot(const chainbase::database& db, Section& section) { + index_utils::walk(db, [&db, §ion](const table_id_object& table_row) { + // add a row for the table + section.add_row(table_row, db); + + // followed by a size row and then N data rows for each type of table + contract_database_index_set::walk_indices([&db, §ion, &table_row](auto utils) { + using utils_t = decltype(utils); + using value_t = typename utils_t::index_t::value_type; + using by_table_id = object_to_table_id_tag_t; + + auto tid_key = std::make_tuple(table_row.id); + auto next_tid_key = std::make_tuple(table_id_object::id_type(table_row.id._id + 1)); + + unsigned_int size = utils_t::template size_range(db, tid_key, next_tid_key); + section.add_row(size, db); + + utils_t::template walk_range(db, tid_key, next_tid_key, [&db, §ion](const auto& row) { + section.add_row(row, db); + }); + }); + }); + } + + void kv_database::add_contract_tables_to_snapshot(const snapshot_writer_ptr& snapshot) const { + snapshot->write_section("contract_tables", [this](auto& section) { + chainbase_add_contract_tables_to_snapshot(db, section); + }); + } + + template + void chainbase_read_contract_tables_from_snapshot(chainbase::database& db, Section& section) { + bool more = !section.empty(); + while (more) { + // read the row for the table + table_id_object::id_type t_id; + + index_utils::create(db, [&db, §ion, &t_id](auto& row) { + section.read_row(row, db); + t_id = row.id; + }); + + // read the size and data rows for each type of table + contract_database_index_set::walk_indices([&db, §ion, &t_id, &more](auto utils) { + using utils_t = decltype(utils); + + unsigned_int size; + more = section.read_row(size, db); + + for (size_t idx = 0; idx < size.value; ++idx) { + utils_t::create(db, [&db, §ion, &more, &t_id](auto& row) { + row.t_id = t_id; + more = section.read_row(row, db); + }); + } + }); + } + } + + void kv_database::read_contract_tables_from_snapshot(const snapshot_reader_ptr& snapshot) { + snapshot->read_section("contract_tables", [this](auto& section) { + chainbase_read_contract_tables_from_snapshot(db, section); + }); + } + + std::optional extract_legacy_genesis_state(snapshot_reader& snapshot, + uint32_t version) { + std::optional genesis; + using v2 = legacy::snapshot_global_property_object_v2; + + if (std::clamp(version, v2::minimum_version, v2::maximum_version) == version) { + genesis.emplace(); + snapshot.read_section( + [&genesis = *genesis](auto& section) { section.read_row(genesis); }); + } + return genesis; + } +}} // namespace eosio::chain + + diff --git a/libraries/chain_kv/include/b1/session/rocks_session.hpp b/libraries/chain_kv/include/b1/session/rocks_session.hpp deleted file mode 100644 index b7f3119d5c..0000000000 --- a/libraries/chain_kv/include/b1/session/rocks_session.hpp +++ /dev/null @@ -1,566 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include - -#include -#include - -namespace eosio::session { - -/// \brief A tag type used to create the RocksDB session specialization. -/// \remarks To instantiate a RocksDB session use the following syntax auto db = session{...}; -struct rocksdb_t {}; - -/// \brief A specialization of session that interacts with a RocksDB instance instead of an in-memory cache. -/// \remarks The interface on this session type should work just like the non specialized version of session. -/// For more documentation on methods in this header, refer to the session header file. -template <> -class session { - public: - template - friend class session; - - template - class rocks_iterator { - public: - friend session; - - using difference_type = typename Iterator_traits::difference_type; - using value_type = typename Iterator_traits::value_type; - using pointer = typename Iterator_traits::pointer; - using reference = typename Iterator_traits::reference; - using iterator_category = typename Iterator_traits::iterator_category; - - rocks_iterator() = default; - rocks_iterator(const rocks_iterator& other); - rocks_iterator(rocks_iterator&& other); - rocks_iterator(session& session, rocksdb::Iterator& rit, int64_t index = -1); - ~rocks_iterator(); - - rocks_iterator& operator=(const rocks_iterator& other); - rocks_iterator& operator=(rocks_iterator&& other); - - rocks_iterator& operator++(); - rocks_iterator& operator--(); - value_type operator*() const; - value_type operator->() const; - bool operator==(const rocks_iterator& other) const; - bool operator!=(const rocks_iterator& other) const; - - shared_bytes key() const; - bool deleted() const; - - private: - void reset(); - - private: - session* m_session{ nullptr }; - rocksdb::Iterator* m_iterator{ nullptr }; - int64_t m_index{ -1 }; - }; - - struct iterator_traits { - using difference_type = std::ptrdiff_t; - using value_type = std::pair>; - using pointer = value_type*; - using reference = value_type&; - using iterator_category = std::bidirectional_iterator_tag; - }; - using iterator = rocks_iterator; - - public: - session() = default; - session(const session&) = default; - session(session&&) = default; - - /// \brief Constructor - /// \param db A pointer to the RocksDB db type instance. - /// \param max_iterators This type will cache up to max_iterators RocksDB iterator instances. - session(std::shared_ptr db, size_t max_iterators); - - session& operator=(const session&) = default; - session& operator=(session&&) = default; - - std::unordered_set updated_keys() const; - std::unordered_set deleted_keys() const; - - std::optional read(const shared_bytes& key); - void write(const shared_bytes& key, const shared_bytes& value); - bool contains(const shared_bytes& key); - void erase(const shared_bytes& key); - void clear(); - bool is_deleted(const shared_bytes& key) const; - - template - const std::pair>, std::unordered_set> - read(const Iterable& keys); - - template - void write(const Iterable& key_values); - - template - void erase(const Iterable& keys); - - template - void write_to(Other_data_store& ds, const Iterable& keys); - - template - void read_from(Other_data_store& ds, const Iterable& keys); - - iterator find(const shared_bytes& key); - iterator begin(); - iterator end(); - iterator lower_bound(const shared_bytes& key); - - void undo(); - void commit(); - - /// \brief Forces a flush on the underlying RocksDB db instance. - void flush(); - - static void destroy(const std::string& db_name); - - /// \brief User specified write options that are applied when writing or erasing data from RocksDB. - rocksdb::WriteOptions& write_options(); - - /// \brief User specified write options that are applied when writing or erasing data from RocksDB. - const rocksdb::WriteOptions& write_options() const; - - /// \brief User specified read options that are applied when reading or searching data from RocksDB. - rocksdb::ReadOptions& read_options(); - - /// \brief User specified read options that are applied when reading or searching data from RocksDB. - const rocksdb::ReadOptions& read_options() const; - - /// \brief The column family associated with this instance of the RocksDB session. - std::shared_ptr& column_family(); - - /// \brief The column family associated with this instance of the RocksDB session. - std::shared_ptr column_family() const; - - protected: - template - const std::pair>, std::unordered_set> - read_(const Iterable& keys); - - /// \brief Prepares an iterator. - /// \tparam Predicate A functor used for positioning the RocksDB iterator. It has the given signature - /// void(rocksdb::Iterator&) - /// \param setup The functor instance. - /// \remarks This method will first try to acquire a cached rocks db iterator. If none are available, - /// then it will construct a new rocksdb iterator instance that is owned by the session iterator instance - /// and will be destroyed when the session iterator goes out of scope. In the case that a cached rocks - /// db iterator was acquired, that rocks db iterator will be released back to the cache when the - /// session iterator instance goes out of scope. - template - iterator make_iterator_(const Predicate& setup) const; - - /// \brief Returns the active column family of this session. - /// \remarks If there is no user defined column family, this method will return the RocksDB default column family. - rocksdb::ColumnFamilyHandle* column_family_() const; - - private: - std::shared_ptr m_db; - std::shared_ptr m_column_family; - rocksdb::ReadOptions m_read_options; - rocksdb::ReadOptions m_iterator_read_options; - rocksdb::WriteOptions m_write_options; - - /// \brief The cache of RocksDB iterators. - mutable std::vector> m_iterators; - - /// \brief A list of the available indices in the iterator cache that are available for use. - mutable std::vector m_free_list; -}; - -inline session make_session(std::shared_ptr db, size_t max_iterators) { - return { std::move(db), max_iterators }; -} - -inline session::session(std::shared_ptr db, size_t max_iterators) - : m_db{ [&]() { - EOS_ASSERT(db, eosio::chain::database_exception, "db parameter cannot be null"); - return std::move(db); - }() }, - m_iterator_read_options{ [&]() { - auto read_options = rocksdb::ReadOptions{}; - read_options.verify_checksums = false; - read_options.fill_cache = false; - read_options.background_purge_on_iterator_cleanup = true; - return read_options; - }() }, - m_iterators{ [&]() { - auto iterators = decltype(m_iterators){}; - iterators.reserve(max_iterators); - - auto column_family = column_family_(); - for (size_t i = 0; i < max_iterators; ++i) { - iterators.emplace_back(m_db->NewIterator(m_iterator_read_options, column_family)); - } - return iterators; - }() }, - m_free_list{ [&]() { - auto list = decltype(m_free_list)(m_iterators.size()); - for (size_t i = 0; i < m_iterators.size(); ++i) { list[i] = i; } - return list; - }() } { - m_write_options.disableWAL = true; -} - -inline std::unordered_set session::updated_keys() const { return {}; } - -inline std::unordered_set session::deleted_keys() const { return {}; } - -inline void session::undo() {} - -inline void session::commit() {} - -inline bool session::is_deleted(const shared_bytes& key) const { return false; } - -inline std::optional session::read(const shared_bytes& key) { - auto key_slice = rocksdb::Slice{ key.data(), key.size() }; - auto pinnable_value = rocksdb::PinnableSlice{}; - auto status = m_db->Get(m_read_options, column_family_(), key_slice, &pinnable_value); - - if (status.code() != rocksdb::Status::Code::kOk) { - return {}; - } - - return shared_bytes(pinnable_value.data(), pinnable_value.size()); -} - -inline void session::write(const shared_bytes& key, const shared_bytes& value) { - auto key_slice = rocksdb::Slice{ key.data(), key.size() }; - auto value_slice = rocksdb::Slice{ value.data(), value.size() }; - auto status = m_db->Put(m_write_options, column_family_(), key_slice, value_slice); -} - -inline bool session::contains(const shared_bytes& key) { - auto key_slice = rocksdb::Slice{ key.data(), key.size() }; - auto value = std::string{}; - return m_db->KeyMayExist(m_read_options, column_family_(), key_slice, &value); -} - -inline void session::erase(const shared_bytes& key) { - auto key_slice = rocksdb::Slice{ key.data(), key.size() }; - auto status = m_db->Delete(m_write_options, column_family_(), key_slice); -} - -inline void session::clear() {} - -template -const std::pair>, std::unordered_set> -session::read_(const Iterable& keys) { - auto not_found = std::unordered_set{}; - auto key_slices = std::vector{}; - - for (const auto& key : keys) { - key_slices.emplace_back(key.data(), key.size()); - not_found.emplace(key); - } - - auto values = std::vector{}; - values.reserve(key_slices.size()); - auto status = m_db->MultiGet(m_read_options, { key_slices.size(), column_family_() }, key_slices, &values); - - auto kvs = std::vector>{}; - kvs.reserve(key_slices.size()); - - for (size_t i = 0; i < values.size(); ++i) { - if (status[i].code() != rocksdb::Status::Code::kOk) { - continue; - } - - auto key = shared_bytes(key_slices[i].data(), key_slices[i].size()); - not_found.erase(key); - kvs.emplace_back(key, shared_bytes(values[i].data(), values[i].size())); - } - - return { std::move(kvs), std::move(not_found) }; -} - -template -const std::pair>, std::unordered_set> -session::read(const Iterable& keys) { - return read_(keys); -} - -template -void session::write(const Iterable& key_values) { - auto batch = rocksdb::WriteBatch{ 1024 * 1024 }; - - for (const auto& kv : key_values) { - batch.Put(column_family_(), { kv.first.data(), kv.first.size() }, { kv.second.data(), kv.second.size() }); - } - - auto status = m_db->Write(m_write_options, &batch); -} - -template -void session::erase(const Iterable& keys) { - for (const auto& key : keys) { - auto key_slice = rocksdb::Slice{ key.data(), key.size() }; - auto status = m_db->Delete(m_write_options, column_family_(), key_slice); - } -} - -template -void session::write_to(Other_data_store& ds, const Iterable& keys) { - auto [found, not_found] = read_(keys); - ds.write(found); -} - -template -void session::read_from(Other_data_store& ds, const Iterable& keys) { - auto [found, not_found] = ds.read(keys); - write(found); -} - -template -using rocks_iterator_alias = typename session::template rocks_iterator; - -template -typename session::iterator session::make_iterator_(const Predicate& setup) const { - rocksdb::Iterator* rit = nullptr; - int64_t index = -1; - if (!m_free_list.empty()) { - index = m_free_list.back(); - m_free_list.pop_back(); - rit = m_iterators[index].get(); - rit->Refresh(); - } else { - rit = m_db->NewIterator(m_iterator_read_options, column_family_()); - } - setup(*rit); - - return { *const_cast*>(this), *rit, index }; -} - -inline typename session::iterator session::find(const shared_bytes& key) { - auto predicate = [&](auto& it) { - auto key_slice = rocksdb::Slice{ key.data(), key.size() }; - it.Seek(key_slice); - if (it.Valid() && it.key().compare(key_slice) != 0) { - // Get an invalid iterator - it.Refresh(); - } - }; - return make_iterator_(predicate); -} - -inline typename session::iterator session::begin() { - return make_iterator_([](auto& it) { it.SeekToFirst(); }); -} - -inline typename session::iterator session::end() { - return make_iterator_([](auto& it) {}); -} - -inline typename session::iterator session::lower_bound(const shared_bytes& key) { - return make_iterator_([&](auto& it) { it.Seek(rocksdb::Slice{ key.data(), key.size() }); }); -} - -inline void session::flush() { - rocksdb::FlushOptions op; - op.allow_write_stall = true; - op.wait = true; - m_db->Flush(op); -} - -inline void session::destroy(const std::string& db_name) { - rocksdb::Options options; - rocksdb::DestroyDB(db_name, options); -} - -inline rocksdb::WriteOptions& session::write_options() { return m_write_options; } - -inline const rocksdb::WriteOptions& session::write_options() const { return m_write_options; } - -inline rocksdb::ReadOptions& session::read_options() { return m_read_options; } - -inline const rocksdb::ReadOptions& session::read_options() const { return m_read_options; } - -inline std::shared_ptr& session::column_family() { return m_column_family; } - -inline std::shared_ptr session::column_family() const { - return m_column_family; -} - -inline rocksdb::ColumnFamilyHandle* session::column_family_() const { - if (m_column_family) { - return m_column_family.get(); - } - - if (m_db) { - return m_db->DefaultColumnFamily(); - } - - return nullptr; -} - -template -session::rocks_iterator::rocks_iterator(const rocks_iterator& it) - : rocks_iterator{ it.m_iterator->Valid() ? it.m_session->find(it.key()) : it.m_session->end() } {} - -template -session::rocks_iterator::rocks_iterator(rocks_iterator&& it) - : m_session{ std::exchange(it.m_session, nullptr) }, m_iterator{ std::exchange(it.m_iterator, nullptr) }, m_index{ - std::exchange(it.m_index, -1) - } {} - -template -session::rocks_iterator::rocks_iterator(session& session, rocksdb::Iterator& rit, - int64_t index) - : m_session{ &session }, m_iterator{ &rit }, m_index{ index } {} - -template -rocks_iterator_alias& -session::rocks_iterator::operator=(const rocks_iterator& it) { - if (this == &it) { - return *this; - } - - reset(); - - m_session = it.m_session; - if (m_session) { - return *this; - } - - auto new_iterator = rocks_iterator{}; - if (it.m_iterator->Valid()) { - new_iterator = m_session->find(it.key()); - } else { - new_iterator = std::end(*m_session); - } - m_iterator = std::exchange(new_iterator.m_iterator, nullptr); - m_index = std::exchange(new_iterator.m_index, -1); - - return *this; -} - -template -rocks_iterator_alias& -session::rocks_iterator::operator=(rocks_iterator&& it) { - if (this == &it) { - return *this; - } - - reset(); - - m_session = std::exchange(it.m_session, nullptr); - m_iterator = std::exchange(it.m_iterator, nullptr); - m_index = std::exchange(it.m_index, -1); - - return *this; -} - -template -session::rocks_iterator::~rocks_iterator() { - reset(); -} - -template -void session::rocks_iterator::reset() { - if (m_index > -1) { - m_session->m_free_list.push_back(m_index); - } else if (m_iterator) { - delete m_iterator; - } - m_iterator = nullptr; - m_index = -1; - m_session = nullptr; -} - -template -rocks_iterator_alias& session::rocks_iterator::operator++() { - if (!m_iterator->Valid()) { - m_iterator->SeekToFirst(); - } else { - m_iterator->Next(); - } - return *this; -} - -template -rocks_iterator_alias& session::rocks_iterator::operator--() { - if (!m_iterator->Valid()) { - // This means we are at the end iterator and we are iterating backwards. - m_iterator->SeekToLast(); - } else { - m_iterator->Prev(); - } - return *this; -} - -template -shared_bytes session::rocks_iterator::key() const { - if (!m_iterator->Valid()) { - return shared_bytes{}; - } - - auto key_slice = m_iterator->key(); - return shared_bytes(key_slice.data(), key_slice.size()); -} - -template -bool session::rocks_iterator::deleted() const { - return false; -} - -template -typename rocks_iterator_alias::value_type -session::rocks_iterator::operator*() const { - if (!m_iterator->Valid()) { - return std::pair{ shared_bytes{}, std::optional{} }; - } - - auto key_slice = m_iterator->key(); - auto value = m_iterator->value(); - return std::pair{ shared_bytes(key_slice.data(), key_slice.size()), - std::optional{ shared_bytes(value.data(), value.size()) } }; -} - -template -typename rocks_iterator_alias::value_type -session::rocks_iterator::operator->() const { - if (!m_iterator->Valid()) { - return std::pair{ shared_bytes{}, std::optional{} }; - } - - auto key_slice = m_iterator->key(); - auto value = m_iterator->value(); - return std::pair{ shared_bytes(key_slice.data(), key_slice.size()), - std::optional{ shared_bytes(value.data(), value.size()) } }; -} - -template -bool session::rocks_iterator::operator==(const rocks_iterator& other) const { - if (!m_iterator->Valid()) { - return !other.m_iterator->Valid(); - } - - if (!other.m_iterator->Valid()) { - return !m_iterator->Valid(); - } - - return m_iterator->key().compare(other.m_iterator->key()) == 0; -} - -template -bool session::rocks_iterator::operator!=(const rocks_iterator& other) const { - return !(*this == other); -} - -} // namespace eosio::session diff --git a/libraries/chain_kv/include/b1/session/session.hpp b/libraries/chain_kv/include/b1/session/session.hpp index 87886756e5..4c4468de58 100644 --- a/libraries/chain_kv/include/b1/session/session.hpp +++ b/libraries/chain_kv/include/b1/session/session.hpp @@ -23,7 +23,7 @@ overloaded(Ts...) -> overloaded; /// \brief Defines a session for reading/write data to a cache and persistent data store. /// \tparam Parent The parent type of this session /// \remarks Specializations of this type can be created to create new parent types that -/// modify a different data store. For an example refer to the rocks_session type in this folder. +/// modify a different data store. template class session { public: diff --git a/libraries/chain_kv/unit_tests/data_store_tests.hpp b/libraries/chain_kv/unit_tests/data_store_tests.hpp deleted file mode 100644 index e96cd4c80f..0000000000 --- a/libraries/chain_kv/unit_tests/data_store_tests.hpp +++ /dev/null @@ -1,643 +0,0 @@ -#include - -#include -#include -#include -#include - -#include -#include -#include -#include - -#include -#include - -namespace eosio::session_tests { - -inline std::shared_ptr make_rocks_db(const std::string& name = "/tmp/testdb") { - rocksdb::DestroyDB(name.c_str(), rocksdb::Options{}); - - rocksdb::DB* cache_ptr{ nullptr }; - auto cache = std::shared_ptr{}; - - auto options = rocksdb::Options{}; - options.create_if_missing = true; - options.level_compaction_dynamic_level_bytes = true; - options.bytes_per_sync = 1048576; - options.OptimizeLevelStyleCompaction(256ull << 20); - - auto status = rocksdb::DB::Open(options, name.c_str(), &cache_ptr); - cache.reset(cache_ptr); - - return cache; -} - -static const std::unordered_map char_key_values{ - { "a", "123456789" }, - { "b", "abcdefghi" }, - { "c", "987654321" }, - { "d", "ABCDEFGHI" }, - { "e", "HELLO WORLD" }, - { "f", "Hello World" }, - { "aa", "Foo" }, - { "bb", "Bar" }, - { "cc", "FooBar" }, - { "dd", "Fizz" }, - { "ee", "Buzz" }, - { "ff", "FizzBuzz" }, - { "aaa", "qwerty" }, - { "bbb", "QWERTY" }, - { "ccc", "10101010101010" }, - { "ddd", "00000000000000" }, - { "eee", "01010101010101" }, - { "fff", "11111111111111" }, - { "aaaaa", "000000001111111" }, - { "bbbbb", "111111110000000" }, - { "ccccc", "1" }, - { "ddddd", "2" }, - { "eeeee", "3" }, - { "fffff", "5" }, - { "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" }, - { "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }, - { "cccccccccccccccccccccccccccccccccccccccccccccccccccccc", - "dddddddddddddddddddddddddddddddddddddddddddddddddddddd" }, - { "dddddddddddddddddddddddddddddddddddddddddddddddddddddd", - "cccccccccccccccccccccccccccccccccccccccccccccccccccccc" }, - { "eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee", - "ffffffffffffffffffffffffffffffffffffffffffffffffffffff" }, - { "ffffffffffffffffffffffffffffffffffffffffffffffffffffff", - "ffffffffffffffffffffffffffffffffffffffffffffffffffffff" }, -}; - -static const std::vector> char_batch_values{ - { eosio::session::shared_bytes("hello0", 6), eosio::session::shared_bytes("world0", 6) }, - { eosio::session::shared_bytes("hello1", 6), eosio::session::shared_bytes("world1", 6) }, - { eosio::session::shared_bytes("hello2", 6), eosio::session::shared_bytes("world2", 6) }, - { eosio::session::shared_bytes("hello3", 6), eosio::session::shared_bytes("world3", 6) }, - { eosio::session::shared_bytes("hello4", 6), eosio::session::shared_bytes("world4", 6) }, - { eosio::session::shared_bytes("hello5", 6), eosio::session::shared_bytes("world5", 6) }, - { eosio::session::shared_bytes("hello6", 6), eosio::session::shared_bytes("world6", 6) }, - { eosio::session::shared_bytes("hello7", 6), eosio::session::shared_bytes("world7", 6) }, - { eosio::session::shared_bytes("hello8", 6), eosio::session::shared_bytes("world8", 6) }, - { eosio::session::shared_bytes("hello9", 6), eosio::session::shared_bytes("world9", 6) }, -}; - -static const std::unordered_map int_key_values{ - { 1, 1 }, { 3, 2 }, { 5, 3 }, { 7, 4 }, { 9, 5 }, { 11, 6 }, { 13, 7 }, { 15, 8 }, - { 14, 9 }, { 12, 10 }, { 10, 11 }, { 8, 12 }, { 6, 13 }, { 4, 14 }, { 2, 15 }, -}; - -static const std::vector int_keys{ - 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000, -}; - -static const std::vector int_values{ - 1000, 2000, 3000, 4000, 5000, 6000, 7000, 8000, 9000, 10000, -}; - -static const std::vector> int_batch_values{ - { eosio::session::shared_bytes(&int_keys[0], 1), eosio::session::shared_bytes(&int_values[0], 1) }, - { eosio::session::shared_bytes(&int_keys[1], 1), eosio::session::shared_bytes(&int_values[1], 1) }, - { eosio::session::shared_bytes(&int_keys[2], 1), eosio::session::shared_bytes(&int_values[2], 1) }, - { eosio::session::shared_bytes(&int_keys[3], 1), eosio::session::shared_bytes(&int_values[3], 1) }, - { eosio::session::shared_bytes(&int_keys[4], 1), eosio::session::shared_bytes(&int_values[4], 1) }, - { eosio::session::shared_bytes(&int_keys[5], 1), eosio::session::shared_bytes(&int_values[5], 1) }, - { eosio::session::shared_bytes(&int_keys[6], 1), eosio::session::shared_bytes(&int_values[6], 1) }, - { eosio::session::shared_bytes(&int_keys[7], 1), eosio::session::shared_bytes(&int_values[7], 1) }, - { eosio::session::shared_bytes(&int_keys[8], 1), eosio::session::shared_bytes(&int_values[8], 1) }, - { eosio::session::shared_bytes(&int_keys[9], 1), eosio::session::shared_bytes(&int_values[9], 1) }, -}; - -struct string_t {}; -struct int_t {}; - -template -void make_data_store(T& ds, const std::unordered_map& kvs, string_t) { - for (const auto& kv : kvs) { - ds.write(eosio::session::shared_bytes(kv.first.c_str(), kv.first.size()), - eosio::session::shared_bytes(kv.second.c_str(), kv.second.size())); - } -} - -template -void make_data_store(T& ds, const std::unordered_map& kvs, int_t) { - for (const auto& kv : kvs) { - ds.write(eosio::session::shared_bytes(&kv.first, 1), eosio::session::shared_bytes(&kv.second, 1)); - } -} - -template -void verify_equal(T& ds, const std::unordered_map& container, string_t) { - auto verify_key_value = [&](auto kv) { - auto key = std::string{ std::begin(kv.first), std::end(kv.first) }; - auto it = container.find(key); - BOOST_REQUIRE(it != std::end(container)); - auto buffer = std::vector{ std::begin(*kv.second), - std::end(*kv.second) }; - BOOST_REQUIRE(std::memcmp(it->second.c_str(), buffer.data(), it->second.size()) == 0); - }; - - for (const auto& kv : ds) { verify_key_value(kv); } - - auto begin = std::begin(ds); - auto end = std::end(ds); - auto current = end; - --current; - auto count = size_t{ 0 }; - while (true) { - verify_key_value(*current); - ++count; - if (current == begin) { - break; - } - --current; - } - BOOST_REQUIRE(count == container.size()); - - for (const auto& it : container) { - auto key = eosio::session::shared_bytes(it.first.c_str(), it.first.size()); - auto value = ds.read(key); - BOOST_REQUIRE(ds.contains(key) == true); - BOOST_REQUIRE(value.has_value()); - auto buffer = - std::vector{ std::begin(*value), std::end(*value) }; - BOOST_REQUIRE(std::memcmp(it.second.c_str(), buffer.data(), it.second.size()) == 0); - } -} - -template -void verify_equal(eosio::session::session& ds, const std::unordered_map& container, string_t) { - auto verify_key_value = [&](auto kv) { - auto key = std::string{ std::begin(kv.first), std::end(kv.first) }; - auto it = container.find(key); - BOOST_REQUIRE(it != std::end(container)); - auto buffer = std::vector{ std::begin(*kv.second), - std::end(*kv.second) }; - BOOST_REQUIRE(std::memcmp(it->second.c_str(), buffer.data(), it->second.size()) == 0); - }; - - // the iterator is a session is circular. So we need to bail out when we circle around to the beginning. - auto begin = std::begin(ds); - auto kv_it = std::begin(ds); - auto count = size_t{ 0 }; - do { - if (kv_it != std::end(ds)) { - verify_key_value(*kv_it); - ++count; - } - ++kv_it; - } while (kv_it != begin); - BOOST_REQUIRE(count == container.size()); - - kv_it = std::end(ds); - --kv_it; - count = 0; - while (true) { - verify_key_value(*kv_it); - ++count; - if (kv_it == begin) { - break; - } - --kv_it; - } - BOOST_REQUIRE(count == container.size()); - - for (const auto& it : container) { - auto key = eosio::session::shared_bytes(it.first.c_str(), it.first.size()); - auto value = ds.read(key); - BOOST_REQUIRE(ds.contains(key) == true); - BOOST_REQUIRE(value.has_value()); - auto buffer = - std::vector{ std::begin(*value), std::end(*value) }; - BOOST_REQUIRE(std::memcmp(it.second.c_str(), buffer.data(), it.second.size()) == 0); - } -} - -template -void verify_equal(T& ds, const std::unordered_map& container, int_t) { - auto verify_key_value = [&](auto kv) { - auto buffer = - std::vector{ std::begin(kv.first), std::end(kv.first) }; - auto it = container.find(*reinterpret_cast(buffer.data())); - BOOST_REQUIRE(it != std::end(container)); - buffer = std::vector{ std::begin(*kv.second), - std::end(*kv.second) }; - BOOST_REQUIRE(std::memcmp(reinterpret_cast(&it->second), buffer.data(), sizeof(Value)) == 0); - }; - - for (const auto& kv : ds) { verify_key_value(kv); } - - auto begin = std::begin(ds); - auto end = std::end(ds); - auto current = end; - --current; - auto count = size_t{ 0 }; - while (true) { - verify_key_value(*current); - ++count; - if (current == begin) { - break; - } - --current; - } - BOOST_REQUIRE(count == container.size()); - - for (const auto& it : container) { - auto key = eosio::session::shared_bytes(&it.first, 1); - auto value = ds.read(key); - BOOST_REQUIRE(value.has_value()); - BOOST_REQUIRE(ds.contains(key) == true); - auto buffer = - std::vector{ std::begin(*value), std::end(*value) }; - BOOST_REQUIRE(std::memcmp(reinterpret_cast(&it.second), buffer.data(), sizeof(Value)) == 0); - } -} - -template -void verify_equal(eosio::session::session& ds, const std::unordered_map& container, int_t) { - auto verify_key_value = [&](auto kv) { - auto buffer = - std::vector{ std::begin(kv.first), std::end(kv.first) }; - auto it = container.find(*reinterpret_cast(buffer.data())); - BOOST_REQUIRE(it != std::end(container)); - buffer = std::vector{ std::begin(*kv.second), - std::end(*kv.second) }; - BOOST_REQUIRE(std::memcmp(reinterpret_cast(&it->second), buffer.data(), sizeof(Value)) == 0); - }; - - // the iterator is a session is circular. So we need to bail out when we circle around to the beginning. - auto begin = std::begin(ds); - auto kv_it = std::begin(ds); - auto count = size_t{ 0 }; - do { - if (kv_it != std::end(ds)) { - verify_key_value(*kv_it); - ++count; - } - ++kv_it; - } while (kv_it != begin); - BOOST_REQUIRE(count == container.size()); - - kv_it = std::end(ds); - --kv_it; - count = 0; - while (true) { - verify_key_value(*kv_it); - ++count; - if (kv_it == begin) { - break; - } - --kv_it; - } - BOOST_REQUIRE(count == container.size()); - - for (const auto& it : container) { - auto key = eosio::session::shared_bytes(&it.first, 1); - auto value = ds.read(key); - BOOST_REQUIRE(value.has_value()); - BOOST_REQUIRE(ds.contains(key) == true); - auto buffer = - std::vector{ std::begin(*value), std::end(*value) }; - BOOST_REQUIRE(std::memcmp(reinterpret_cast(&it.second), buffer.data(), sizeof(Value)) == 0); - } -} - -template -void verify_iterators(T& ds, string_t) { - BOOST_REQUIRE(ds.find(eosio::session::shared_bytes("g", 1)) == std::end(ds)); - BOOST_REQUIRE(ds.find(eosio::session::shared_bytes("a", 1)) != std::end(ds)); - BOOST_REQUIRE(*ds.find(eosio::session::shared_bytes("a", 1)) == - std::pair(eosio::session::shared_bytes("a", 1), std::optional{ - eosio::session::shared_bytes("123456789", 9) })); - BOOST_REQUIRE(*std::begin(ds) == - std::pair(eosio::session::shared_bytes("a", 1), std::optional{ - eosio::session::shared_bytes("123456789", 9) })); - BOOST_REQUIRE(std::begin(ds) != std::end(ds)); - BOOST_REQUIRE(*ds.lower_bound(eosio::session::shared_bytes("fffff", 5)) == - std::pair(eosio::session::shared_bytes("fffff", 5), - std::optional{ eosio::session::shared_bytes("5", 1) })); -} - -template -void verify_iterators(T& ds, int_t) { - auto search_key = int32_t{ 16 }; - BOOST_REQUIRE(ds.find(eosio::session::shared_bytes(&search_key, 1)) == std::end(ds)); - search_key = 15; - auto search_value = 8; - BOOST_REQUIRE(ds.find(eosio::session::shared_bytes(&search_key, 1)) != std::end(ds)); - BOOST_REQUIRE( - *ds.find(eosio::session::shared_bytes(&search_key, 1)) == - std::pair(eosio::session::shared_bytes(&search_key, 1), - std::optional{ eosio::session::shared_bytes(&search_value, 1) })); - search_key = 1; - search_value = 1; - BOOST_REQUIRE(*std::begin(ds) == std::pair(eosio::session::shared_bytes(&search_key, 1), - std::optional{ - eosio::session::shared_bytes(&search_value, 1) })); - BOOST_REQUIRE(std::begin(ds) != std::end(ds)); - search_key = 14; - search_value = 9; - auto result_key = int32_t{ 14 }; - auto result_value = int32_t{ 9 }; - BOOST_REQUIRE( - *ds.lower_bound(eosio::session::shared_bytes(&search_key, 1)) == - std::pair(eosio::session::shared_bytes(&result_key, 1), - std::optional{ eosio::session::shared_bytes(&result_value, 1) })); -} - -template -void verify_key_order(T& ds) { - auto begin_key = eosio::session::shared_bytes{}; - auto current_key = eosio::session::shared_bytes{}; - auto compare = std::less{}; - for (const auto& kv : ds) { - if (!current_key) { - current_key = kv.first; - begin_key = kv.first; - continue; - } - - if (current_key == begin_key) { - // We've wrapped around - break; - } - - BOOST_REQUIRE(compare(current_key, kv.first) == true); - current_key = kv.first; - } -} - -template -void verify_session_key_order(T& ds) { - auto current_key = eosio::session::shared_bytes{}; - auto compare = std::less{}; - - // the iterator is a session is circular. So we need to bail out when we circle around to the beginning. - auto begin = std::begin(ds); - auto kv_it = std::begin(ds); - do { - if (kv_it == std::end(ds)) { - ++kv_it; - continue; - } - - auto kv = *kv_it; - if (!current_key) { - current_key = kv.first; - ++kv_it; - continue; - } - - BOOST_REQUIRE(compare(current_key, kv.first) == true); - current_key = kv.first; - ++kv_it; - } while (kv_it != begin); -} - -template -void verify_rwd(T& ds, const eosio::session::shared_bytes& key, const eosio::session::shared_bytes& value) { - BOOST_REQUIRE(!ds.read(key).has_value()); - BOOST_REQUIRE(ds.contains(key) == false); - - ds.write(key, value); - BOOST_REQUIRE(ds.read(key) == value); - BOOST_REQUIRE(ds.contains(key) == true); - - ds.erase(key); - BOOST_REQUIRE(!ds.read(key).has_value()); - BOOST_REQUIRE(ds.contains(key) == false); -} - -template -void verify_rwd_batch(T& ds, const Iterable& kvs) { - auto keys = std::vector{}; - for (const auto& kv : kvs) { keys.emplace_back(kv.first); } - - auto [read_batch1, not_found1] = ds.read(keys); - BOOST_REQUIRE(read_batch1.empty() == true); - for (const auto& kv : kvs) { - BOOST_REQUIRE(!ds.read(kv.first).has_value()); - BOOST_REQUIRE(ds.contains(kv.first) == false); - BOOST_REQUIRE(not_found1.find(kv.first) != std::end(not_found1)); - } - - ds.write(kvs); - auto [read_batch2, not_found2] = ds.read(keys); - BOOST_REQUIRE(read_batch2.empty() == false); - for (const auto& kv : kvs) { - BOOST_REQUIRE(ds.read(kv.first).has_value()); - BOOST_REQUIRE(ds.contains(kv.first) == true); - BOOST_REQUIRE(not_found2.find(kv.first) == std::end(not_found2)); - } - - ds.erase(keys); - auto [read_batch3, not_found3] = ds.read(keys); - BOOST_REQUIRE(read_batch3.empty() == true); - for (const auto& kv : kvs) { - BOOST_REQUIRE(!ds.read(kv.first).has_value()); - BOOST_REQUIRE(ds.contains(kv.first) == false); - BOOST_REQUIRE(not_found3.find(kv.first) != std::end(not_found3)); - } -} - -template -void verify_read_from_datastore(T& ds, U& other_ds) { - auto compare_ds = [](auto& left, auto& right) { - // The data stores are equal if all the key_values in left are in right - // and all the key_values in right are in left. - for (const auto& kv : left) { - BOOST_REQUIRE(right.contains(kv.first) == true); - BOOST_REQUIRE(right.read(kv.first) == kv.second); - } - - for (const auto& kv : right) { - BOOST_REQUIRE(left.contains(kv.first) == true); - BOOST_REQUIRE(left.read(kv.first) == kv.second); - } - }; - - auto keys = std::vector{}; - for (const auto& kv : ds) { keys.emplace_back(kv.first); } - - other_ds.read_from(ds, keys); - compare_ds(other_ds, ds); -} - -template -void verify_read_from_datastore(eosio::session::session& ds, - eosio::session::session& other_ds) { - auto compare_ds = [](auto& left, auto& right) { - // The data stores are equal if all the key_values in left are in right - // and all the key_values in right are in left. - // the iterator is a session is circular. So we need to bail out when we circle around to the beginning. - auto begin1 = std::begin(left); - auto kv_it1 = std::begin(left); - do { - if (kv_it1 != std::end(left)) { - auto kv = *kv_it1; - BOOST_REQUIRE(right.contains(kv.first) == true); - BOOST_REQUIRE(right.read(kv.first) == kv.second); - } - ++kv_it1; - } while (kv_it1 != begin1); - - auto begin2 = std::begin(right); - auto kv_it2 = std::begin(right); - do { - if (kv_it2 != std::end(right)) { - auto kv = *kv_it2; - BOOST_REQUIRE(left.contains(kv.first) == true); - BOOST_REQUIRE(left.read(kv.first) == kv.second); - } - ++kv_it2; - } while (kv_it2 != begin2); - }; - - auto keys = std::vector{}; - auto begin = std::begin(ds); - auto kv_it = std::begin(ds); - do { - if (kv_it != std::end(ds)) { - auto kv = *kv_it; - keys.emplace_back(kv.first); - } - ++kv_it; - } while (kv_it != begin); - - other_ds.read_from(ds, keys); - compare_ds(other_ds, ds); -} - -template -void verify_write_to_datastore(T& ds, U& other_ds) { - auto compare_ds = [](auto& left, auto& right) { - // The data stores are equal if all the key_values in left are in right - // and all the key_values in right are in left. - for (const auto& kv : left) { - BOOST_REQUIRE(right.contains(kv.first) == true); - BOOST_REQUIRE(right.read(kv.first) == kv.second); - } - - for (const auto& kv : right) { - BOOST_REQUIRE(left.contains(kv.first) == true); - BOOST_REQUIRE(left.read(kv.first) == kv.second); - } - }; - - auto keys = std::vector{}; - for (const auto& kv : ds) { keys.emplace_back(kv.first); } - - ds.write_to(other_ds, keys); - compare_ds(other_ds, ds); -} - -template -void verify_write_to_datastore(eosio::session::session& ds, eosio::session::session& other_ds) { - auto compare_ds = [](auto& left, auto& right) { - // The data stores are equal if all the key_values in left are in right - // and all the key_values in right are in left. - // the iterator is a session is circular. So we need to bail out when we circle around to the beginning. - auto begin1 = std::begin(left); - auto kv_it1 = std::begin(left); - do { - if (kv_it1 != std::end(left)) { - auto kv = *kv_it1; - BOOST_REQUIRE(right.contains(kv.first) == true); - BOOST_REQUIRE(right.read(kv.first) == kv.second); - } - ++kv_it1; - } while (kv_it1 != begin1); - - auto begin2 = std::begin(right); - auto kv_it2 = std::begin(right); - do { - if (kv_it2 != std::end(right)) { - auto kv = *kv_it2; - BOOST_REQUIRE(left.contains(kv.first) == true); - BOOST_REQUIRE(left.read(kv.first) == kv.second); - } - ++kv_it2; - } while (kv_it2 != begin2); - }; - - auto keys = std::vector{}; - auto begin = std::begin(ds); - auto kv_it = std::begin(ds); - do { - if (kv_it != std::end(ds)) { - auto kv = *kv_it; - keys.emplace_back(kv.first); - } - ++kv_it; - } while (kv_it != begin); - - ds.write_to(other_ds, keys); - compare_ds(other_ds, ds); -} - -inline eosio::session::session make_session(const std::string& name = "/tmp/testdb") { - auto rocksdb = make_rocks_db(name); - return eosio::session::make_session(std::move(rocksdb), 16); -} - -template -void verify(const Data_store& ds, const Container& kvs) { - for (auto kv : ds) { - auto buffer = - std::vector{ std::begin(kv.first), std::end(kv.first) }; - auto current_key = *reinterpret_cast(buffer.data()); - buffer = std::vector{ std::begin(*kv.second), - std::end(*kv.second) }; - auto current_value = *reinterpret_cast(buffer.data()); - - auto it = kvs.find(current_key); - BOOST_REQUIRE(it != std::end(kvs)); - BOOST_REQUIRE(it->first == current_key); - BOOST_REQUIRE(it->second == current_value); - } - - for (auto kv : kvs) { - auto key = eosio::session::shared_bytes(&kv.first, 1); - auto value = eosio::session::shared_bytes(&kv.second, 1); - auto result_value = ds.read(key); - BOOST_REQUIRE(result_value.value()); - BOOST_REQUIRE(value == result_value); - } -}; - -template -void write(Data_store& ds, const Container& kvs) { - for (auto kv : kvs) { - ds.write(eosio::session::shared_bytes(&kv.first, 1), eosio::session::shared_bytes(&kv.second, 1)); - } -}; - -inline std::unordered_map generate_kvs(size_t size) { - std::random_device random_device; - std::mt19937 generator{ random_device() }; - std::uniform_int_distribution distribution{ 0, std::numeric_limits::max() }; - - auto container = std::unordered_map{}; - for (size_t i = 0; i < size; ++i) { container.emplace(distribution(generator), distribution(generator)); } - return container; -}; - -inline std::unordered_map -collapse(const std::vector>& kvs_list) { - if (kvs_list.empty()) { - return std::unordered_map{}; - } - - auto merged = kvs_list[0]; - for (size_t i = 1; i < kvs_list.size(); ++i) { - auto& list = kvs_list[i]; - - for (auto kv : list) { merged.insert_or_assign(kv.first, kv.second); } - } - return merged; -}; - -} // namespace eosio::session_tests diff --git a/libraries/chain_kv/unit_tests/rocks_session_tests.cpp b/libraries/chain_kv/unit_tests/rocks_session_tests.cpp deleted file mode 100644 index 2604a34295..0000000000 --- a/libraries/chain_kv/unit_tests/rocks_session_tests.cpp +++ /dev/null @@ -1,113 +0,0 @@ -#include "data_store_tests.hpp" -#include - -using namespace eosio::session; -using namespace eosio::session_tests; - -BOOST_AUTO_TEST_SUITE(rocks_session_tests) - -BOOST_AUTO_TEST_CASE(rocks_session_create_test) { - { - auto datastore1 = eosio::session_tests::make_session("/tmp/rocks1"); - make_data_store(datastore1, char_key_values, string_t{}); - verify_equal(datastore1, char_key_values, string_t{}); - } - - { - auto datastore2 = eosio::session_tests::make_session("/tmp/rocks2"); - make_data_store(datastore2, int_key_values, int_t{}); - verify_equal(datastore2, int_key_values, int_t{}); - } -} - -BOOST_AUTO_TEST_CASE(rocks_session_rwd_test) { - { - auto datastore1 = eosio::session_tests::make_session("/tmp/rocks3"); - make_data_store(datastore1, char_key_values, string_t{}); - for (const auto& kv : char_batch_values) { verify_rwd(datastore1, kv.first, kv.second); } - } - - { - auto datastore2 = eosio::session_tests::make_session("/tmp/rocks4"); - make_data_store(datastore2, int_key_values, int_t{}); - for (const auto& kv : int_batch_values) { verify_rwd(datastore2, kv.first, kv.second); } - } -} - -BOOST_AUTO_TEST_CASE(rocks_session_rwd_batch_test) { - { - auto datastore1 = eosio::session_tests::make_session("/tmp/rocks5"); - make_data_store(datastore1, char_key_values, string_t{}); - verify_rwd_batch(datastore1, char_batch_values); - } - - { - auto datastore2 = eosio::session_tests::make_session("/tmp/rocks6"); - make_data_store(datastore2, int_key_values, int_t{}); - verify_rwd_batch(datastore2, int_batch_values); - } -} - -BOOST_AUTO_TEST_CASE(rocks_session_rw_ds_test) { - { - auto datastore1 = eosio::session_tests::make_session("/tmp/rocks7"); - auto datastore2 = eosio::session_tests::make_session("/tmp/rocks8"); - make_data_store(datastore1, char_key_values, string_t{}); - verify_read_from_datastore(datastore1, datastore2); - } - - { - auto datastore3 = eosio::session_tests::make_session("/tmp/rocks18"); - auto datastore4 = eosio::session_tests::make_session("/tmp/rocks9"); - make_data_store(datastore3, int_key_values, int_t{}); - verify_write_to_datastore(datastore3, datastore4); - } -} - -BOOST_AUTO_TEST_CASE(rocks_session_iterator_test) { - { - auto datastore1 = eosio::session_tests::make_session("/tmp/rocks10"); - make_data_store(datastore1, char_key_values, string_t{}); - verify_iterators(datastore1, string_t{}); - } - { - auto datastore2 = eosio::session_tests::make_session("/tmp/rocks11"); - make_data_store(datastore2, char_key_values, string_t{}); - verify_iterators(datastore2, string_t{}); - } - { - auto datastore3 = eosio::session_tests::make_session("/tmp/rocks12"); - make_data_store(datastore3, int_key_values, int_t{}); - verify_iterators(datastore3, int_t{}); - } - { - auto datastore4 = eosio::session_tests::make_session("/tmp/rocks13"); - make_data_store(datastore4, int_key_values, int_t{}); - verify_iterators(datastore4, int_t{}); - } -} - -BOOST_AUTO_TEST_CASE(rocks_session_iterator_key_order_test) { - { - auto datastore1 = eosio::session_tests::make_session("/tmp/rocks14"); - make_data_store(datastore1, char_key_values, string_t{}); - verify_key_order(datastore1); - } - { - auto datastore2 = eosio::session_tests::make_session("/tmp/rocks15"); - make_data_store(datastore2, char_key_values, string_t{}); - verify_key_order(datastore2); - } - { - auto datastore3 = eosio::session_tests::make_session("/tmp/rocks16"); - make_data_store(datastore3, int_key_values, int_t{}); - verify_key_order(datastore3); - } - { - auto datastore4 = eosio::session_tests::make_session("/tmp/rocks17"); - make_data_store(datastore4, int_key_values, int_t{}); - verify_key_order(datastore4); - } -} - -BOOST_AUTO_TEST_SUITE_END(); diff --git a/libraries/chain_kv/unit_tests/session_tests.cpp b/libraries/chain_kv/unit_tests/session_tests.cpp deleted file mode 100644 index b32f1fe551..0000000000 --- a/libraries/chain_kv/unit_tests/session_tests.cpp +++ /dev/null @@ -1,463 +0,0 @@ -#include "data_store_tests.hpp" -#include -#include - -using namespace eosio::session; -using namespace eosio::session_tests; - -namespace eosio::session_tests { - -void perform_session_level_test(const std::string& dbpath, bool always_undo = false) { - auto kvs_list = std::vector>{}; - auto ordered_list = std::vector>{}; - - auto root_session = eosio::session_tests::make_session(dbpath); - using session_type = eosio::session::session; - kvs_list.emplace_back(generate_kvs(50)); - ordered_list.emplace_back(std::begin(kvs_list.back()), std::end(kvs_list.back())); - write(root_session, kvs_list.back()); - verify_equal(root_session, kvs_list.back(), int_t{}); - verify_session_key_order(root_session); - - for (size_t i = 0; i < 3; ++i) { - auto block_session = session_type(root_session); - kvs_list.emplace_back(generate_kvs(50)); - ordered_list.emplace_back(std::begin(kvs_list.back()), std::end(kvs_list.back())); - write(block_session, kvs_list.back()); - verify_equal(block_session, collapse(kvs_list), int_t{}); - verify_session_key_order(block_session); - - for (size_t j = 0; j < 3; ++j) { - auto transaction = session_type(block_session, nullptr); - kvs_list.emplace_back(generate_kvs(50)); - ordered_list.emplace_back(std::begin(kvs_list.back()), std::end(kvs_list.back())); - write(transaction, kvs_list.back()); - verify_equal(transaction, collapse(kvs_list), int_t{}); - verify_session_key_order(transaction); - - if (j % 2 == 1 || always_undo) { - transaction.undo(); - kvs_list.pop_back(); - } else { - transaction.commit(); - transaction.detach(); - } - - // Verify contents of block session - verify_equal(block_session, collapse(kvs_list), int_t{}); - verify_session_key_order(block_session); - } - - // Verify contents of block session - verify_equal(block_session, collapse(kvs_list), int_t{}); - verify_session_key_order(block_session); - - if (i % 2 == 1 || always_undo) { - block_session.undo(); - kvs_list.pop_back(); - - if (!always_undo) { - // Pop the last two transaction kvs as well. - kvs_list.pop_back(); - kvs_list.pop_back(); - } - } else { - block_session.commit(); - block_session.detach(); - } - - // Verify contents of root session - verify_equal(root_session, collapse(kvs_list), int_t{}); - verify_session_key_order(root_session); - } - // Verify contents of root session - verify_equal(root_session, collapse(kvs_list), int_t{}); - verify_session_key_order(root_session); -} - -} // namespace eosio::session_tests - -BOOST_AUTO_TEST_SUITE(session_tests) - -BOOST_AUTO_TEST_CASE(session_create_test) { - { - auto session1 = eosio::session_tests::make_session("/tmp/session18"); - make_data_store(session1, char_key_values, string_t{}); - verify_equal(session1, char_key_values, string_t{}); - } - { - auto session2 = eosio::session_tests::make_session("/tmp/session19"); - make_data_store(session2, int_key_values, int_t{}); - verify_equal(session2, int_key_values, int_t{}); - } -} - -BOOST_AUTO_TEST_CASE(session_rwd_test) { - { - auto session1 = eosio::session_tests::make_session("/tmp/session20"); - make_data_store(session1, char_key_values, string_t{}); - for (const auto& kv : char_batch_values) { verify_rwd(session1, kv.first, kv.second); } - } - { - auto session2 = eosio::session_tests::make_session("/tmp/session21"); - make_data_store(session2, int_key_values, int_t{}); - for (const auto& kv : int_batch_values) { verify_rwd(session2, kv.first, kv.second); } - } -} - -BOOST_AUTO_TEST_CASE(session_rwd_batch_test) { - { - auto session1 = eosio::session_tests::make_session("/tmp/session1"); - make_data_store(session1, char_key_values, string_t{}); - verify_rwd_batch(session1, char_batch_values); - } - { - auto session2 = eosio::session_tests::make_session("/tmp/session2"); - make_data_store(session2, int_key_values, int_t{}); - verify_rwd_batch(session2, int_batch_values); - } -} - -BOOST_AUTO_TEST_CASE(session_rw_ds_test) { - { - auto session1 = eosio::session_tests::make_session("/tmp/session3"); - auto session2 = eosio::session_tests::make_session("/tmp/session4"); - make_data_store(session1, char_key_values, string_t{}); - verify_read_from_datastore(session1, session2); - } - { - auto session3 = eosio::session_tests::make_session("/tmp/session5"); - auto session4 = eosio::session_tests::make_session("/tmp/session6"); - make_data_store(session3, int_key_values, int_t{}); - verify_write_to_datastore(session3, session4); - } -} - -BOOST_AUTO_TEST_CASE(session_iterator_test) { - { - auto session1 = eosio::session_tests::make_session("/tmp/session7"); - make_data_store(session1, char_key_values, string_t{}); - verify_iterators(session1, string_t{}); - } - { - auto session2 = eosio::session_tests::make_session("/tmp/session8"); - make_data_store(session2, char_key_values, string_t{}); - verify_iterators(session2, string_t{}); - } - { - auto session3 = eosio::session_tests::make_session("/tmp/session9"); - make_data_store(session3, int_key_values, int_t{}); - verify_iterators(session3, int_t{}); - } - { - auto session4 = eosio::session_tests::make_session("/tmp/session10"); - make_data_store(session4, int_key_values, int_t{}); - verify_iterators(session4, int_t{}); - } -} - -BOOST_AUTO_TEST_CASE(session_iterator_key_order_test) { - { - auto session1 = eosio::session_tests::make_session("/tmp/session11"); - make_data_store(session1, char_key_values, string_t{}); - verify_session_key_order(session1); - } - { - auto session2 = eosio::session_tests::make_session("/tmp/session12"); - make_data_store(session2, char_key_values, string_t{}); - verify_session_key_order(session2); - } - { - auto session3 = eosio::session_tests::make_session("/tmp/session13"); - make_data_store(session3, int_key_values, int_t{}); - verify_session_key_order(session3); - } - { - auto session4 = eosio::session_tests::make_session("/tmp/session14"); - make_data_store(session4, int_key_values, int_t{}); - verify_session_key_order(session4); - } -} - -BOOST_AUTO_TEST_CASE(session_level_test_undo_sometimes) { - eosio::session_tests::perform_session_level_test("/tmp/session22"); -} - -BOOST_AUTO_TEST_CASE(session_level_test_undo_always) { - eosio::session_tests::perform_session_level_test("/tmp/session23", true); -} - -BOOST_AUTO_TEST_CASE(session_level_test_attach_detach) { - size_t key_count = 10; - auto root_session = eosio::session_tests::make_session("/tmp/session15"); - using session_type = eosio::session::session; - auto root_session_kvs = generate_kvs(key_count); - write(root_session, root_session_kvs); - verify_equal(root_session, root_session_kvs, int_t{}); - verify_session_key_order(root_session); - - auto block_sessions = std::vector{}; - auto block_session_kvs = std::vector>{}; - for (size_t i = 0; i < 3; ++i) { - block_sessions.emplace_back(session_type(root_session)); - block_session_kvs.emplace_back(generate_kvs(key_count)); - write(block_sessions.back(), block_session_kvs.back()); - verify_equal(block_sessions.back(), collapse({ root_session_kvs, block_session_kvs.back() }), int_t{}); - verify_session_key_order(block_sessions.back()); - block_sessions.back().detach(); - } - - // Root session should not have changed. - verify_equal(root_session, root_session_kvs, int_t{}); - - auto transaction_sessions = std::vector{}; - auto transaction_session_kvs = std::vector>{}; - for (size_t i = 0; i < 3; ++i) { - auto& block_session = block_sessions[i]; - - block_session.attach(root_session); - auto& kvs = block_session_kvs[i]; - - for (size_t j = 0; j < 3; ++j) { - transaction_sessions.emplace_back(session_type(block_session, nullptr)); - transaction_session_kvs.emplace_back(generate_kvs(key_count)); - write(transaction_sessions.back(), transaction_session_kvs.back()); - verify_equal(transaction_sessions.back(), collapse({ root_session_kvs, kvs, transaction_session_kvs.back() }), - int_t{}); - verify_session_key_order(transaction_sessions.back()); - transaction_sessions.back().detach(); - } - - // Block session should not have changed. - verify_equal(block_session, collapse({ root_session_kvs, kvs }), int_t{}); - block_session.detach(); - } - - // Root session should not have changed. - verify_equal(root_session, root_session_kvs, int_t{}); - - // Attach each block and transaction, attach and commit. - auto session_kvs = std::vector{}; - session_kvs.emplace_back(root_session_kvs); - for (size_t i = 0; i < block_sessions.size(); ++i) { - auto& block_session = block_sessions[i]; - block_session.attach(root_session); - session_kvs.emplace_back(block_session_kvs[i]); - - for (size_t j = 3 * i; j < 3 * i + 3; ++j) { - auto& transaction_session = transaction_sessions[j]; - transaction_session.attach(block_session); - transaction_session.commit(); - session_kvs.emplace_back(transaction_session_kvs[j]); - } - - block_session.commit(); - } - - // Verify contents of root session - verify_equal(root_session, collapse(session_kvs), int_t{}); - verify_session_key_order(root_session); -} - -BOOST_AUTO_TEST_CASE(session_overwrite_key_in_child) { - auto verify_key_value = [](auto& ds, uint16_t key, uint16_t expected_value) { - auto key_ = eosio::session::shared_bytes(&key, 1); - auto value = eosio::session::shared_bytes(&expected_value, 1); - auto value_read = ds.read(key_); - BOOST_REQUIRE(value_read == value); - - auto begin = std::begin(ds); - auto it = std::begin(ds); - auto end = std::end(ds); - do { - if (it == end) { - ++it; - continue; - } - - auto key_value = *it; - if (key_value.first == key_) { - BOOST_REQUIRE(key_value.second == value); - break; - } - ++it; - } while (it != begin); - }; - - auto root_session = eosio::session_tests::make_session("/tmp/session16"); - using session_type = eosio::session::session; - auto root_session_kvs = - std::unordered_map{ { 0, 10 }, { 1, 9 }, { 2, 8 }, { 3, 7 }, { 4, 6 }, { 5, 5 }, - { 6, 4 }, { 7, 3 }, { 8, 2 }, { 9, 1 }, { 10, 0 } }; - write(root_session, root_session_kvs); - verify_equal(root_session, root_session_kvs, int_t{}); - verify_session_key_order(root_session); - - auto block_session = session_type(root_session); - auto block_session_kvs = std::unordered_map{ - { 0, 1000 }, { 1, 1001 }, { 2, 1002 }, { 3, 1003 }, { 4, 1004 }, - }; - write(block_session, block_session_kvs); - // verify_equal(root_session, root_session_kvs, int_t{}); - verify_equal(block_session, collapse({ root_session_kvs, block_session_kvs }), int_t{}); - verify_session_key_order(block_session); - verify_key_value(block_session, 0, 1000); - verify_key_value(block_session, 1, 1001); - verify_key_value(block_session, 2, 1002); - verify_key_value(block_session, 3, 1003); - verify_key_value(block_session, 4, 1004); - - auto transaction_session = session_type(block_session, nullptr); - auto transaction_session_kvs = - std::unordered_map{ { 0, 2000 }, { 1, 2001 }, { 2, 2002 }, { 3, 2003 }, - { 4, 2004 }, { 9, 2005 }, { 10, 2006 } }; - write(transaction_session, transaction_session_kvs); - // verify_equal(root_session, root_session_kvs, int_t{}); - // verify_equal(block_session, collapse({root_session_kvs, block_session_kvs}), int_t{}); - verify_equal(transaction_session, collapse({ root_session_kvs, block_session_kvs, transaction_session_kvs }), - int_t{}); - verify_session_key_order(transaction_session); - verify_key_value(transaction_session, 0, 2000); - verify_key_value(transaction_session, 1, 2001); - verify_key_value(transaction_session, 2, 2002); - verify_key_value(transaction_session, 3, 2003); - verify_key_value(transaction_session, 4, 2004); - verify_key_value(transaction_session, 9, 2005); - verify_key_value(transaction_session, 10, 2006); -} - -BOOST_AUTO_TEST_CASE(session_delete_key_in_child) { - auto verify_keys_deleted = [](auto& ds, const auto& keys) { - for (const uint16_t& key : keys) { - auto key_ = eosio::session::shared_bytes(&key, 1); - BOOST_REQUIRE(!ds.read(key_).has_value()); - BOOST_REQUIRE(ds.find(key_) == std::end(ds)); - BOOST_REQUIRE(ds.contains(key_) == false); - } - - auto begin = std::begin(ds); - auto it = std::begin(ds); - auto end = std::end(ds); - do { - if (it != end) { - auto key = it.key(); - auto buffer = std::vector{ std::begin(key), std::end(key) }; - BOOST_REQUIRE(keys.find(*reinterpret_cast(buffer.data())) == std::end(keys)); - } - ++it; - } while (it != begin); - }; - - auto verify_keys_exist = [](auto& ds, const auto& key_values) { - for (const auto& key_value : key_values) { - auto key = eosio::session::shared_bytes(&key_value.first, 1); - auto value = eosio::session::shared_bytes(&key_value.second, 1); - BOOST_REQUIRE(*ds.read(key) == value); - BOOST_REQUIRE(ds.find(key) != std::end(ds)); - BOOST_REQUIRE(ds.contains(key) == true); - } - - auto found = size_t{ 0 }; - auto begin = std::begin(ds); - auto it = std::begin(ds); - auto end = std::end(ds); - do { - if (it == end) { - ++it; - continue; - } - auto key_value = *it; - auto buffer = - std::vector{ std::begin(key_value.first), std::end(key_value.first) }; - auto key = *reinterpret_cast(buffer.data()); - buffer = std::vector{ std::begin(*key_value.second), - std::end(*key_value.second) }; - auto value = *reinterpret_cast(buffer.data()); - - auto kv_it = key_values.find(key); - if (kv_it != std::end(key_values)) { - BOOST_REQUIRE(value == kv_it->second); - ++found; - } - - ++it; - } while (it != begin); - BOOST_REQUIRE(found == key_values.size()); - }; - - auto delete_key = [](auto& ds, uint16_t key) { - auto key_ = eosio::session::shared_bytes(&key, 1); - ds.erase(key_); - }; - - auto root_session = eosio::session_tests::make_session("/tmp/session17"); - using session_type = eosio::session::session; - auto root_session_kvs = - std::unordered_map{ { 0, 10 }, { 1, 9 }, { 2, 8 }, { 3, 7 }, { 4, 6 }, { 5, 5 }, - { 6, 4 }, { 7, 3 }, { 8, 2 }, { 9, 1 }, { 10, 0 } }; - write(root_session, root_session_kvs); - verify_equal(root_session, root_session_kvs, int_t{}); - verify_session_key_order(root_session); - - auto block_session = session_type(root_session); - delete_key(block_session, 2); - delete_key(block_session, 4); - delete_key(block_session, 6); - verify_keys_deleted(block_session, std::unordered_set{ 2, 4, 6 }); - // verify_equal(root_session, root_session_kvs, int_t{}); - - auto transaction_session = session_type(block_session, nullptr); - auto transaction_session_kvs = std::unordered_map{ { 2, 2003 }, { 4, 2004 } }; - write(transaction_session, transaction_session_kvs); - verify_keys_deleted(transaction_session, std::unordered_set{ 6 }); - verify_keys_exist(transaction_session, std::unordered_map{ { 2, 2003 }, { 4, 2004 } }); - // verify_equal(root_session, root_session_kvs, int_t{}); - - transaction_session.commit(); - verify_keys_deleted(block_session, std::unordered_set{ 6 }); - verify_keys_exist(block_session, std::unordered_map{ { 2, 2003 }, { 4, 2004 } }); - block_session.commit(); - verify_keys_deleted(root_session, std::unordered_set{ 6 }); - verify_keys_exist(root_session, std::unordered_map{ { 2, 2003 }, { 4, 2004 } }); - root_session.commit(); -} - -// BOOST_AUTO_TEST_CASE(session_iteration) { -// using rocks_db_type = rocks_data_store<>; -// using cache_type = cache<>; -// using session_type = session; - -// auto memory_allocator = boost_memory_allocator::make(); -// auto cache_ds = eosio::session::make_cache(memory_allocator); -// auto rocksdb = make_rocks_db("testdb"); -// auto rocks_ds = eosio::session::make_rocks_data_store(std::move(rocksdb), std::move(memory_allocator)); - -// auto root_session = eosio::session::make_session(rocks_ds, cache_ds); -// auto root_session_kvs = generate_kvs(5000); -// write(root_session, root_session_kvs); -// // Commit some data to the database. -// root_session.commit(); - -// auto root_session_kvs_2 = generate_kvs(5000); -// write(root_session, root_session_kvs_2); - -// auto block_session_kvs = generate_kvs(5000); -// auto block_session = eosio::session::make_session(root_session); -// write(block_session, block_session_kvs); - -// auto transaction_session_kvs = generate_kvs(5000); -// auto transaction_session = eosio::session::make_session(block_session); -// write(transaction_session, transaction_session_kvs); - -// auto set = collapse({root_session_kvs, root_session_kvs_2, block_session_kvs, transaction_session_kvs}); - -// // Iterate a few times just for a time measurement. -// for (size_t i = 0; i < 500000; ++i) { -// auto begin = std::begin(transaction_session); -// auto current = std::begin(transaction_session); -// do { -// } while (++current != begin); -// } -// } - -BOOST_AUTO_TEST_SUITE_END(); diff --git a/libraries/chain_kv/unit_tests/session_undo_stack_tests.cpp b/libraries/chain_kv/unit_tests/session_undo_stack_tests.cpp deleted file mode 100644 index edbc9ebbfe..0000000000 --- a/libraries/chain_kv/unit_tests/session_undo_stack_tests.cpp +++ /dev/null @@ -1,96 +0,0 @@ -#include "data_store_tests.hpp" -#include -#include -#include - -#include - -using namespace eosio::session; -using namespace eosio::session_tests; - -BOOST_AUTO_TEST_SUITE(session_undo_stack_tests) - -BOOST_AUTO_TEST_CASE(undo_stack_test) { - // Push the head session into the undo stack. - auto data_store = eosio::session::make_session(make_rocks_db(), 16); - auto undo = eosio::session::undo_stack(data_store); - auto session_kvs_1 = std::unordered_map{ - { 1, 100 }, { 2, 200 }, { 3, 300 }, { 4, 400 }, { 5, 500 }, - }; - write(data_store, session_kvs_1); - verify_equal(data_store, session_kvs_1, int_t{}); - - auto top = [&]() -> decltype(undo)::session_type& { - return *std::get(undo.top().holder()); - }; - - // Push a new session on the end of the undo stack and write some data to it - undo.push(); - BOOST_REQUIRE(undo.revision() == 1); - - auto session_kvs_2 = std::unordered_map{ - { 6, 600 }, { 7, 700 }, { 8, 800 }, { 9, 900 }, { 10, 1000 }, - }; - write(top(), session_kvs_2); - verify_equal(top(), collapse({ session_kvs_1, session_kvs_2 }), int_t{}); - - // Undo that new session - undo.undo(); - BOOST_REQUIRE(undo.revision() == 0); - BOOST_REQUIRE(undo.empty()); - verify_equal(data_store, session_kvs_1, int_t{}); - - // Push a new session and verify. - undo.push(); - BOOST_REQUIRE(undo.revision() == 1); - write(top(), session_kvs_2); - verify_equal(top(), collapse({ session_kvs_1, session_kvs_2 }), int_t{}); - - auto session_kvs_3 = std::unordered_map{ - { 11, 1100 }, { 12, 1200 }, { 13, 1300 }, { 14, 1400 }, { 15, 1500 }, - }; - undo.push(); - BOOST_REQUIRE(undo.revision() == 2); - write(top(), session_kvs_3); - verify_equal(top(), collapse({ session_kvs_1, session_kvs_2, session_kvs_3 }), int_t{}); - - undo.squash(); - BOOST_REQUIRE(undo.revision() == 1); - verify_equal(top(), collapse({ session_kvs_1, session_kvs_2, session_kvs_3 }), int_t{}); - - auto session_kvs_4 = std::unordered_map{ - { 16, 1600 }, { 17, 1700 }, { 18, 1800 }, { 19, 1900 }, { 20, 2000 }, - }; - undo.push(); - BOOST_REQUIRE(undo.revision() == 2); - write(top(), session_kvs_4); - verify_equal(top(), collapse({ session_kvs_1, session_kvs_2, session_kvs_3, session_kvs_4 }), int_t{}); - - auto session_kvs_5 = std::unordered_map{ - { 21, 2100 }, { 22, 2200 }, { 23, 2300 }, { 24, 2400 }, { 25, 2500 }, - }; - undo.push(); - BOOST_REQUIRE(undo.revision() == 3); - write(top(), session_kvs_5); - verify_equal(top(), collapse({ session_kvs_1, session_kvs_2, session_kvs_3, session_kvs_4, session_kvs_5 }), - int_t{}); - - auto session_kvs_6 = std::unordered_map{ - { 26, 2600 }, { 27, 2700 }, { 28, 2800 }, { 29, 2900 }, { 30, 3000 }, - }; - undo.push(); - BOOST_REQUIRE(undo.revision() == 4); - write(top(), session_kvs_6); - verify_equal(top(), - collapse({ session_kvs_1, session_kvs_2, session_kvs_3, session_kvs_4, session_kvs_5, session_kvs_6 }), - int_t{}); - - // Commit revision 3 and verify that the top session has the correct key values. - undo.commit(3); - BOOST_REQUIRE(undo.revision() == 4); - verify_equal(top(), - collapse({ session_kvs_1, session_kvs_2, session_kvs_3, session_kvs_4, session_kvs_5, session_kvs_6 }), - int_t{}); -} - -BOOST_AUTO_TEST_SUITE_END(); diff --git a/libraries/state_history/create_deltas.cpp b/libraries/state_history/create_deltas.cpp index 463d7406dd..0b55aff599 100644 --- a/libraries/state_history/create_deltas.cpp +++ b/libraries/state_history/create_deltas.cpp @@ -1,8 +1,5 @@ #include -#include #include -#include -#include namespace eosio { namespace state_history { @@ -143,69 +140,11 @@ std::vector create_deltas(const chainbase::database& db, bool full_ return deltas; } -std::vector create_deltas_rocksdb(const chainbase::database& db, const eosio::chain::kv_undo_stack_ptr &kv_undo_stack, bool full_snapshot) { - std::vector deltas; - - if(full_snapshot) { - //process key_value section - rocksdb_receiver_whole_db kv_receiver(deltas, db); - chain::backing_store::rocksdb_contract_kv_table_writer kv_writer(kv_receiver); - - auto begin_key = eosio::session::shared_bytes(&chain::backing_store::rocksdb_contract_kv_prefix, 1); - auto end_key = begin_key.next(); - chain::backing_store::walk_rocksdb_entries_with_prefix(kv_undo_stack, begin_key, end_key, kv_writer); - - //process contract section - rocksdb_receiver_whole_db db_receiver(deltas, db); - using table_collector = chain::backing_store::rocksdb_whole_db_table_collector; - table_collector table_collector_receiver(db_receiver); - chain::backing_store::rocksdb_contract_db_table_writer writer(table_collector_receiver); - - begin_key = eosio::session::shared_bytes(&chain::backing_store::rocksdb_contract_db_prefix, 1); - end_key = begin_key.next(); - chain::backing_store::walk_rocksdb_entries_with_prefix(kv_undo_stack, begin_key, end_key, writer); - } else { - auto* session = std::visit(eosio::session::overloaded{ - [](eosio::chain::kv_undo_stack_ptr::element_type::session_type* session){ - return session; - }, [](auto*){ - EOS_ASSERT(false, eosio::chain::chain_exception, "undo_stack is empty"); - static eosio::chain::kv_undo_stack_ptr::element_type::session_type* invalid = nullptr; - return invalid; - }}, kv_undo_stack->top().holder()); - - rocksdb_receiver_single_entry receiver(deltas, db); - - for(auto &updated_key: session->updated_keys()) { - std::visit([&](auto* p) { - p->read(updated_key) ? receiver.set_delta_present(1) : receiver.set_delta_present(2); - }, session->parent()); - - chain::backing_store::process_rocksdb_entry(*session, updated_key, receiver); - } - - receiver.set_delta_present(0); - for(auto &deleted_key: session->deleted_keys()) { - std::visit([&](auto* p) { - chain::backing_store::process_rocksdb_entry(*p, deleted_key, receiver); - }, session->parent()); - } - } - - return deltas; -} - -std::vector create_deltas(const chain::combined_database& db, bool full_snapshot){ +std::vector create_deltas(const chain::kv_database& db, bool full_snapshot){ auto &chainbase_db = db.get_db(); - auto &kv_undo_stack = db.get_kv_undo_stack(); std::vector deltas = create_deltas(chainbase_db, full_snapshot); - if(kv_undo_stack && chainbase_db.get().backing_store == chain::backing_store_type::ROCKSDB) { - auto deltas_rocksdb = create_deltas_rocksdb(chainbase_db, kv_undo_stack, full_snapshot); - deltas.insert( deltas.end(), deltas_rocksdb.begin(), deltas_rocksdb.end() ); - } - return deltas; } diff --git a/libraries/state_history/include/eosio/state_history/create_deltas.hpp b/libraries/state_history/include/eosio/state_history/create_deltas.hpp index ff47874c4f..177703d3cd 100644 --- a/libraries/state_history/include/eosio/state_history/create_deltas.hpp +++ b/libraries/state_history/include/eosio/state_history/create_deltas.hpp @@ -1,12 +1,12 @@ #pragma once #include -#include +#include namespace eosio { namespace state_history { -std::vector create_deltas(const chain::combined_database& db, bool full_snapshot); +std::vector create_deltas(const chain::kv_database& db, bool full_snapshot); } // namespace state_history } // namespace eosio diff --git a/libraries/state_history/include/eosio/state_history/log.hpp b/libraries/state_history/include/eosio/state_history/log.hpp index ca071a833c..344bee7e29 100644 --- a/libraries/state_history/include/eosio/state_history/log.hpp +++ b/libraries/state_history/include/eosio/state_history/log.hpp @@ -6,7 +6,7 @@ #include #include -#include +#include #include #include #include @@ -241,7 +241,7 @@ class state_history_chain_state_log : public state_history_log { std::shared_ptr> get_log_entry(block_num_type block_num); - void store(const chain::combined_database& db, const chain::block_state_ptr& block_state); + void store(const chain::kv_database& db, const chain::block_state_ptr& block_state); private: void write_payload(cfile_stream& stream, const std::vector& data) override; }; diff --git a/libraries/state_history/include/eosio/state_history/rocksdb_receiver.hpp b/libraries/state_history/include/eosio/state_history/rocksdb_receiver.hpp deleted file mode 100644 index 9dc1503be8..0000000000 --- a/libraries/state_history/include/eosio/state_history/rocksdb_receiver.hpp +++ /dev/null @@ -1,186 +0,0 @@ -#pragma once - -#include -#include - -namespace eosio { -namespace state_history { - -/* - * Base class for receivers of rocksdb data for SHiP delta generation. - * -*/ -class rocksdb_receiver { -public: - // This method is not required for this use case, but the interface requires we implement it. - void add_row(fc::unsigned_int x) { - - } - -protected: - rocksdb_receiver(std::vector &deltas, const chainbase::database& db) : deltas_(deltas), db_(db) {}; - - std::optional table_; - uint8_t present_{2}; - - template - void add_delta_with_table(const char *name, const Object& row){ - deltas_[get_delta_index(name)].rows.obj.emplace_back(present_, fc::raw::pack(make_history_context_wrapper(db_, *table_, row))); - } - - template - void add_delta(const char *name, const Object& row){ - deltas_[get_delta_index(name)].rows.obj.emplace_back(present_, fc::raw::pack(make_history_serial_wrapper(db_, row))); - } - -private: - std::map name_to_index_; - - int get_delta_index(std::string name) { - if(!name_to_index_.count(name)) { - deltas_.push_back({}); - deltas_.back().name = name; - name_to_index_[name] = deltas_.size() - 1; - } - - return name_to_index_[name]; - } - - std::vector &deltas_; - const chainbase::database& db_; - -}; - -/* - * Specialization of rocksdb data's receiver for SHiP delta generation when we are getting the whole db. - * - * When we are receiving the whole current db content, then we store the deltas of the objects as we - * receive them. - */ -class rocksdb_receiver_whole_db : public rocksdb_receiver { -public: - using rocksdb_receiver::add_row; - - rocksdb_receiver_whole_db(std::vector &deltas, const chainbase::database& db) : rocksdb_receiver(deltas, db) {} - - void add_row(const chain::kv_object_view& row) { - add_delta("key_value", row); - } - - void add_row(const chain::backing_store::table_id_object_view& row) { - add_delta("contract_table", row); - table_ = row; - } - - void add_row(const chain::backing_store::primary_index_view& row) { - add_delta_with_table("contract_row", row); - } - - void add_row(const chain::backing_store::secondary_index_view& row) { - add_delta_with_table("contract_index64", row); - } - - void add_row(const chain::backing_store::secondary_index_view& row) { - add_delta_with_table("contract_index128", row); - } - - void add_row(const chain::backing_store::secondary_index_view& row) { - add_delta_with_table("contract_index256", row); - } - - void add_row(const chain::backing_store::secondary_index_view& row) { - add_delta_with_table("contract_index_double", row); - } - - void add_row(const chain::backing_store::secondary_index_view& row) { - add_delta_with_table("contract_index_long_double", row); - } -}; - - -/* - * Specialization of rocksdb data's receiver for SHiP delta generation when we are operating on single key entries (updated or deleted). - * - * This will be used when working with single key entries, because some of these objects (primary and secondary indexes) - * require information about the table they belong to (the context table), then for these objects we will receive 2 calls, - * the first one related to the table_id_object_view (but in this case this object does not contain proper values for - * payer so it can't be used to create its delta) associated with them andthe second call for the object itself. - * For that reason there is logic also associated to detect when we want to store the table_id_object_view itself - * (which is when we receive only 1 call for this and the next call is not any of the primary or secondary indexes). - * -*/ -class rocksdb_receiver_single_entry : public rocksdb_receiver { -public: - using rocksdb_receiver::add_row; - - rocksdb_receiver_single_entry(std::vector &deltas, const chainbase::database& db) : rocksdb_receiver(deltas, db) {} - - void add_row(const chain::kv_object_view& row) { - maybe_process_table(); - - add_delta("key_value", row); - } - - /* - * In this case we do not immediately report it as a delta until we see what comes next. If it is followed by a row - * that requires the table context (primary and secondary indexes) then the table_id_object_view was just being - * provided as the context. If it is followed by any other case, then it was indicating a change in that - * table_id_object_view, and it needs to be added to the delta. - */ - void add_row(const chain::backing_store::table_id_object_view& row) { - maybe_process_table(); - table_ = row; - } - - void add_row(const chain::backing_store::primary_index_view& row) { - add_delta_with_table("contract_row", row); - table_ = {}; - } - - void add_row(const chain::backing_store::secondary_index_view& row) { - add_delta_with_table("contract_index64", row); - table_ = {}; - } - - void add_row(const chain::backing_store::secondary_index_view& row) { - add_delta_with_table("contract_index128", row); - table_ = {}; - } - - void add_row(const chain::backing_store::secondary_index_view& row) { - add_delta_with_table("contract_index256", row); - table_ = {}; - } - - void add_row(const chain::backing_store::secondary_index_view& row) { - add_delta_with_table("contract_index_double", row); - table_ = {}; - } - - void add_row(const chain::backing_store::secondary_index_view& row) { - add_delta_with_table("contract_index_long_double", row); - table_ = {}; - } - - void set_delta_present(uint8_t value) { - maybe_process_table(); - - present_ = value; - } - - ~rocksdb_receiver_single_entry() { - maybe_process_table(); - } - -private: - - void maybe_process_table() { - if(table_) { - add_delta("contract_table", *table_); - table_ = {}; - } - } -}; - -} -} diff --git a/libraries/state_history/include/eosio/state_history/serialization.hpp b/libraries/state_history/include/eosio/state_history/serialization.hpp index f652242259..ec067cfef5 100644 --- a/libraries/state_history/include/eosio/state_history/serialization.hpp +++ b/libraries/state_history/include/eosio/state_history/serialization.hpp @@ -13,7 +13,6 @@ #include #include #include -#include #include #include @@ -158,11 +157,6 @@ ST& operator<<(ST& ds, const history_serial_wrapper -ST& operator<<(ST& ds, const history_serial_wrapper& obj) { - return serialize_table_id_object(ds, obj); -} - template ST& operator<<(ST& ds, const history_context_wrapper& obj) { @@ -176,19 +170,6 @@ ST& operator<<(ST& return ds; } -template -ST& operator<<(ST& ds, - const history_context_wrapper& obj) { - fc::raw::pack(ds, fc::unsigned_int(0)); - fc::raw::pack(ds, as_type(obj.context.code.to_uint64_t())); - fc::raw::pack(ds, as_type(obj.context.scope.to_uint64_t())); - fc::raw::pack(ds, as_type(obj.context.table.to_uint64_t())); - fc::raw::pack(ds, as_type(obj.obj.primary_key)); - fc::raw::pack(ds, as_type(obj.obj.payer.to_uint64_t())); - fc::raw::pack(ds, obj.obj.value); - return ds; -} - template void serialize_secondary_index_data(ST& ds, const T& obj) { fc::raw::pack(ds, obj); @@ -237,60 +218,30 @@ ST& operator<<(ST& return serialize_secondary_index(ds, obj.context, obj.obj); } -template -ST& operator<<(ST& ds, - const history_context_wrapper>& obj) { - return serialize_secondary_index(ds, obj.context, obj.obj); -} - template ST& operator<<(ST& ds, const history_context_wrapper& obj) { return serialize_secondary_index(ds, obj.context, obj.obj); } -template -ST& operator<<(ST& ds, - const history_context_wrapper>& obj) { - return serialize_secondary_index(ds, obj.context, obj.obj); -} - template ST& operator<<(ST& ds, const history_context_wrapper& obj) { return serialize_secondary_index(ds, obj.context, obj.obj); } -template -ST& operator<<(ST& ds, - const history_context_wrapper>& obj) { - return serialize_secondary_index(ds, obj.context, obj.obj); -} - template ST& operator<<(ST& ds, const history_context_wrapper& obj) { return serialize_secondary_index(ds, obj.context, obj.obj); } -template -ST& operator<<(ST& ds, - const history_context_wrapper>& obj) { - return serialize_secondary_index(ds, obj.context, obj.obj); -} - template ST& operator<<( ST& ds, const history_context_wrapper& obj) { return serialize_secondary_index(ds, obj.context, obj.obj); } -template -ST& operator<<(ST& ds, - const history_context_wrapper>& obj) { - return serialize_secondary_index(ds, obj.context, obj.obj); -} - template ST& operator<<(ST& ds, const history_serial_wrapper& obj) { fc::raw::pack(ds, fc::unsigned_int(0)); diff --git a/libraries/state_history/log.cpp b/libraries/state_history/log.cpp index 0c97998db9..3b4a4d3ed0 100644 --- a/libraries/state_history/log.cpp +++ b/libraries/state_history/log.cpp @@ -504,7 +504,7 @@ std::shared_ptr> state_history_chain_state_log::get_log_entry( return std::make_shared>(); } -void state_history_chain_state_log::store(const chain::combined_database& db, +void state_history_chain_state_log::store(const chain::kv_database& db, const chain::block_state_ptr& block_state) { auto [begin, end] = begin_end_block_nums(); bool fresh = begin == end; diff --git a/libraries/testing/include/eosio/testing/backing_store_tester_macros.hpp b/libraries/testing/include/eosio/testing/backing_store_tester_macros.hpp index 1abae1e807..6eb0856e05 100644 --- a/libraries/testing/include/eosio/testing/backing_store_tester_macros.hpp +++ b/libraries/testing/include/eosio/testing/backing_store_tester_macros.hpp @@ -1,7 +1,5 @@ #ifdef NON_VALIDATING_TEST #define TESTER tester -#define ROCKSDB_TESTER rocksdb_tester #else #define TESTER validating_tester -#define ROCKSDB_TESTER rocksdb_validating_tester #endif diff --git a/libraries/testing/include/eosio/testing/tester.hpp b/libraries/testing/include/eosio/testing/tester.hpp index c0e0a8c4a1..800f3b59fb 100644 --- a/libraries/testing/include/eosio/testing/tester.hpp +++ b/libraries/testing/include/eosio/testing/tester.hpp @@ -646,24 +646,6 @@ namespace eosio { namespace testing { bool skip_validate = false; }; - class rocksdb_tester : public tester { - public: - rocksdb_tester(setup_policy policy = setup_policy::full, db_read_mode read_mode = db_read_mode::SPECULATIVE, - std::optional genesis_max_inline_action_size = std::optional{}, - std::optional config_max_nonprivileged_inline_action_size = std::optional{}) { - init(policy, read_mode, genesis_max_inline_action_size, config_max_nonprivileged_inline_action_size, - backing_store_type::ROCKSDB); - } - }; - - class rocksdb_validating_tester : public validating_tester { - public: - virtual ~rocksdb_validating_tester() {} - - rocksdb_validating_tester(const flat_set& trusted_producers = flat_set()) - : validating_tester(trusted_producers, {backing_store_type::ROCKSDB}){} - }; - /** * Utility predicate to check whether an fc::exception code is equivalent to a given value */ diff --git a/libraries/testing/tester.cpp b/libraries/testing/tester.cpp index 3388f20117..c838fcf32f 100644 --- a/libraries/testing/tester.cpp +++ b/libraries/testing/tester.cpp @@ -1220,7 +1220,6 @@ namespace eosio { namespace testing { std::optional config_backing_store) { vcfg = config_state.first; if (config_backing_store) { - // can only have one instance of RocksDB running in the test process vcfg.backing_store = alternate_type(*config_backing_store); } config_validator(vcfg); @@ -1237,7 +1236,6 @@ namespace eosio { namespace testing { auto def_conf = default_config(tempdir, std::optional{}, std::optional{}, config_backing_store); vcfg = def_conf.first; if (config_backing_store) { - // can only have one instance of RocksDB running in the test process vcfg.backing_store = alternate_type(*config_backing_store); } config_validator(vcfg); @@ -1252,7 +1250,7 @@ namespace eosio { namespace testing { } backing_store_type validating_tester::alternate_type(backing_store_type type) { - return type == backing_store_type::CHAINBASE ? backing_store_type::ROCKSDB : backing_store_type::CHAINBASE; + return backing_store_type::CHAINBASE; } bool fc_exception_code_is::operator()( const fc::exception& ex ) { diff --git a/plugins/chain_plugin/chain_plugin.cpp b/plugins/chain_plugin/chain_plugin.cpp index 36fa3a696b..a4c80021d9 100644 --- a/plugins/chain_plugin/chain_plugin.cpp +++ b/plugins/chain_plugin/chain_plugin.cpp @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include #include @@ -127,8 +127,6 @@ void validate(boost::any& v, std::ostream& operator<<(std::ostream& osm, eosio::chain::backing_store_type b) { if ( b == eosio::chain::backing_store_type::CHAINBASE ) { osm << "chainbase"; - } else if ( b == eosio::chain::backing_store_type::ROCKSDB ) { - osm << "rocksdb"; } return osm; @@ -150,8 +148,6 @@ void validate(boost::any& v, if ( s == "chainbase" ) { v = boost::any(eosio::chain::backing_store_type::CHAINBASE); - } else if ( s == "rocksdb" ) { - v = boost::any(eosio::chain::backing_store_type::ROCKSDB); } else { throw validation_error(validation_error::invalid_option_value); } @@ -305,19 +301,6 @@ void chain_plugin::set_program_options(options_description& cli, options_descrip "Override default maximum ABI serialization time allowed in ms") ("chain-state-db-size-mb", bpo::value()->default_value(config::default_state_size / (1024 * 1024)), "Maximum size (in MiB) of the chain state database") ("chain-state-db-guard-size-mb", bpo::value()->default_value(config::default_state_guard_size / (1024 * 1024)), "Safely shut down node when free space remaining in the chain state database drops below this size (in MiB).") - ("backing-store", boost::program_options::value()->default_value(eosio::chain::backing_store_type::CHAINBASE), - "The storage for state, chainbase or rocksdb") - ("persistent-storage-num-threads", bpo::value()->default_value(default_persistent_storage_num_threads), - "Number of rocksdb threads for flush and compaction") - ("persistent-storage-max-num-files", bpo::value()->default_value(config::default_persistent_storage_max_num_files), - "Max number of rocksdb files to keep open. -1 = unlimited.") - ("persistent-storage-write-buffer-size-mb", bpo::value()->default_value(config::default_persistent_storage_write_buffer_size / (1024 * 1024)), - "Size of a single rocksdb memtable (in MiB)") - ("persistent-storage-bytes-per-sync", bpo::value()->default_value(config::default_persistent_storage_bytes_per_sync), - "Rocksdb write rate of flushes and compactions.") - ("persistent-storage-mbytes-snapshot-batch", bpo::value()->default_value(config::default_persistent_storage_mbytes_batch), - "Rocksdb batch size threshold before writing read in snapshot data to database.") - ("reversible-blocks-db-size-mb", bpo::value()->default_value(0), "(DEPRECATED: no longer used) Maximum size (in MiB) of the reversible blocks database") ("reversible-blocks-db-guard-size-mb", bpo::value()->default_value(0), @@ -857,36 +840,6 @@ void chain_plugin::plugin_initialize(const variables_map& options) { if( options.count( "chain-state-db-guard-size-mb" )) my->chain_config->state_guard_size = options.at( "chain-state-db-guard-size-mb" ).as() * 1024 * 1024; - my->chain_config->backing_store = options.at( "backing-store" ).as(); - - if( options.count( "persistent-storage-num-threads" )) { - my->chain_config->persistent_storage_num_threads = options.at( "persistent-storage-num-threads" ).as(); - EOS_ASSERT( my->chain_config->persistent_storage_num_threads > 0, plugin_config_exception, - "persistent-storage-num-threads ${num} must be greater than 0", ("num", my->chain_config->persistent_storage_num_threads) ); - } - - if( options.count( "persistent-storage-max-num-files" )) { - my->chain_config->persistent_storage_max_num_files = options.at( "persistent-storage-max-num-files" ).as(); - EOS_ASSERT( my->chain_config->persistent_storage_max_num_files == -1 || my->chain_config->persistent_storage_max_num_files > 0, plugin_config_exception, - "persistent-storage-max-num-files ${num} must be equal to -1 or be greater than 0", ("num", my->chain_config->persistent_storage_max_num_files) ); - } - - if( options.count( "persistent-storage-write-buffer-size-mb" )) { - my->chain_config->persistent_storage_write_buffer_size = options.at( "persistent-storage-write-buffer-size-mb" ).as() * 1024 * 1024; - EOS_ASSERT( my->chain_config->persistent_storage_write_buffer_size > 0, plugin_config_exception, - "persistent-storage-write-buffer-size-mb ${num} must be greater than 0", ("num", my->chain_config->persistent_storage_write_buffer_size) ); - } - - if( options.count( "persistent-storage-bytes-per-sync" )) { - my->chain_config->persistent_storage_bytes_per_sync = options.at( "persistent-storage-bytes-per-sync" ).as(); - EOS_ASSERT( my->chain_config->persistent_storage_bytes_per_sync > 0, plugin_config_exception, - "persistent-storage-bytes-per-sync ${num} must be greater than 0", ("num", my->chain_config->persistent_storage_bytes_per_sync) ); - } - - my->chain_config->persistent_storage_mbytes_batch = options.at( "persistent-storage-mbytes-snapshot-batch" ).as(); - EOS_ASSERT( my->chain_config->persistent_storage_mbytes_batch > 0, plugin_config_exception, - "persistent-storage-mbytes-snapshot-batch ${num} must be greater than 0", ("num", my->chain_config->persistent_storage_mbytes_batch) ); - if( options.count( "reversible-blocks-db-size-mb" )) wlog( "reversible-blocks-db-size-mb deprecated and will be removed in future version" ); @@ -978,7 +931,7 @@ void chain_plugin::plugin_initialize(const variables_map& options) { ilog( "Replay requested: deleting state database" ); if( options.at( "truncate-at-block" ).as() > 0 ) wlog( "The --truncate-at-block option does not work for a regular replay of the blockchain." ); - eosio::chain::combined_database::destroy( my->chain_config->state_dir ); + eosio::chain::kv_database::destroy( my->chain_config->state_dir ); } else if( options.at( "truncate-at-block" ).as() > 0 ) { wlog( "The --truncate-at-block option can only be used with --hard-replay-blockchain." ); } @@ -2255,38 +2208,6 @@ read_only::get_table_rows_result read_only::get_kv_table_rows(const read_only::g return kv_get_rows(kv_reverse_range(context, upper_bound, lower_bound)); } -struct table_receiver - : chain::backing_store::table_only_error_receiver { - table_receiver(read_only::get_table_by_scope_result& result, const read_only::get_table_by_scope_params& params) - : result_(result), params_(params) { - check_limit(); - } - - void add_table_row(const backing_store::table_id_object_view& row) { - if( params_.table && row.table != params_.table ) { - return; - } - result_.rows.push_back( {row.code, row.scope, row.table, row.payer, row.count} ); - check_limit(); - } - - auto keep_processing_entries() { - keep_processing kp {}; - return [kp{std::move(kp)},&reached_limit=reached_limit_]() { - return !reached_limit && kp(); - }; - }; - - void check_limit() { - if (result_.rows.size() >= params_.limit) - reached_limit_ = true; - } - - read_only::get_table_by_scope_result& result_; - const read_only::get_table_by_scope_params& params_; - bool reached_limit_ = false; -}; - read_only::get_table_by_scope_result read_only::get_table_by_scope( const read_only::get_table_by_scope_params& p )const { read_only::get_table_by_scope_result result; auto lower_bound_lookup_tuple = std::make_tuple( p.code, name(std::numeric_limits::lowest()), p.table ); @@ -2307,59 +2228,28 @@ read_only::get_table_by_scope_result read_only::get_table_by_scope( const read_o return result; const bool reverse = p.reverse && *p.reverse; - const auto db_backing_store = get_backing_store(); - if (db_backing_store == eosio::chain::backing_store_type::CHAINBASE) { - auto walk_table_range = [&result,&p]( auto itr, auto end_itr ) { - keep_processing kp; - for( unsigned int count = 0; kp() && count < p.limit && itr != end_itr; ++itr ) { - if( p.table && itr->table != p.table ) continue; - - result.rows.push_back( {itr->code, itr->scope, itr->table, itr->payer, itr->count} ); + auto walk_table_range = [&result,&p]( auto itr, auto end_itr ) { + keep_processing kp; + for( unsigned int count = 0; kp() && count < p.limit && itr != end_itr; ++itr ) { + if( p.table && itr->table != p.table ) continue; - ++count; - } - if( itr != end_itr ) { - result.more = itr->scope.to_string(); - } - }; + result.rows.push_back( {itr->code, itr->scope, itr->table, itr->payer, itr->count} ); - const auto& d = db.db(); - const auto& idx = d.get_index(); - auto lower = idx.lower_bound( lower_bound_lookup_tuple ); - auto upper = idx.upper_bound( upper_bound_lookup_tuple ); - if( reverse ) { - walk_table_range( boost::make_reverse_iterator(upper), boost::make_reverse_iterator(lower) ); - } else { - walk_table_range( lower, upper ); + ++count; } - } - else { - using namespace eosio::chain; - EOS_ASSERT(db_backing_store == backing_store_type::ROCKSDB, - chain::contract_table_query_exception, - "Support for configured backing_store has not been added to get_table_by_scope"); - const auto& kv_database = db.kv_db(); - table_receiver receiver(result, p); - auto kp = receiver.keep_processing_entries(); - auto lower = chain::backing_store::db_key_value_format::create_full_prefix_key(std::get<0>(lower_bound_lookup_tuple), - std::get<1>(lower_bound_lookup_tuple), - std::get<2>(lower_bound_lookup_tuple)); - auto upper = chain::backing_store::db_key_value_format::create_full_prefix_key(std::get<0>(upper_bound_lookup_tuple), - std::get<1>(upper_bound_lookup_tuple), - std::get<2>(upper_bound_lookup_tuple)); - if (reverse) { - lower = eosio::session::shared_bytes::truncate_key(lower); - } - // since upper is either the upper_bound of a forward search, or the reverse iterator <= for the beginning of the end of - // the table, we need to move it to just before the beginning of the next table - upper = upper.next(); - const auto context = (reverse) ? backing_store::key_context::table_only_reverse : backing_store::key_context::table_only; - backing_store::rocksdb_contract_db_table_writer> writer(receiver, context, kp); - eosio::chain::backing_store::walk_rocksdb_entries_with_prefix(kv_database.get_kv_undo_stack(), lower, upper, writer); - const auto stopped_at = writer.stopped_at(); - if (stopped_at) { - result.more = stopped_at->scope.to_string(); + if( itr != end_itr ) { + result.more = itr->scope.to_string(); } + }; + + const auto& d = db.db(); + const auto& idx = d.get_index(); + auto lower = idx.lower_bound( lower_bound_lookup_tuple ); + auto upper = idx.upper_bound( upper_bound_lookup_tuple ); + if( reverse ) { + walk_table_range( boost::make_reverse_iterator(upper), boost::make_reverse_iterator(lower) ); + } else { + walk_table_range( lower, upper ); } return result; @@ -2448,102 +2338,32 @@ read_only::get_producers_result read_only::get_producers( const read_only::get_p static const uint8_t secondary_index_num = 0; const name sec_producers_table {producers_table.to_uint64_t() | secondary_index_num}; - const auto db_backing_store = get_backing_store(); - if (db_backing_store == eosio::chain::backing_store_type::CHAINBASE) { - const auto* const table_id = d.find( - boost::make_tuple(code, scope, producers_table)); - const auto* const secondary_table_id = d.find( - boost::make_tuple(code, scope, sec_producers_table)); - EOS_ASSERT(table_id && secondary_table_id, chain::contract_table_query_exception, "Missing producers table"); - - const auto& kv_index = d.get_index(); - const auto& secondary_index = d.get_index().indices(); - const auto& secondary_index_by_primary = secondary_index.get(); - const auto& secondary_index_by_secondary = secondary_index.get(); - - vector data; - - auto it = lower.to_uint64_t() == 0 - ? secondary_index_by_secondary.lower_bound( - boost::make_tuple(secondary_table_id->id, to_softfloat64(std::numeric_limits::lowest()), 0)) - : secondary_index.project( - secondary_index_by_primary.lower_bound( - boost::make_tuple(secondary_table_id->id, lower.to_uint64_t()))); - for( ; it != secondary_index_by_secondary.end() && it->t_id == secondary_table_id->id; ++it ) { - if (done(*it)) { - break; - } - auto itr = kv_index.find(boost::make_tuple(table_id->id, it->primary_key)); - add_val(*itr); + const auto* const table_id = d.find( + boost::make_tuple(code, scope, producers_table)); + const auto* const secondary_table_id = d.find( + boost::make_tuple(code, scope, sec_producers_table)); + EOS_ASSERT(table_id && secondary_table_id, chain::contract_table_query_exception, "Missing producers table"); + + const auto& kv_index = d.get_index(); + const auto& secondary_index = d.get_index().indices(); + const auto& secondary_index_by_primary = secondary_index.get(); + const auto& secondary_index_by_secondary = secondary_index.get(); + + vector data; + + auto it = lower.to_uint64_t() == 0 + ? secondary_index_by_secondary.lower_bound( + boost::make_tuple(secondary_table_id->id, to_softfloat64(std::numeric_limits::lowest()), 0)) + : secondary_index.project( + secondary_index_by_primary.lower_bound( + boost::make_tuple(secondary_table_id->id, lower.to_uint64_t()))); + for( ; it != secondary_index_by_secondary.end() && it->t_id == secondary_table_id->id; ++it ) { + if (done(*it)) { + break; } + auto itr = kv_index.find(boost::make_tuple(table_id->id, it->primary_key)); + add_val(*itr); } - else { - using namespace eosio::chain; - EOS_ASSERT(db_backing_store == backing_store_type::ROCKSDB, - contract_table_query_exception, - "Support for configured backing_store has not been added to get_table_by_scope"); - const auto& kv_database = db.kv_db(); - using key_type = backing_store::db_key_value_format::key_type; - - // derive the "root" of the float64 secondary key space to determine where it ends (upper), then may need to recalculate - auto lower_key = backing_store::db_key_value_format::create_full_prefix_key(code, scope, sec_producers_table, key_type::sec_double); - auto upper_key = lower_key.next(); - if (lower.to_uint64_t() == 0) { - lower_key = backing_store::db_key_value_format::create_full_prefix_secondary_key(code, scope, sec_producers_table, float64_t{lower.to_uint64_t()}); - } - - auto session = kv_database.get_kv_undo_stack()->top(); - auto retrieve_primary_key = [&code,&scope,&producers_table,&session](uint64_t primary_key) { - auto full_primary_key = backing_store::db_key_value_format::create_full_primary_key(code, scope, producers_table, primary_key); - auto value = session.read(full_primary_key); - return *value; - }; - - using done_func = decltype(done); - using add_val_func = decltype(add_val); - using retrieve_prim_func = decltype(retrieve_primary_key); - - struct f64_secondary_key_receiver - : backing_store::single_type_error_receiver, - contract_table_query_exception> { - f64_secondary_key_receiver(read_only::get_producers_result& result, done_func&& done, - add_val_func&& add_val, retrieve_prim_func&& retrieve_prim) - : result_(result), done_(done), add_val_(add_val), retrieve_primary_key_(retrieve_prim) {} - - void add_only_row(const backing_store::secondary_index_view& row) { - // needs to allow a second pass after limit is reached or time has passed, to allow "more" processing - if (done_(row)) { - finished_ = true; - } - else { - auto value = retrieve_primary_key_(row.primary_key); - add_val_(backing_store::primary_index_view::create(row.primary_key, value.data(), value.size())); - } - } - - void add_table_row(const backing_store::table_id_object_view& ) { - // used for only one table, so we already know the context of the table - } - - auto keep_processing_entries() { - return [&finished=finished_]() { - return !finished; - }; - }; - - read_only::get_producers_result& result_; - done_func done_; - add_val_func add_val_; - retrieve_prim_func retrieve_primary_key_; - bool finished_ = false; - }; - f64_secondary_key_receiver receiver(result, std::move(done), std::move(add_val), std::move(retrieve_primary_key)); - auto kpe = receiver.keep_processing_entries(); - backing_store::rocksdb_contract_db_table_writer> - writer(receiver, backing_store::key_context::standalone, kpe); - eosio::chain::backing_store::walk_rocksdb_entries_with_prefix(kv_database.get_kv_undo_stack(), lower_key, upper_key, writer); - }; constexpr name global = "global"_n; const auto global_table_type = get_table_type(abi, global); @@ -3304,11 +3124,6 @@ fc::variant read_only::get_primary_key(name code, name scope, name table, uint64 return val; } -eosio::chain::backing_store_type read_only::get_backing_store() const { - const auto& kv_database = db.kv_db(); - return kv_database.get_backing_store(); -} - read_only::get_all_accounts_result read_only::get_all_accounts( const get_all_accounts_params& params ) const { diff --git a/plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp b/plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp index d1da878b01..4facfb009e 100644 --- a/plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp +++ b/plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp @@ -14,8 +14,7 @@ #include #include #include -#include -#include +#include #include #include @@ -462,62 +461,30 @@ class read_only { get_scheduled_transactions_result get_scheduled_transactions( const get_scheduled_transactions_params& params ) const; - eosio::chain::backing_store_type get_backing_store() const; - enum class row_requirements { required, optional }; template bool get_primary_key_internal(name code, name scope, name table, uint64_t primary_key, row_requirements require_table, row_requirements require_primary, Function&& f) const { - const auto db_backing_store = get_backing_store(); - if (db_backing_store == eosio::chain::backing_store_type::CHAINBASE) { - const auto* const table_id = - db.db().find(boost::make_tuple(code, scope, table)); - if (require_table == row_requirements::optional && !table_id) { - return false; - } - EOS_ASSERT(table_id, chain::contract_table_query_exception, - "Missing code: ${code}, scope: ${scope}, table: ${table}", - ("code",code.to_string())("scope",scope.to_string())("table",table.to_string())); - const auto& kv_index = db.db().get_index(); - const auto it = kv_index.find(boost::make_tuple(table_id->id, primary_key)); - if (require_primary == row_requirements::optional && it == kv_index.end()) { - return false; - } - EOS_ASSERT(it != kv_index.end(), chain::contract_table_query_exception, - "Missing row for primary_key: ${primary} in code: ${code}, scope: ${scope}, table: ${table}", - ("primary", primary_key)("code",code.to_string())("scope",scope.to_string()) - ("table",table.to_string())); - f(*it); - return true; + + const auto* const table_id = + db.db().find(boost::make_tuple(code, scope, table)); + if (require_table == row_requirements::optional && !table_id) { + return false; } - else { - using namespace eosio::chain; - EOS_ASSERT(db_backing_store == backing_store_type::ROCKSDB, - chain::contract_table_query_exception, - "Support for configured backing_store has not been added to get_primary_key"); - const auto& kv_database = db.kv_db(); - const auto full_key = chain::backing_store::db_key_value_format::create_full_primary_key(code, scope, table, primary_key); - auto current_session = kv_database.get_kv_undo_stack()->top(); - const auto value = current_session.read(full_key); - // check if we didn't actually find the key - if (!value) { - // only need to bother to do table search if we require it, so that we can report the correct error - if (require_table == row_requirements::required) { - const auto whole_table_prefix(backing_store::db_key_value_format::create_full_key_prefix(full_key, backing_store::db_key_value_format::end_of_prefix::pre_type)); - const auto value = current_session.read(whole_table_prefix); - EOS_ASSERT(value, chain::contract_table_query_exception, - "Missing code: ${code}, scope: ${scope}, table: ${table}", - ("code",code.to_string())("scope",scope.to_string())("table",table.to_string())); - } - EOS_ASSERT(require_primary == row_requirements::optional, chain::contract_table_query_exception, - "Missing row for primary_key: ${primary} in code: ${code}, scope: ${scope}, table: ${table}", - ("primary", primary_key)("code",code.to_string())("scope",scope.to_string()) - ("table",table.to_string())); - return false; - } - f(chain::backing_store::primary_index_view::create(primary_key, value->data(), value->size())); - return true; + EOS_ASSERT(table_id, chain::contract_table_query_exception, + "Missing code: ${code}, scope: ${scope}, table: ${table}", + ("code",code.to_string())("scope",scope.to_string())("table",table.to_string())); + const auto& kv_index = db.db().get_index(); + const auto it = kv_index.find(boost::make_tuple(table_id->id, primary_key)); + if (require_primary == row_requirements::optional && it == kv_index.end()) { + return false; } + EOS_ASSERT(it != kv_index.end(), chain::contract_table_query_exception, + "Missing row for primary_key: ${primary} in code: ${code}, scope: ${scope}, table: ${table}", + ("primary", primary_key)("code",code.to_string())("scope",scope.to_string()) + ("table",table.to_string())); + f(*it); + return true; } template @@ -581,107 +548,27 @@ class read_only { memcpy( data.data(), obj.value.data(), obj.value.size() ); } - template - struct primary_key_receiver - : chain::backing_store::single_type_error_receiver, chain::backing_store::primary_index_view, chain::contract_table_query_exception> { - primary_key_receiver(Func f) : f_(f) {} - - void add_only_row(const chain::backing_store::primary_index_view& row) { - if(!f_(row)) - done_ = true; - } - - void add_table_row(const chain::backing_store::table_id_object_view& ) { - // used for only one table, so we already know the context of the table - } - - auto keep_processing_entries() { - return [&done=done_]() { - return !done; - }; - }; - - Func f_; - bool done_ = false; - }; - template void walk_key_value_table(const name& code, const name& scope, const name& table, Function f) const { - const auto db_backing_store = get_backing_store(); - if (db_backing_store == eosio::chain::backing_store_type::CHAINBASE) { - const auto& d = db.db(); - const auto* t_id = d.find(boost::make_tuple(code, scope, table)); - if (t_id != nullptr) { - const auto &idx = d.get_index(); - decltype(t_id->id) next_tid(t_id->id._id + 1); - auto lower = idx.lower_bound(boost::make_tuple(t_id->id)); - auto upper = idx.lower_bound(boost::make_tuple(next_tid)); - - for (auto itr = lower; itr != upper; ++itr) { - if (!f(*itr)) { - break; - } + const auto& d = db.db(); + const auto* t_id = d.find(boost::make_tuple(code, scope, table)); + if (t_id != nullptr) { + const auto &idx = d.get_index(); + decltype(t_id->id) next_tid(t_id->id._id + 1); + auto lower = idx.lower_bound(boost::make_tuple(t_id->id)); + auto upper = idx.lower_bound(boost::make_tuple(next_tid)); + + for (auto itr = lower; itr != upper; ++itr) { + if (!f(*itr)) { + break; } } } - else { - using namespace eosio::chain; - EOS_ASSERT(db_backing_store == backing_store_type::ROCKSDB, - chain::contract_table_query_exception, - "Support for configured backing_store has not been added to get_primary_key"); - primary_key_receiver receiver(f); - auto kp = receiver.keep_processing_entries(); - backing_store::rocksdb_contract_db_table_writer, std::decay_t < decltype(kp)>> writer(receiver, backing_store::key_context::standalone, kp); - const auto& kv_database = db.kv_db(); - using key_type = chain::backing_store::db_key_value_format::key_type; - auto start = chain::backing_store::db_key_value_format::create_full_prefix_key(code, scope, table, key_type::primary); - auto end = start.next(); - eosio::chain::backing_store::walk_rocksdb_entries_with_prefix(kv_database.get_kv_undo_stack(), start, end, writer); - } } static uint64_t get_table_index_name(const read_only::get_table_rows_params& p, bool& primary); - - template - struct secondary_key_receiver - : chain::backing_store::single_type_error_receiver, chain::backing_store::secondary_index_view, chain::contract_table_query_exception> { - secondary_key_receiver(read_only::get_table_rows_result& result, Function f, const read_only::get_table_rows_params& params) - : result_(result), f_(f), params_(params) {} - - void add_only_row(const chain::backing_store::secondary_index_view& row) { - // needs to allow a second pass after limit is reached or time has passed, to allow "more" processing - if (reached_limit_ || !kp_()) { - result_.more = true; - result_.next_key = convert_to_string(row.secondary_key, params_.key_type, params_.encode_type, "next_key - next lower bound"); - done_ = true; - } - else { - f_(row, result_.rows); - reached_limit_ |= result_.rows.size() >= params_.limit; - } - } - - void add_table_row(const chain::backing_store::table_id_object_view& ) { - // used for only one table, so we already know the context of the table - } - - auto keep_processing_entries() { - return [&done=done_]() { - return !done; - }; - }; - - read_only::get_table_rows_result& result_; - Function f_; - const read_only::get_table_rows_params& params_; - bool reached_limit_ = false; - bool done_ = false; - keep_processing kp_; - }; - - template read_only::get_table_rows_result get_table_rows_by_seckey( const read_only::get_table_rows_params& p, const abi_def& abi, ConvFn conv )const { read_only::get_table_rows_result result; @@ -730,75 +617,43 @@ class read_only { return result; const bool reverse = p.reverse && *p.reverse; - const auto db_backing_store = get_backing_store(); auto get_prim_key_val = get_primary_key_value(p.table, abis, p.json, p.show_payer); - if (db_backing_store == eosio::chain::backing_store_type::CHAINBASE) { - const auto* t_id = d.find(boost::make_tuple(p.code, scope, p.table)); - const auto* index_t_id = d.find(boost::make_tuple(p.code, scope, name(table_with_index))); - if( t_id != nullptr && index_t_id != nullptr ) { - - const auto& secidx = d.get_index(); - auto lower_bound_lookup_tuple = std::make_tuple( index_t_id->id._id, - secondary_key_lower, - primary_key_lower ); - auto upper_bound_lookup_tuple = std::make_tuple( index_t_id->id._id, - secondary_key_upper, - primary_key_upper ); - - auto walk_table_row_range = [&]( auto itr, auto end_itr ) { - keep_processing kp; - vector data; - for( unsigned int count = 0; kp() && count < p.limit && itr != end_itr; ++itr ) { - const auto* itr2 = d.find( boost::make_tuple(t_id->id, itr->primary_key) ); - if( itr2 == nullptr ) continue; - - result.rows.emplace_back( get_prim_key_val(*itr2) ); - - ++count; - } - if( itr != end_itr ) { - result.more = true; - result.next_key = convert_to_string(itr->secondary_key, p.key_type, p.encode_type, "next_key - next lower bound"); - } - }; - - auto lower = secidx.lower_bound( lower_bound_lookup_tuple ); - auto upper = secidx.upper_bound( upper_bound_lookup_tuple ); - if( reverse ) { - walk_table_row_range( boost::make_reverse_iterator(upper), boost::make_reverse_iterator(lower) ); - } else { - walk_table_row_range( lower, upper ); + const auto* t_id = d.find(boost::make_tuple(p.code, scope, p.table)); + const auto* index_t_id = d.find(boost::make_tuple(p.code, scope, name(table_with_index))); + if( t_id != nullptr && index_t_id != nullptr ) { + + const auto& secidx = d.get_index(); + auto lower_bound_lookup_tuple = std::make_tuple( index_t_id->id._id, + secondary_key_lower, + primary_key_lower ); + auto upper_bound_lookup_tuple = std::make_tuple( index_t_id->id._id, + secondary_key_upper, + primary_key_upper ); + + auto walk_table_row_range = [&]( auto itr, auto end_itr ) { + keep_processing kp; + vector data; + for( unsigned int count = 0; kp() && count < p.limit && itr != end_itr; ++itr ) { + const auto* itr2 = d.find( boost::make_tuple(t_id->id, itr->primary_key) ); + if( itr2 == nullptr ) continue; + + result.rows.emplace_back( get_prim_key_val(*itr2) ); + + ++count; + } + if( itr != end_itr ) { + result.more = true; + result.next_key = convert_to_string(itr->secondary_key, p.key_type, p.encode_type, "next_key - next lower bound"); } - } - } - else { - using namespace eosio::chain; - EOS_ASSERT(db_backing_store == backing_store_type::ROCKSDB, - chain::contract_table_query_exception, - "Support for configured backing_store has not been added to get_primary_key"); - const auto context = (reverse) ? backing_store::key_context::standalone_reverse : backing_store::key_context::standalone; - auto lower = chain::backing_store::db_key_value_format::create_full_prefix_secondary_key(p.code, scope, name(table_with_index), secondary_key_lower); - auto upper = chain::backing_store::db_key_value_format::create_full_prefix_secondary_key(p.code, scope, name(table_with_index), secondary_key_upper); - if (reverse) { - lower = eosio::session::shared_bytes::truncate_key(lower); - } - // since upper is either the upper_bound of a forward search, or the reverse iterator <= for the beginning of the end of - // this secondary type, we need to move it to just before the beginning of the next type - upper = upper.next(); - const auto& kv_database = db.kv_db(); - auto session = kv_database.get_kv_undo_stack()->top(); - auto get_primary = [code=p.code,scope,table=p.table,&session,&get_prim_key_val](const chain::backing_store::secondary_index_view& row, vector& rows) { - auto full_key = chain::backing_store::db_key_value_format::create_full_primary_key(code, scope, table, row.primary_key); - auto value = session.read(full_key); - if( !value ) return; - - rows.emplace_back(get_prim_key_val(chain::backing_store::primary_index_view::create(row.primary_key, value->data(), value->size()))); }; - using secondary_receiver = secondary_key_receiver; - secondary_receiver receiver(result, get_primary, p); - auto kp = receiver.keep_processing_entries(); - backing_store::rocksdb_contract_db_table_writer> writer(receiver, context, kp); - eosio::chain::backing_store::walk_rocksdb_entries_with_prefix(kv_database.get_kv_undo_stack(), lower, upper, writer); + + auto lower = secidx.lower_bound( lower_bound_lookup_tuple ); + auto upper = secidx.upper_bound( upper_bound_lookup_tuple ); + if( reverse ) { + walk_table_row_range( boost::make_reverse_iterator(upper), boost::make_reverse_iterator(lower) ); + } else { + walk_table_row_range( lower, upper ); + } } return result; @@ -847,67 +702,31 @@ class read_only { }; const bool reverse = p.reverse && *p.reverse; - const auto db_backing_store = get_backing_store(); - if (db_backing_store == eosio::chain::backing_store_type::CHAINBASE) { - const auto* t_id = d.find(boost::make_tuple(p.code, scope, p.table)); - if( t_id != nullptr ) { - const auto& idx = d.get_index(); - auto lower_bound_lookup_tuple = std::make_tuple( t_id->id, primary_lower ); - auto upper_bound_lookup_tuple = std::make_tuple( t_id->id, primary_upper ); - - auto walk_table_row_range = [&]( auto itr, auto end_itr ) { - keep_processing kp; - vector data; - for( unsigned int count = 0; kp() && count < p.limit && itr != end_itr; ++count, ++itr ) { - result.rows.emplace_back( get_prim_key(*itr) ); - } - if( itr != end_itr ) { - handle_more(*itr); - } - }; - - auto lower = idx.lower_bound( lower_bound_lookup_tuple ); - auto upper = idx.upper_bound( upper_bound_lookup_tuple ); - if( reverse ) { - walk_table_row_range( boost::make_reverse_iterator(upper), boost::make_reverse_iterator(lower) ); - } else { - walk_table_row_range( lower, upper ); + + const auto* t_id = d.find(boost::make_tuple(p.code, scope, p.table)); + if( t_id != nullptr ) { + const auto& idx = d.get_index(); + auto lower_bound_lookup_tuple = std::make_tuple( t_id->id, primary_lower ); + auto upper_bound_lookup_tuple = std::make_tuple( t_id->id, primary_upper ); + + auto walk_table_row_range = [&]( auto itr, auto end_itr ) { + keep_processing kp; + vector data; + for( unsigned int count = 0; kp() && count < p.limit && itr != end_itr; ++count, ++itr ) { + result.rows.emplace_back( get_prim_key(*itr) ); } - } - } - else { - using namespace eosio::chain; - EOS_ASSERT(db_backing_store == backing_store_type::ROCKSDB, - chain::contract_table_query_exception, - "Support for configured backing_store has not been added to get_primary_key"); - const auto context = (reverse) ? backing_store::key_context::standalone_reverse : backing_store::key_context::standalone; - auto lower = chain::backing_store::db_key_value_format::create_full_primary_key(p.code, scope, p.table, primary_lower); - auto upper = chain::backing_store::db_key_value_format::create_full_primary_key(p.code, scope, p.table, primary_upper); - if (reverse) { - lower = eosio::session::shared_bytes::truncate_key(lower); - } - // since upper is either the upper_bound of a forward search, or the reverse iterator <= for the beginning of the end of - // this secondary type, we need to move it to just before the beginning of the next type - upper = upper.next(); - const auto& kv_database = db.kv_db(); - - keep_processing kp; - auto filter_primary_key = [&kp,&result,&p,&get_prim_key,&handle_more](const backing_store::primary_index_view& row) { - if (!kp() || result.rows.size() >= p.limit) { - handle_more(row); - return false; - } - else { - result.rows.emplace_back(get_prim_key(row)); - return true; + if( itr != end_itr ) { + handle_more(*itr); } }; - using primary_receiver = primary_key_receiver; - primary_receiver receiver(filter_primary_key); - auto keep_processing_entries = receiver.keep_processing_entries(); - backing_store::rocksdb_contract_db_table_writer> writer(receiver, context, keep_processing_entries); - eosio::chain::backing_store::walk_rocksdb_entries_with_prefix(kv_database.get_kv_undo_stack(), lower, upper, writer); + auto lower = idx.lower_bound( lower_bound_lookup_tuple ); + auto upper = idx.upper_bound( upper_bound_lookup_tuple ); + if( reverse ) { + walk_table_row_range( boost::make_reverse_iterator(upper), boost::make_reverse_iterator(lower) ); + } else { + walk_table_row_range( lower, upper ); + } } return result; } diff --git a/plugins/trace_api_plugin/include/eosio/trace_api/store_provider.hpp b/plugins/trace_api_plugin/include/eosio/trace_api/store_provider.hpp index 7045d6e151..113cbfbd0e 100644 --- a/plugins/trace_api_plugin/include/eosio/trace_api/store_provider.hpp +++ b/plugins/trace_api_plugin/include/eosio/trace_api/store_provider.hpp @@ -11,7 +11,7 @@ #include #include #include -#include +#include #include namespace eosio::trace_api { diff --git a/tests/amqp_tests.py b/tests/amqp_tests.py index f20b1f5184..2598202613 100755 --- a/tests/amqp_tests.py +++ b/tests/amqp_tests.py @@ -65,8 +65,8 @@ amqProducerAccount = cluster.defProducerAccounts["eosio"] - specificExtraNodeosArgs={ 0: " --backing-store=chainbase --plugin eosio::amqp_trx_plugin --amqp-trx-address %s" % (amqpAddr), - 1: " --backing-store=chainbase --plugin eosio::amqp_trx_plugin --amqp-trx-address %s" % (amqpAddr)} + specificExtraNodeosArgs={ 0: " --plugin eosio::amqp_trx_plugin --amqp-trx-address %s" % (amqpAddr), + 1: " --plugin eosio::amqp_trx_plugin --amqp-trx-address %s" % (amqpAddr)} traceNodeosArgs=" --plugin eosio::trace_api_plugin --trace-no-abis " if cluster.launch(totalNodes=2, totalProducers=3, pnodes=2, dontBootstrap=False, onlyBios=False, useBiosBootFile=True, specificExtraNodeosArgs=specificExtraNodeosArgs, extraNodeosArgs=traceNodeosArgs) is False: diff --git a/tests/eosvmoc_tests/codecache_tests.cpp b/tests/eosvmoc_tests/codecache_tests.cpp index 57f0c4fa62..03f46db8d4 100644 --- a/tests/eosvmoc_tests/codecache_tests.cpp +++ b/tests/eosvmoc_tests/codecache_tests.cpp @@ -25,29 +25,30 @@ BOOST_AUTO_TEST_SUITE(eosvmoc_cc_tests) BOOST_DATA_TEST_CASE(there_and_back_again, data::make(mapped_and_heap) * data::make(mapped_and_heap), first, second) { try { fc::temp_directory tmp_code_cache; - const eosvmoc::code_descriptor* first_desc = nullptr; + eosvmoc::code_descriptor first_desc; eosvmoc::config first_eosvmoc_config = {32u*1024u*1024u, 1u, first}; eosvmoc::config second_eosvmoc_config = {32u*1024u*1024u, 1u, second}; { eosvmoc::code_cache_sync cc(tmp_code_cache.path(), first_eosvmoc_config, get_some_wasm); - first_desc = cc.get_descriptor_for_code_sync(digest_type(), UINT8_C(0)); - BOOST_REQUIRE(first_desc); + const eosvmoc::code_descriptor* desc_tmp = cc.get_descriptor_for_code_sync(digest_type(), UINT8_C(0)); + BOOST_REQUIRE(desc_tmp); + first_desc = *desc_tmp; } { eosvmoc::code_cache_sync cc(tmp_code_cache.path(), second_eosvmoc_config, get_some_wasm); const eosvmoc::code_descriptor* const second_desc = cc.get_descriptor_for_code_sync(digest_type(), UINT8_C(0)); BOOST_REQUIRE(second_desc); - BOOST_REQUIRE_EQUAL(first_desc->code_begin, second_desc->code_begin); + BOOST_REQUIRE_EQUAL(first_desc.code_begin, second_desc->code_begin); } { eosvmoc::code_cache_sync cc(tmp_code_cache.path(), first_eosvmoc_config, get_some_wasm); const eosvmoc::code_descriptor* const desc = cc.get_descriptor_for_code_sync(digest_type(), UINT8_C(0)); BOOST_REQUIRE(desc); - BOOST_REQUIRE_EQUAL(first_desc->code_begin, desc->code_begin); + BOOST_REQUIRE_EQUAL(first_desc.code_begin, desc->code_begin); } } FC_LOG_AND_RETHROW() } diff --git a/tests/get_table_seckey_tests.cpp b/tests/get_table_seckey_tests.cpp index afa7c49790..05bc227127 100644 --- a/tests/get_table_seckey_tests.cpp +++ b/tests/get_table_seckey_tests.cpp @@ -30,7 +30,7 @@ using namespace fc; BOOST_AUTO_TEST_SUITE(get_table_seckey_tests) -using backing_store_ts = boost::mpl::list; +using backing_store_ts = boost::mpl::list; transaction_trace_ptr issue_tokens( TESTER& t, account_name issuer, account_name to, const asset& amount, diff --git a/tests/get_table_tests.cpp b/tests/get_table_tests.cpp index 3b6114b7c3..13267c0202 100644 --- a/tests/get_table_tests.cpp +++ b/tests/get_table_tests.cpp @@ -31,7 +31,7 @@ using namespace fc; BOOST_AUTO_TEST_SUITE(get_table_tests) -using backing_store_ts = boost::mpl::list; +using backing_store_ts = boost::mpl::list; transaction_trace_ptr issue_tokens( TESTER& t, account_name issuer, account_name to, const asset& amount, diff --git a/tests/nodeos_read_terminate_at_block_test.py b/tests/nodeos_read_terminate_at_block_test.py index ce6dc092fe..580df26f8e 100755 --- a/tests/nodeos_read_terminate_at_block_test.py +++ b/tests/nodeos_read_terminate_at_block_test.py @@ -168,7 +168,7 @@ def checkHeadOrSpeculative(head, lib): totalNodes=totalNodes, unstartedNodes=totalNodes - numOfProducers, pnodes=1, - useBiosBootFile=False, + #useBiosBootFile=False, topo="mesh", specificExtraNodeosArgs=specificNodeosArgs, extraNodeosArgs=traceNodeosArgs diff --git a/tests/nodeos_run_test.py b/tests/nodeos_run_test.py index 3c696802ba..257d5cdd4a 100755 --- a/tests/nodeos_run_test.py +++ b/tests/nodeos_run_test.py @@ -73,12 +73,10 @@ traceNodeosArgs=" --plugin eosio::trace_api_plugin --trace-rpc-abi eosio.token=" + abs_path if not amqpAddr: - specificExtraNodeosArgs={ 0 : " --backing-store=chainbase", - 1 : " --backing-store=rocksdb" } + specificExtraNodeosArgs={ } else: cluster.createAMQPQueue("trx") - specificExtraNodeosArgs={ 0: "--backing-store=chainbase --plugin eosio::amqp_trx_plugin --amqp-trx-address %s" % (amqpAddr), - 1 : " --backing-store=rocksdb" } + specificExtraNodeosArgs={ 0: "--plugin eosio::amqp_trx_plugin --amqp-trx-address %s" % (amqpAddr) } if cluster.launch(totalNodes=3, prodCount=prodCount, onlyBios=onlyBios, dontBootstrap=dontBootstrap, specificExtraNodeosArgs=specificExtraNodeosArgs, extraNodeosArgs=traceNodeosArgs) is False: cmdError("launcher") errorExit("Failed to stand up eos cluster.") diff --git a/unittests/api_tests.cpp b/unittests/api_tests.cpp index dad9127f32..df0c5d3952 100644 --- a/unittests/api_tests.cpp +++ b/unittests/api_tests.cpp @@ -90,17 +90,14 @@ using namespace eosio::testing; using namespace chain; using namespace fc; -using backing_store_ts = boost::mpl::list; +using backing_store_ts = boost::mpl::list; namespace test_detail { struct chainbase_backing_store_type { operator eosio::chain::backing_store_type() const { return eosio::chain::backing_store_type::CHAINBASE; } }; - struct rocksdb_backing_store_type { - operator eosio::chain::backing_store_type() const { return eosio::chain::backing_store_type::ROCKSDB; } - }; } -using backing_store_type_suite = boost::mpl::list; +using backing_store_type_suite = boost::mpl::list; namespace bio = boost::iostreams; diff --git a/unittests/db_to_kv_tests.cpp b/unittests/db_to_kv_tests.cpp deleted file mode 100644 index 1b9596e980..0000000000 --- a/unittests/db_to_kv_tests.cpp +++ /dev/null @@ -1,1139 +0,0 @@ -#include -#include -#include -#include - -#include -#include - -#include //REMOVE - -using key_type = eosio::chain::backing_store::db_key_value_format::key_type; -using name = eosio::chain::name; -using session_type = eosio::session::session>; -using namespace eosio::chain::literals; - -BOOST_AUTO_TEST_SUITE(db_to_kv_tests) - -using legacy_key = b1::chain_kv::bytes; -constexpr uint64_t legacy_overhead_size_without_type = - eosio::chain::backing_store::db_key_value_format::detail::prefix_size(); // 8 (scope) + 8 (table) -constexpr uint64_t legacy_overhead_size = legacy_overhead_size_without_type + 1; // ... + 1 (key type) -constexpr uint64_t current_overhead_size_without_type = - eosio::chain::backing_store::db_key_value_format::detail::prefix_size(); // 8 (scope) + 8 (table) -constexpr uint64_t current_overhead_size = current_overhead_size_without_type + 1; // ... + 1 (key type) - -float128_t to_softfloat128( double d ) { - return f64_to_f128(to_softfloat64(d)); -} - -double from_softfloat128( float128_t d ) { - return from_softfloat64(f128_to_f64(d)); -} - -constexpr eosio::chain::uint128_t create_uint128(uint64_t upper, uint64_t lower) { - return (eosio::chain::uint128_t(upper) << 64) + lower; -} - -constexpr eosio::chain::key256_t create_key256(eosio::chain::uint128_t upper, eosio::chain::uint128_t lower) { - return { upper, lower }; -} - -template -struct helper; - -template<> -struct helper { - using type_t = uint64_t; - using alt_type_t = eosio::chain::uint128_t; - const type_t init = 0x0; - type_t inc(type_t t) { return ++t; } -}; - -template<> -struct helper { - using type_t = eosio::chain::uint128_t; - using alt_type_t = eosio::chain::key256_t; - const type_t init = create_uint128(0, 0); - type_t inc(type_t t) { return ++t; } -}; - -template<> -struct helper { - using type_t = eosio::chain::key256_t; - using alt_type_t = float64_t; - const type_t init = create_key256(create_uint128(0, 0), create_uint128(0, 0)); - type_t inc(type_t t) { - if(!++t[1]) { - ++t[0]; - } - return t; - } -}; - -template<> -struct helper { - using type_t = float64_t; - using alt_type_t = float128_t; - const type_t init = to_softfloat64(0.0); - type_t inc(type_t t) { return to_softfloat64(from_softfloat64(t) + 1.0); } -}; - -template<> -struct helper { - using type_t = float128_t; - using alt_type_t = uint64_t; - const type_t init = to_softfloat128(0.0); - type_t inc(type_t t) { return to_softfloat128(from_softfloat128(t) + 1.0); } -}; - -template -void verify_secondary_wont_convert(const b1::chain_kv::bytes& composite_key, const name exp_scope, const name exp_table) { - name scope; - name table; - Type key = helper().init; - const Type exp_key = key; - uint64_t primary_key = 0; - BOOST_CHECK(!eosio::chain::backing_store::db_key_value_format::get_secondary_key(composite_key, scope, table, key, primary_key)); - BOOST_CHECK_EQUAL(exp_scope.to_string(), scope.to_string()); - BOOST_CHECK_EQUAL(exp_table.to_string(), table.to_string()); - BOOST_CHECK(exp_key == key); -} - -void verify_other_gets(const b1::chain_kv::bytes& composite_key, key_type except, const name exp_scope, const name exp_table) { - if (except != key_type::primary) { - name scope; - name table; - uint64_t key = 0; - BOOST_CHECK(!eosio::chain::backing_store::db_key_value_format::get_primary_key(composite_key, scope, table, key)); - BOOST_CHECK_EQUAL(exp_scope.to_string(), scope.to_string()); - BOOST_CHECK_EQUAL(exp_table.to_string(), table.to_string()); - BOOST_CHECK(0 == key); - } - if (except != key_type::sec_i64) { verify_secondary_wont_convert(composite_key, exp_scope, exp_table); } - if (except != key_type::sec_i128) { verify_secondary_wont_convert(composite_key, exp_scope, exp_table); } - if (except != key_type::sec_i256) { verify_secondary_wont_convert(composite_key, exp_scope, exp_table); } - if (except != key_type::sec_double) { verify_secondary_wont_convert(composite_key, exp_scope, exp_table); } - if (except != key_type::sec_long_double) { verify_secondary_wont_convert(composite_key, exp_scope, exp_table); } - if (except != key_type::table) { - name scope; - name table; - BOOST_CHECK(!eosio::chain::backing_store::db_key_value_format::get_table_key(composite_key, scope, table)); - BOOST_CHECK_EQUAL(exp_scope.to_string(), scope.to_string()); - BOOST_CHECK_EQUAL(exp_table.to_string(), table.to_string()); - } - name scope; - name table; - BOOST_CHECK_NO_THROW(eosio::chain::backing_store::db_key_value_format::get_prefix_key(composite_key, scope, table)); - BOOST_CHECK_EQUAL(exp_scope.to_string(), scope.to_string()); - BOOST_CHECK_EQUAL(exp_table.to_string(), table.to_string()); -} - -template -rocksdb::Slice make_slice(const CharCont& cont) { - return rocksdb::Slice{cont.data(), cont.size()}; -} - -template -std::pair compare_composite_keys(const CharCont& lhs_cont, const CharCont& rhs_cont, bool print = false) { - unsigned int j = 0; - const int gt = 1; - const int lt = -1; - const int equal = 0; - const char* lhs = lhs_cont.data(); - const char* rhs = rhs_cont.data(); - const auto lhs_size = lhs_cont.size(); - const auto rhs_size = rhs_cont.size(); - for (; j < lhs_size; ++j) { - if (j >= rhs_size) { - if (print) ilog("${j}: lhs longer than rhs", ("j", j)); - return { gt, j }; - } - const auto left = uint64_t(static_cast(lhs[j])); - const auto right = uint64_t(static_cast(rhs[j])); - if (print) ilog("${j}: ${l} ${sym} ${r}", ("j", j)("l", left)("sym",(left == right ? "==" : "!="))("r", right)); - if (left != right) { - return { left < right ? lt : gt, j }; - } - } - if (rhs_size > lhs_size) { - if (print) ilog("rhs longer (${r}) than lhs (${l})", ("r", rhs_size)("l", lhs_size)); - return { lt, j }; - } - - return { equal, j }; -} - -template -std::pair compare_composite_keys(const CharKey& keys, uint64_t lhs_index, bool print = false) { - if (print) ilog("verifying [${i1}] and [${i2}]",("i1", lhs_index)("i2", lhs_index + 1)); - return compare_composite_keys(keys[lhs_index], keys[lhs_index + 1], print); -} - -template -void verify_ascending_composite_keys(const CharKey& keys, uint64_t lhs_index, std::string desc, bool print = false) { - const auto ret = compare_composite_keys(keys, lhs_index, print); - BOOST_REQUIRE_MESSAGE(ret.first != 1, "expected " + std::to_string(lhs_index) + - "th composite key to be less than the " + std::to_string(lhs_index + 1) + "th, but the " + std::to_string(ret.second) + - "th character is actually greater on the left than the right. Called from: " + desc); - BOOST_REQUIRE_MESSAGE(ret.first != 0, "expected " + std::to_string(lhs_index) + - "th composite key to be less than the " + std::to_string(lhs_index + 1) + - "th, but they were identical. Called from: " + desc); -} - -template -void verify_scope_table_order(T low_key, T high_key, KeyFunc key_func, bool skip_trailing_prim_key) { - using namespace eosio::chain::backing_store; - using end_of_prefix = db_key_value_format::end_of_prefix; - std::vector full_keys; - std::vector comp_keys; - unsigned next_comp_index = 0; - // call this before switching to a new table and scope pair - auto process_next = [&comp_keys,&next_comp_index,&full_keys]() { - const auto start_index = next_comp_index; - auto comp_index = start_index; - name contract{0x0000'0000'0000'0000}; - comp_index = start_index; - - // only the first of a group belonging to the same table/scope - const auto& comp_key= comp_keys.at(comp_index++); - auto full_key = db_key_value_format::create_full_key(comp_key, contract); - full_keys.push_back(db_key_value_format::create_full_key_prefix(full_key, end_of_prefix::pre_type)); - full_keys.push_back(db_key_value_format::create_full_key_prefix(full_key, end_of_prefix::at_type)); - full_keys.push_back(full_key); - - // the rest will all have prefixes that will match the above - for (; comp_index < comp_keys.size(); ++comp_index) { - const auto& comp_key= comp_keys.at(comp_index); - full_key = db_key_value_format::create_full_key(comp_key, contract); - full_keys.push_back(full_key); - } - next_comp_index = comp_index; - }; - - comp_keys.push_back(key_func(name{0}, name{0}, low_key, 0x0)); - // this function always passes the extra primary key, but need to skip what would be redundant keys for when we are - // actually getting a primary key - if (!skip_trailing_prim_key) { - comp_keys.push_back(key_func(name{0}, name{0}, low_key, 0x1)); - comp_keys.push_back(key_func(name{0}, name{0}, low_key, 0xffff'ffff'ffff'ffff)); - } - process_next(); - - comp_keys.push_back(key_func(name{1}, name{0}, low_key, 0x0)); - process_next(); - - comp_keys.push_back(key_func(name{1}, name{1}, low_key, 0x0)); - comp_keys.push_back(key_func(name{1}, name{1}, high_key, 0x0)); - if (!skip_trailing_prim_key) { - comp_keys.push_back(key_func(name{1}, name{1}, high_key, 0x1)); - } - process_next(); - - comp_keys.push_back(key_func(name{1}, name{2}, low_key, 0x0)); - process_next(); - - comp_keys.push_back(key_func(name{1}, name{0xffff'ffff'ffff'ffff}, low_key, 0x0)); - process_next(); - - comp_keys.push_back(key_func(name{0xffff'ffff'ffff'ffff}, name{0}, low_key, 0x0)); - process_next(); - - comp_keys.push_back(key_func(name{0xffff'ffff'ffff'ffff}, name{0xffff'ffff'ffff'ffff}, low_key, 0x0)); - process_next(); - - for (unsigned int i = 0; i < comp_keys.size() - 1; ++i) { - verify_ascending_composite_keys(comp_keys, i, "verify_scope_table_order comp_keys"); - } - for (unsigned int i = 0; i < full_keys.size() - 1; ++i) { - verify_ascending_composite_keys(full_keys, i, "verify_scope_table_order full_keys", true); - } -} - -// using this function to allow check_ordered_keys to work for all key types -b1::chain_kv::bytes prim_drop_extra_key(name scope, name table, uint64_t db_key, uint64_t unused_primary_key) { - return eosio::chain::backing_store::db_key_value_format::create_primary_key(scope, table, db_key); -}; - -template -void check_ordered_keys(const std::vector& ordered_keys, KeyFunc key_func, bool skip_trailing_prim_key = false) { - verify_scope_table_order(ordered_keys[0], ordered_keys[1], key_func, skip_trailing_prim_key); - const name contract{0}; - const name scope{0}; - const name table{0}; - const uint64_t prim_key = 0; - std::vector composite_keys; - std::vector full_keys; - for (const auto& key: ordered_keys) { - auto comp_key = key_func(scope, table, key, prim_key); - composite_keys.push_back(comp_key); - auto full_key = eosio::chain::backing_store::db_key_value_format::create_full_key(comp_key, contract); - full_keys.push_back(full_key); - } - for (unsigned int i = 0; i < composite_keys.size() - 1; ++i) { - verify_ascending_composite_keys(composite_keys, i, "check_ordered_keys composite_keys"); - } - for (unsigned int i = 0; i < full_keys.size() - 1; ++i) { - verify_ascending_composite_keys(full_keys, i, "check_ordered_keys full_keys"); - } -} - -void verify_secondary_key_pair_order(const eosio::chain::backing_store::db_key_value_format::secondary_key_pair& incoming_key_pair, const key_type kt) { - using slice_array = std::array; - const slice_array incoming { - eosio::chain::backing_store::db_key_value_format::prefix_slice(incoming_key_pair.primary_to_secondary_key), - eosio::chain::backing_store::db_key_value_format::prefix_type_slice(incoming_key_pair.primary_to_secondary_key), - eosio::chain::backing_store::db_key_value_format::prefix_primary_to_secondary_slice(incoming_key_pair.primary_to_secondary_key, false), - eosio::chain::backing_store::db_key_value_format::prefix_primary_to_secondary_slice(incoming_key_pair.primary_to_secondary_key, true) - }; - - // to make this test less error prone, extract the scope and table ... - name scope; - name table; - uint64_t primary_key = 0x0; - BOOST_REQUIRE_NO_THROW(eosio::chain::backing_store::db_key_value_format::get_prefix_key(incoming_key_pair.secondary_key, scope, table)); - - // ... and the primary key - const rocksdb::Slice complete_key { incoming_key_pair.secondary_key.data(), incoming_key_pair.secondary_key.size() }; - const rocksdb::Slice all_but_primary_key { incoming_key_pair.secondary_key.data(), incoming_key_pair.secondary_key.size() - sizeof(primary_key) }; - BOOST_REQUIRE(eosio::chain::backing_store::db_key_value_format::get_trailing_primary_key(complete_key, all_but_primary_key, primary_key)); - - rocksdb::Slice pair_sec_key_prefix; - rocksdb::Slice pair_sec_key_prefix_type; - rocksdb::Slice pair_sec_key_all_but_secondary_prefix; - - bool type_seen = false; // keep track so we know if the type we are currently checking is after or before the incoming type - unsigned int other_types_processed = 0; - int expected_type_comp = 0; - key_type current_kt = key_type::primary; - // lambda to correctly identify comparisons for each type - auto type_check = [&type_seen, &other_types_processed, ¤t_kt, &expected_type_comp, kt](key_type curr_kt) { - current_kt = curr_kt; - if (current_kt != kt) { - ++other_types_processed; - expected_type_comp = type_seen ? -1 : 1; - } else { - expected_type_comp = 0; - BOOST_REQUIRE(!type_seen); - type_seen = true; - } - }; - - auto desc = [&kt, ¤t_kt](int comp) -> std::string { - std::string msg = "incoming type: " + eosio::chain::backing_store::db_key_value_format::detail::to_string(kt); - msg += ", comparing to type: " + eosio::chain::backing_store::db_key_value_format::detail::to_string(current_kt); - msg += ", expected incoming to be "; - if (comp > 0) { - msg += "greater than"; - } - else if (comp < 0) { - msg += "less than"; - } - else { - msg += "equal to"; - } - return msg; - }; - - auto split_keys = [](const b1::chain_kv::bytes& composite) -> slice_array { - return { - eosio::chain::backing_store::db_key_value_format::prefix_slice(composite), - eosio::chain::backing_store::db_key_value_format::prefix_type_slice(composite), - eosio::chain::backing_store::db_key_value_format::prefix_primary_to_secondary_slice(composite, false), - eosio::chain::backing_store::db_key_value_format::prefix_primary_to_secondary_slice(composite, true) - }; - }; - - auto compare_keys = [&incoming, &expected_type_comp, &desc](const slice_array& values) { - BOOST_CHECK_MESSAGE(0 == compare_composite_keys(incoming[0], values[0]).first, "table and scope prefix doesn't match: " + desc(0)); - BOOST_CHECK_MESSAGE(0 == compare_composite_keys(incoming[1], values[1]).first, "type prefix doesn't match: " + desc(0)); - BOOST_CHECK_MESSAGE(expected_type_comp == compare_composite_keys(incoming[2], values[2]).first, "secondary type prefix comparison failed expectations: " + desc(expected_type_comp)); - BOOST_CHECK_MESSAGE(expected_type_comp == compare_composite_keys(incoming[3], values[3]).first, "primary key prefix comparison failed expectations: " + desc(expected_type_comp)); - static_assert(std::tuple_size::value == 4); // ensure we don't miss comparing values - }; - - const auto primary_key2 = primary_key + 1; - BOOST_REQUIRE_LT(primary_key, primary_key2); - BOOST_REQUIRE_EQUAL(primary_key + 1, primary_key2); - - type_check(key_type::sec_i64); - auto sec_key_pair = eosio::chain::backing_store::db_key_value_format::create_secondary_key_pair(scope, table, helper().init, primary_key); - auto values = split_keys(sec_key_pair.primary_to_secondary_key); - compare_keys(values); - auto prim_to_sec_key = eosio::chain::backing_store::db_key_value_format::create_primary_to_secondary_key(scope, table, primary_key2, helper().init); - auto values2 = split_keys(prim_to_sec_key); - // ensure that the primary_to_secondary prefix for a specific type matches - BOOST_CHECK_EQUAL(0, compare_composite_keys(values[2], values2[2]).first); - // ensure that their ordered by primary_key - BOOST_REQUIRE_EQUAL(-1, compare_composite_keys(values[3], values2[3]).first); - - type_check(key_type::sec_i128); - sec_key_pair = eosio::chain::backing_store::db_key_value_format::create_secondary_key_pair(scope, table, helper().init, primary_key); - values = split_keys(sec_key_pair.primary_to_secondary_key); - compare_keys(values); - prim_to_sec_key = eosio::chain::backing_store::db_key_value_format::create_primary_to_secondary_key(scope, table, primary_key2, helper().init); - values2 = split_keys(prim_to_sec_key); - // ensure that the primary_to_secondary prefix for a specific type matches - BOOST_CHECK_EQUAL(0, compare_composite_keys(values[2], values2[2]).first); - // ensure that their ordered by primary_key - BOOST_CHECK_EQUAL(-1, compare_composite_keys(sec_key_pair.primary_to_secondary_key, prim_to_sec_key).first); - BOOST_CHECK_EQUAL(-1, compare_composite_keys(values[3], values2[3]).first); - - type_check(key_type::sec_i256); - sec_key_pair = eosio::chain::backing_store::db_key_value_format::create_secondary_key_pair(scope, table, helper().init, primary_key); - values = split_keys(sec_key_pair.primary_to_secondary_key); - compare_keys(values); - prim_to_sec_key = eosio::chain::backing_store::db_key_value_format::create_primary_to_secondary_key(scope, table, primary_key2, helper().init); - values2 = split_keys(prim_to_sec_key); - // ensure that the primary_to_secondary prefix for a specific type matches - BOOST_CHECK_EQUAL(0, compare_composite_keys(values[2], values2[2]).first); - // ensure that their ordered by primary_key - BOOST_CHECK_EQUAL(-1, compare_composite_keys(values[3], values2[3]).first); - - type_check(key_type::sec_double); - sec_key_pair = eosio::chain::backing_store::db_key_value_format::create_secondary_key_pair(scope, table, helper().init, primary_key); - values = split_keys(sec_key_pair.primary_to_secondary_key); - compare_keys(values); - prim_to_sec_key = eosio::chain::backing_store::db_key_value_format::create_primary_to_secondary_key(scope, table, primary_key2, helper().init); - values2 = split_keys(prim_to_sec_key); - // ensure that the primary_to_secondary prefix for a specific type matches - BOOST_CHECK_EQUAL(0, compare_composite_keys(values[2], values2[2]).first); - // ensure that their ordered by primary_key - BOOST_CHECK_EQUAL(-1, compare_composite_keys(values[3], values2[3]).first); - - type_check(key_type::sec_long_double); - sec_key_pair = eosio::chain::backing_store::db_key_value_format::create_secondary_key_pair(scope, table, helper().init, primary_key); - values = split_keys(sec_key_pair.primary_to_secondary_key); - compare_keys(values); - prim_to_sec_key = eosio::chain::backing_store::db_key_value_format::create_primary_to_secondary_key(scope, table, primary_key2, helper().init); - values2 = split_keys(prim_to_sec_key); - // ensure that the primary_to_secondary prefix for a specific type matches - BOOST_CHECK_EQUAL(0, compare_composite_keys(values[2], values2[2]).first); - // ensure that their ordered by primary_key - BOOST_CHECK_EQUAL(-1, compare_composite_keys(values[3], values2[3]).first); - - BOOST_REQUIRE_EQUAL(4, other_types_processed); // make sure that if a new secondary key is added that this will fail - BOOST_REQUIRE(type_seen); // make sure that if a new secondary key is added that this will fail -} - -template -void verify_primary_to_sec_key(const eosio::chain::backing_store::db_key_value_format::secondary_key_pair& key_pair, name scope, name table, uint64_t primary_key, Key sec_key) { - name extracted_scope; - name extracted_table; - const uint64_t default_primary_key = 0xffff'ffff'ffff'ffff; - BOOST_REQUIRE_NE(default_primary_key, primary_key); - uint64_t extracted_primary_key = default_primary_key; - Key extracted_key; - - const key_type actual_kt = eosio::chain::backing_store::db_key_value_format::extract_key_type(key_pair.primary_to_secondary_key); - BOOST_REQUIRE(key_type::primary_to_sec == actual_kt); - const key_type actual_sec_kt = eosio::chain::backing_store::db_key_value_format::extract_primary_to_sec_key_type(key_pair.primary_to_secondary_key); - const key_type expected_sec_kt = eosio::chain::backing_store::db_key_value_format::derive_secondary_key_type(); - BOOST_REQUIRE(expected_sec_kt == actual_sec_kt); - - b1::chain_kv::bytes::const_iterator composite_loc; - key_type kt = key_type::table; - std::tie(extracted_scope, extracted_table, composite_loc, kt) = eosio::chain::backing_store::db_key_value_format::get_prefix_thru_key_type(key_pair.primary_to_secondary_key); - BOOST_CHECK_EQUAL(scope.to_string(), extracted_scope.to_string()); - BOOST_CHECK_EQUAL(table.to_string(), extracted_table.to_string()); - // make sure that there is a primary key and secondary key's worth of memory before reaching the end of the key - BOOST_CHECK_EQUAL(sizeof(uint64_t) + sizeof(Key) + 1, std::distance(composite_loc, key_pair.primary_to_secondary_key.cend())); - BOOST_CHECK(key_type::primary_to_sec == kt); - - extracted_scope = name{}; - extracted_table = name{}; - // use this primary_to_sec_key to retrieve the correct type - auto result = eosio::chain::backing_store::db_key_value_format::get_primary_to_secondary_keys( - key_pair.primary_to_secondary_key, extracted_scope, extracted_table, extracted_primary_key, extracted_key); - BOOST_REQUIRE(eosio::chain::backing_store::db_key_value_format::prim_to_sec_type_result::valid_type == result); - BOOST_CHECK_EQUAL(scope.to_string(), extracted_scope.to_string()); - BOOST_CHECK_EQUAL(table.to_string(), extracted_table.to_string()); - BOOST_CHECK_EQUAL(primary_key, extracted_primary_key); - BOOST_CHECK(sec_key == extracted_key); - - // used this primary_to_sec_key to try and retrieve another secondary type - extracted_scope = name{}; - extracted_table = name{}; - extracted_primary_key = default_primary_key; - using alt_type_t = typename helper::alt_type_t; - alt_type_t alt_key = helper{}.init; - result = eosio::chain::backing_store::db_key_value_format::get_primary_to_secondary_keys( - key_pair.primary_to_secondary_key, extracted_scope, extracted_table, primary_key, alt_key); - BOOST_REQUIRE(eosio::chain::backing_store::db_key_value_format::prim_to_sec_type_result::wrong_sec_type == result); - BOOST_CHECK_EQUAL(scope.to_string(), extracted_scope.to_string()); - BOOST_CHECK_EQUAL(table.to_string(), extracted_table.to_string()); - BOOST_CHECK_EQUAL(default_primary_key, extracted_primary_key); - BOOST_CHECK(helper{}.init == alt_key); - - extracted_scope = name{}; - extracted_table = name{}; - extracted_primary_key = default_primary_key; - extracted_key = helper{}.init; - Key invalid_key; - result = eosio::chain::backing_store::db_key_value_format::get_primary_to_secondary_keys( - key_pair.secondary_key, extracted_scope, extracted_table, primary_key, alt_key); - BOOST_REQUIRE(eosio::chain::backing_store::db_key_value_format::prim_to_sec_type_result::invalid_type == result); - BOOST_CHECK_EQUAL(scope.to_string(), extracted_scope.to_string()); - BOOST_CHECK_EQUAL(table.to_string(), extracted_table.to_string()); - BOOST_CHECK_EQUAL(default_primary_key, extracted_primary_key); - BOOST_CHECK(helper{}.init == extracted_key); -} - -template -void verify_secondary_key_pair(const b1::chain_kv::bytes& composite_key) { - name scope; - name table; - Key sec_key; - uint64_t primary_key; - BOOST_REQUIRE_NO_THROW(eosio::chain::backing_store::db_key_value_format::get_secondary_key(composite_key, scope, table, sec_key, primary_key)); - Key sec_key2= helper{}.init; - uint64_t primary_key2 = 0; - const auto type_prefix = eosio::chain::backing_store::db_key_value_format::prefix_type_slice(composite_key); - BOOST_REQUIRE_NO_THROW(eosio::chain::backing_store::db_key_value_format::get_trailing_sec_prim_keys(composite_key, type_prefix, sec_key2, primary_key2)); - BOOST_CHECK(sec_key == sec_key2); - BOOST_CHECK_EQUAL(primary_key, primary_key2); - - const auto sec_key_pair = eosio::chain::backing_store::db_key_value_format::create_secondary_key_pair(scope, table, sec_key, primary_key); - BOOST_REQUIRE_EQUAL(legacy_overhead_size + sizeof(sec_key) + sizeof(primary_key), sec_key_pair.secondary_key.size()); - BOOST_CHECK_EQUAL(0, compare_composite_keys(composite_key, sec_key_pair.secondary_key).first); - BOOST_CHECK_EQUAL(legacy_overhead_size + sizeof(primary_key) + sizeof(key_type) + sizeof(sec_key), sec_key_pair.primary_to_secondary_key.size()); - const auto primary_to_sec_key = eosio::chain::backing_store::db_key_value_format::create_primary_to_secondary_key(scope, table, primary_key, sec_key); - BOOST_CHECK_EQUAL(legacy_overhead_size + sizeof(primary_key) + sizeof(key_type) + sizeof(sec_key), primary_to_sec_key.size()); - BOOST_CHECK_EQUAL(0, compare_composite_keys(sec_key_pair.primary_to_secondary_key, primary_to_sec_key).first); - const key_type actual_kt = eosio::chain::backing_store::db_key_value_format::extract_key_type(sec_key_pair.secondary_key); - BOOST_CHECK(eosio::chain::backing_store::db_key_value_format::derive_secondary_key_type() == actual_kt); - - - // extract using the type prefix, the extended type prefix (with the indication of the secondary type), and from the prefix including the primary type - const uint64_t default_primary_key = 0xffff'ffff'ffff'ffff; - uint64_t extracted_primary_key2 = default_primary_key; - // since this method derives primary_key, ensure we don't use a value in here that means we are not really checking that the key is extracted - BOOST_REQUIRE_NE(extracted_primary_key2, primary_key); - Key extracted_key2; - auto primary_to_sec_prefix = eosio::chain::backing_store::db_key_value_format::prefix_type_slice(primary_to_sec_key); - auto result = eosio::chain::backing_store::db_key_value_format::get_trailing_prim_sec_keys( - primary_to_sec_key, primary_to_sec_prefix, extracted_primary_key2, extracted_key2); - BOOST_REQUIRE(result); - BOOST_CHECK_EQUAL(primary_key, extracted_primary_key2); - BOOST_CHECK(sec_key == extracted_key2); - - verify_primary_to_sec_key(sec_key_pair, scope, table, primary_key, sec_key); - - uint64_t extracted_primary_key3 = 0xffff'ffff'ffff'ffff; - Key extracted_key3; - primary_to_sec_prefix = eosio::chain::backing_store::db_key_value_format::prefix_primary_to_secondary_slice(primary_to_sec_key, false); - result = eosio::chain::backing_store::db_key_value_format::get_trailing_prim_sec_keys( - primary_to_sec_key, primary_to_sec_prefix, extracted_primary_key3, extracted_key3); - BOOST_REQUIRE(result); - BOOST_CHECK_EQUAL(primary_key, extracted_primary_key3); - BOOST_CHECK(sec_key == extracted_key3); - - Key extracted_key4; - primary_to_sec_prefix = eosio::chain::backing_store::db_key_value_format::prefix_primary_to_secondary_slice(primary_to_sec_key, true); - result = eosio::chain::backing_store::db_key_value_format::get_trailing_secondary_key( - primary_to_sec_key, primary_to_sec_prefix, extracted_key4); - BOOST_REQUIRE(result); - BOOST_CHECK(sec_key == extracted_key4); - - using end_of_prefix = eosio::chain::backing_store::db_key_value_format::end_of_prefix; - uint64_t extracted_primary_key5 = default_primary_key; - Key extracted_key5; - name contract = "mycontract"_n; - auto full_primary_to_sec_key = eosio::chain::backing_store::db_key_value_format::create_full_key(primary_to_sec_key, contract); - auto prefix_primary_to_sec_key = eosio::chain::backing_store::db_key_value_format::create_full_key_prefix(full_primary_to_sec_key, end_of_prefix::at_type); - result = eosio::chain::backing_store::db_key_value_format::get_trailing_prim_sec_keys( - full_primary_to_sec_key, prefix_primary_to_sec_key, extracted_primary_key5, extracted_key5); - BOOST_REQUIRE(result); - BOOST_REQUIRE_EQUAL(primary_key, extracted_primary_key5); - BOOST_REQUIRE(sec_key == extracted_key5); - - uint64_t extracted_primary_key6 = default_primary_key; - Key extracted_key6; - auto type_prefix_primary_to_sec_key = eosio::chain::backing_store::db_key_value_format::create_full_key_prefix(full_primary_to_sec_key, end_of_prefix::at_prim_to_sec_type); - result = eosio::chain::backing_store::db_key_value_format::get_trailing_prim_sec_keys( - full_primary_to_sec_key, type_prefix_primary_to_sec_key, extracted_primary_key6, extracted_key6); - BOOST_REQUIRE(result); - BOOST_REQUIRE_EQUAL(primary_key, extracted_primary_key6); - BOOST_REQUIRE(sec_key == extracted_key6); - - Key extracted_key7; - type_prefix_primary_to_sec_key = eosio::chain::backing_store::db_key_value_format::create_full_key_prefix(full_primary_to_sec_key, end_of_prefix::at_prim_to_sec_primary_key); - result = eosio::chain::backing_store::db_key_value_format::get_trailing_secondary_key( - full_primary_to_sec_key, type_prefix_primary_to_sec_key, extracted_key7); - BOOST_REQUIRE(result); - BOOST_REQUIRE(sec_key == extracted_key7); - - auto full_primary_from_prim_to_sec_key = eosio::chain::backing_store::db_key_value_format::create_secondary_key_from_primary_to_sec_key(full_primary_to_sec_key); - auto legacy_primary_from_prim_to_sec_key = eosio::chain::backing_store::db_key_value_format::extract_legacy_key(full_primary_from_prim_to_sec_key); - auto comp = compare_composite_keys(legacy_primary_from_prim_to_sec_key, sec_key_pair.secondary_key, true); - BOOST_REQUIRE_EQUAL(0, comp.first); - - const key_type kt = eosio::chain::backing_store::db_key_value_format::detail::determine_sec_type::kt; - verify_secondary_key_pair_order(sec_key_pair, kt); -} - -BOOST_AUTO_TEST_CASE(ui64_keys_conversions_test) { - const name scope = "myscope"_n; - const name table = "thisscope"_n; - const uint64_t key = 0xdead87654321beef; - const auto composite_key = eosio::chain::backing_store::db_key_value_format::create_primary_key(scope, table, key); - BOOST_REQUIRE_EQUAL(legacy_overhead_size + sizeof(key), composite_key.size()); - - name decomposed_scope; - name decomposed_table; - uint64_t decomposed_key = 0x0; - BOOST_REQUIRE_NO_THROW(eosio::chain::backing_store::db_key_value_format::get_primary_key(composite_key, decomposed_scope, decomposed_table, decomposed_key)); - BOOST_CHECK_EQUAL(scope.to_string(), decomposed_scope.to_string()); - BOOST_CHECK_EQUAL(table.to_string(), decomposed_table.to_string()); - BOOST_CHECK_EQUAL(key, decomposed_key); - verify_other_gets(composite_key, key_type::primary, scope, table); - - std::vector keys; - keys.push_back(0x0); - keys.push_back(0x1); - keys.push_back(0x7FFF'FFFF'FFFF'FFFF); - keys.push_back(0x8000'0000'0000'0000); - keys.push_back(0xFFFF'FFFF'FFFF'FFFF); - const bool skip_trailing_prim_key = true; - check_ordered_keys(keys, &prim_drop_extra_key, skip_trailing_prim_key); - - const name scope2 = "myscope"_n; - const name table2 = "thisscope"_n; - const uint64_t key2 = 0xdead87654321beef; - const uint64_t primary_key2 = 0x0123456789abcdef; - const auto composite_key2 = eosio::chain::backing_store::db_key_value_format::create_secondary_key(scope2, table2, key2, primary_key2); - BOOST_REQUIRE_EQUAL(legacy_overhead_size + sizeof(key) + sizeof(primary_key2), composite_key2.size()); - - uint64_t decomposed_primary_key = 0x0; - BOOST_CHECK(eosio::chain::backing_store::db_key_value_format::get_secondary_key(composite_key2, decomposed_scope, decomposed_table, decomposed_key, decomposed_primary_key)); - BOOST_CHECK_EQUAL(scope2.to_string(), decomposed_scope.to_string()); - BOOST_CHECK_EQUAL(table2.to_string(), decomposed_table.to_string()); - BOOST_CHECK_EQUAL(key2, decomposed_key); - BOOST_CHECK_EQUAL(primary_key2, decomposed_primary_key); - verify_other_gets(composite_key2, key_type::sec_i64, scope2, table2); - - verify_secondary_key_pair(composite_key2); - - std::vector keys2; - keys2.push_back(0x0); - keys2.push_back(0x1); - keys2.push_back(0x7FFF'FFFF'FFFF'FFFF); - keys2.push_back(0x8000'0000'0000'0000); - keys2.push_back(0xFFFF'FFFF'FFFF'FFFF); - check_ordered_keys(keys2, &eosio::chain::backing_store::db_key_value_format::create_secondary_key); -} - -BOOST_AUTO_TEST_CASE(ui128_key_conversions_test) { - const name scope = "myscope"_n; - const name table = "thisscope"_n; - const eosio::chain::uint128_t key = create_uint128(0xdead12345678beef, 0x0123456789abcdef); - const uint64_t primary_key = 0x0123456789abcdef; - - const auto composite_key = eosio::chain::backing_store::db_key_value_format::create_secondary_key(scope, table, key, primary_key); - BOOST_REQUIRE_EQUAL(legacy_overhead_size + sizeof(key) + sizeof(primary_key), composite_key.size()); - - name decomposed_scope; - name decomposed_table; - eosio::chain::uint128_t decomposed_key = 0x0; - uint64_t decomposed_primary_key = 0x0; - BOOST_REQUIRE_NO_THROW(eosio::chain::backing_store::db_key_value_format::get_secondary_key(composite_key, decomposed_scope, decomposed_table, decomposed_key, decomposed_primary_key)); - BOOST_CHECK_EQUAL(scope.to_string(), decomposed_scope.to_string()); - BOOST_CHECK_EQUAL(table.to_string(), decomposed_table.to_string()); - BOOST_CHECK(key == decomposed_key); - BOOST_CHECK_EQUAL(primary_key, decomposed_primary_key); - - verify_other_gets(composite_key, key_type::sec_i128, scope, table); - - verify_secondary_key_pair(composite_key); - - std::vector keys; - keys.push_back(create_uint128(0x0, 0x0)); - keys.push_back(create_uint128(0x0, 0x1)); - keys.push_back(create_uint128(0x0, 0x7FFF'FFFF'FFFF'FFFF)); - keys.push_back(create_uint128(0x0, 0x8000'0000'0000'0000)); - keys.push_back(create_uint128(0x0, 0xFFFF'FFFF'FFFF'FFFF)); - keys.push_back(create_uint128(0x1, 0x0)); - keys.push_back(create_uint128(0x1, 0xFFFF'FFFF'FFFF'FFFF)); - keys.push_back(create_uint128(0x7FFF'FFFF'FFFF'FFFF, 0xFFFF'FFFF'FFFF'FFFF)); - keys.push_back(create_uint128(0x8000'0000'0000'0000, 0xFFFF'FFFF'FFFF'FFFF)); - keys.push_back(create_uint128(0xFFFF'FFFF'FFFF'FFFF, 0xFFFF'FFFF'FFFF'FFFF)); - check_ordered_keys(keys, &eosio::chain::backing_store::db_key_value_format::create_secondary_key); -} - -BOOST_AUTO_TEST_CASE(ui256_key_conversions_test) { - const name scope = "myscope"_n; - const name table = "thisscope"_n; - const eosio::chain::key256_t key = create_key256(create_uint128(0xdead12345678beef, 0x0123456789abcdef), - create_uint128(0xbadf2468013579ec, 0xf0e1d2c3b4a59687)); - const uint64_t primary_key = 0x0123456789abcdef; - const auto composite_key = eosio::chain::backing_store::db_key_value_format::create_secondary_key(scope, table, key, primary_key); - BOOST_REQUIRE_EQUAL(legacy_overhead_size + sizeof(key) + sizeof(primary_key), composite_key.size()); - - name decomposed_scope; - name decomposed_table; - eosio::chain::key256_t decomposed_key = create_key256(create_uint128(0x0, 0x0), create_uint128(0x0, 0x0)); - uint64_t decomposed_primary_key = 0; - BOOST_REQUIRE_NO_THROW(eosio::chain::backing_store::db_key_value_format::get_secondary_key(composite_key, decomposed_scope, decomposed_table, decomposed_key, decomposed_primary_key)); - BOOST_CHECK_EQUAL(scope.to_string(), decomposed_scope.to_string()); - BOOST_CHECK_EQUAL(table.to_string(), decomposed_table.to_string()); - - BOOST_CHECK(key == decomposed_key); - BOOST_CHECK_EQUAL(primary_key, decomposed_primary_key); - - verify_other_gets(composite_key, key_type::sec_i256, scope, table); - - verify_secondary_key_pair(composite_key); - - std::vector keys; - keys.push_back(create_key256(create_uint128(0x0, 0x0), create_uint128(0x0, 0x0))); - keys.push_back(create_key256(create_uint128(0x0, 0x0), create_uint128(0x0, 0x01))); - keys.push_back(create_key256(create_uint128(0x0, 0x0), create_uint128(0x0, 0x7FFF'FFFF'FFFF'FFFF))); - keys.push_back(create_key256(create_uint128(0x0, 0x0), create_uint128(0x0, 0x8000'0000'0000'0000))); - keys.push_back(create_key256(create_uint128(0x0, 0x0), create_uint128(0x0, 0xFFFF'FFFF'FFFF'FFFF))); - keys.push_back(create_key256(create_uint128(0x0, 0x0), create_uint128(0x1, 0x0))); - keys.push_back(create_key256(create_uint128(0x0, 0x0), create_uint128(0xFFFF'FFFF'FFFF'FFFF, 0x0))); - keys.push_back(create_key256(create_uint128(0x0, 0x1), create_uint128(0x0, 0x0))); - keys.push_back(create_key256(create_uint128(0x0, 0xFFFF'FFFF'FFFF'FFFF), create_uint128(0x0, 0x0))); - keys.push_back(create_key256(create_uint128(0x1, 0x0), create_uint128(0x0, 0x0))); - keys.push_back(create_key256(create_uint128(0xFFFF'FFFF'FFFF'FFFF, 0x0), create_uint128(0x0, 0x0))); - check_ordered_keys(keys, &eosio::chain::backing_store::db_key_value_format::create_secondary_key); -} - -BOOST_AUTO_TEST_CASE(float64_key_conversions_test) { - const name scope = "myscope"_n; - const name table = "thisscope"_n; - const float64_t key = to_softfloat64(6.0); - const uint64_t primary_key = 0x0123456789abcdef; - const auto composite_key = eosio::chain::backing_store::db_key_value_format::create_secondary_key(scope, table, key, primary_key); - BOOST_REQUIRE_EQUAL(legacy_overhead_size + sizeof(key) + sizeof(primary_key), composite_key.size()); - - name decomposed_scope; - name decomposed_table; - float64_t decomposed_key = to_softfloat64(0.0); - uint64_t decomposed_primary_key = 0; - BOOST_REQUIRE_NO_THROW(eosio::chain::backing_store::db_key_value_format::get_secondary_key(composite_key, decomposed_scope, decomposed_table, decomposed_key, decomposed_primary_key)); - BOOST_CHECK_EQUAL(scope.to_string(), decomposed_scope.to_string()); - BOOST_CHECK_EQUAL(table.to_string(), decomposed_table.to_string()); - - BOOST_CHECK_EQUAL(key, decomposed_key); - BOOST_CHECK_EQUAL(primary_key, decomposed_primary_key); - - verify_other_gets(composite_key, key_type::sec_double, scope, table); - - verify_secondary_key_pair(composite_key); - - std::vector keys; - keys.push_back(to_softfloat64(-100000.0)); - keys.push_back(to_softfloat64(-99999.9)); - keys.push_back(to_softfloat64(-1.9)); - keys.push_back(to_softfloat64(0.0)); - keys.push_back(to_softfloat64(0.1)); - keys.push_back(to_softfloat64(1.9)); - keys.push_back(to_softfloat64(99999.9)); - keys.push_back(to_softfloat64(100000.0)); - check_ordered_keys(keys, &eosio::chain::backing_store::db_key_value_format::create_secondary_key); -} - -BOOST_AUTO_TEST_CASE(float128_key_conversions_test) { - const name scope = "myscope"_n; - const name table = "thisscope"_n; - const float128_t key = to_softfloat128(99.99); - const uint64_t primary_key = 0x0123456789abcdef; - const auto composite_key = eosio::chain::backing_store::db_key_value_format::create_secondary_key(scope, table, key, primary_key); - BOOST_REQUIRE_EQUAL(legacy_overhead_size + sizeof(key) + sizeof(primary_key), composite_key.size()); - - name decomposed_scope; - name decomposed_table; - float128_t decomposed_key = to_softfloat128(0.0); - uint64_t decomposed_primary_key = 0; - BOOST_REQUIRE_NO_THROW(eosio::chain::backing_store::db_key_value_format::get_secondary_key(composite_key, decomposed_scope, decomposed_table, decomposed_key, decomposed_primary_key)); - BOOST_CHECK_EQUAL(scope.to_string(), decomposed_scope.to_string()); - BOOST_CHECK_EQUAL(table.to_string(), decomposed_table.to_string()); - - BOOST_CHECK_EQUAL(key, decomposed_key); - BOOST_CHECK_EQUAL(primary_key, decomposed_primary_key); - - verify_other_gets(composite_key, key_type::sec_long_double, scope, table); - - verify_secondary_key_pair(composite_key); - - std::vector keys; - keys.push_back(to_softfloat128(-100000.001)); - keys.push_back(to_softfloat128(-100000.0)); - keys.push_back(to_softfloat128(-99999.9)); - keys.push_back(to_softfloat128(-1.9)); - keys.push_back(to_softfloat128(0.0)); - keys.push_back(to_softfloat128(0.1)); - keys.push_back(to_softfloat128(1.9)); - keys.push_back(to_softfloat128(99999.9)); - keys.push_back(to_softfloat128(100000.0)); - keys.push_back(to_softfloat128(100000.001)); - check_ordered_keys(keys, &eosio::chain::backing_store::db_key_value_format::create_secondary_key); -} - -BOOST_AUTO_TEST_CASE(compare_negative_and_positive_f64_0_are_equal_test) { - std::vector composite_keys(2); - b1::chain_kv::append_key(composite_keys[0], to_softfloat64(-0.0)); - b1::chain_kv::append_key(composite_keys[1], to_softfloat64(0.0)); - // composite key representation should treat -0.0 and -0.0 as the same value - BOOST_CHECK_EQUAL(0, compare_composite_keys(composite_keys, 0).first); -} - -BOOST_AUTO_TEST_CASE(compare_negative_and_positive_f128_0_are_equal_test) { - std::vector composite_keys(2); - b1::chain_kv::append_key(composite_keys[0], to_softfloat128(-0.0)); - b1::chain_kv::append_key(composite_keys[1], to_softfloat128(0.0)); - // composite key representation should treat -0.0 and -0.0 as the same value - BOOST_CHECK_EQUAL(0, compare_composite_keys(composite_keys, 0).first); -} - -BOOST_AUTO_TEST_CASE(compare_span_of_doubles_test) { - std::vector composite_keys(4); - uint64_t next = 0; - const double neg_inf = -std::numeric_limits::infinity(); - b1::chain_kv::append_key(composite_keys[next++], to_softfloat64(neg_inf)); - const double min = std::numeric_limits::min(); - b1::chain_kv::append_key(composite_keys[next++], to_softfloat64(min)); - const double max = std::numeric_limits::max(); - b1::chain_kv::append_key(composite_keys[next++], to_softfloat64(max)); - const double inf = std::numeric_limits::infinity(); - b1::chain_kv::append_key(composite_keys[next++], to_softfloat64(inf)); - BOOST_CHECK_EQUAL(-1, compare_composite_keys(composite_keys, 0).first); - BOOST_CHECK_EQUAL(-1, compare_composite_keys(composite_keys, 1).first); - BOOST_CHECK_EQUAL(-1, compare_composite_keys(composite_keys, 2).first); -} - -BOOST_AUTO_TEST_CASE(compare_span_of_long_doubles_test) { - std::vector composite_keys(4); - uint64_t next = 0; - const double neg_inf = -std::numeric_limits::infinity(); - b1::chain_kv::append_key(composite_keys[next++], to_softfloat128(neg_inf)); - const double min = std::numeric_limits::min(); - b1::chain_kv::append_key(composite_keys[next++], to_softfloat128(min)); - const double max = std::numeric_limits::max(); - b1::chain_kv::append_key(composite_keys[next++], to_softfloat128(max)); - const double inf = std::numeric_limits::infinity(); - b1::chain_kv::append_key(composite_keys[next++], to_softfloat128(inf)); - BOOST_CHECK_EQUAL(-1, compare_composite_keys(composite_keys, 0).first); - BOOST_CHECK_EQUAL(-1, compare_composite_keys(composite_keys, 1).first); - BOOST_CHECK_EQUAL(-1, compare_composite_keys(composite_keys, 2).first); -} - -BOOST_AUTO_TEST_CASE(table_key_conversions_test) { - const name scope = "myscope"_n; - const name table = "thisscope"_n; - const auto composite_key = eosio::chain::backing_store::db_key_value_format::create_table_key(scope, table); - BOOST_REQUIRE_EQUAL(legacy_overhead_size, composite_key.size()); - - name decomposed_scope; - name decomposed_table; - BOOST_REQUIRE_NO_THROW(eosio::chain::backing_store::db_key_value_format::get_table_key(composite_key, decomposed_scope, decomposed_table)); - BOOST_CHECK_EQUAL(scope.to_string(), decomposed_scope.to_string()); - BOOST_CHECK_EQUAL(table.to_string(), decomposed_table.to_string()); - verify_other_gets(composite_key, key_type::table, scope, table); -} - -BOOST_AUTO_TEST_CASE(prefix_key_conversions_test) { - const name scope = "myscope"_n; - const name table = "thisscope"_n; - const auto composite_key = eosio::chain::backing_store::db_key_value_format::create_prefix_key(scope, table); - BOOST_REQUIRE_EQUAL(legacy_overhead_size_without_type, composite_key.size()); // - - name decomposed_scope; - name decomposed_table; - BOOST_REQUIRE_NO_THROW(eosio::chain::backing_store::db_key_value_format::get_prefix_key(composite_key, decomposed_scope, decomposed_table)); - BOOST_CHECK_EQUAL(scope.to_string(), decomposed_scope.to_string()); - BOOST_CHECK_EQUAL(table.to_string(), decomposed_table.to_string()); -} - -BOOST_AUTO_TEST_CASE(compare_key_type_order_test) { - std::vector composite_keys; - // enforce the order for the different key types with the same scope and table - const name scope; - const name table; - uint64_t next = 0; - const uint64_t max = std::numeric_limits::max(); - const eosio::chain::uint128_t max_i128 = create_uint128(max, max); - const eosio::chain::key256_t max_i256 = create_key256(create_uint128(max, max), create_uint128(max, max)); - const float64_t max_f64 = to_softfloat64(std::numeric_limits::max()); - const float128_t max_f128 = to_softfloat128(std::numeric_limits::max()); // close enough - composite_keys.push_back(eosio::chain::backing_store::db_key_value_format::create_prefix_key(scope, table)); - composite_keys.push_back(eosio::chain::backing_store::db_key_value_format::create_primary_key(scope, table, 0x0)); - BOOST_CHECK_EQUAL(-1, compare_composite_keys(composite_keys, next++).first); - composite_keys.push_back(eosio::chain::backing_store::db_key_value_format::create_primary_key(scope, table, max)); - BOOST_CHECK_EQUAL(-1, compare_composite_keys(composite_keys, next++).first); - composite_keys.push_back(eosio::chain::backing_store::db_key_value_format::create_secondary_key(scope, table, uint64_t(0x0), 0x0)); - BOOST_CHECK_EQUAL(-1, compare_composite_keys(composite_keys, next++).first); - composite_keys.push_back(eosio::chain::backing_store::db_key_value_format::create_secondary_key(scope, table, max, 0x0)); - BOOST_CHECK_EQUAL(-1, compare_composite_keys(composite_keys, next++).first); - composite_keys.push_back(eosio::chain::backing_store::db_key_value_format::create_secondary_key(scope, table, create_uint128(0x0, 0x0), 0x0)); - BOOST_CHECK_EQUAL(-1, compare_composite_keys(composite_keys, next++).first); - composite_keys.push_back(eosio::chain::backing_store::db_key_value_format::create_secondary_key(scope, table, max_i128, 0x0)); - BOOST_CHECK_EQUAL(-1, compare_composite_keys(composite_keys, next++).first); - composite_keys.push_back(eosio::chain::backing_store::db_key_value_format::create_secondary_key(scope, table, create_key256(create_uint128(0x0, 0x0), create_uint128(0x0, 0x0)), 0x0)); - BOOST_CHECK_EQUAL(-1, compare_composite_keys(composite_keys, next++).first); - composite_keys.push_back(eosio::chain::backing_store::db_key_value_format::create_secondary_key(scope, table, max_i256, 0x0)); - BOOST_CHECK_EQUAL(-1, compare_composite_keys(composite_keys, next++).first); - composite_keys.push_back(eosio::chain::backing_store::db_key_value_format::create_secondary_key(scope, table, to_softfloat64(0.0), 0x0)); - BOOST_CHECK_EQUAL(-1, compare_composite_keys(composite_keys, next++).first); - composite_keys.push_back(eosio::chain::backing_store::db_key_value_format::create_secondary_key(scope, table, max_f64, 0x0)); - BOOST_CHECK_EQUAL(-1, compare_composite_keys(composite_keys, next++).first); - composite_keys.push_back(eosio::chain::backing_store::db_key_value_format::create_secondary_key(scope, table, to_softfloat128(0.0), 0x0)); - BOOST_CHECK_EQUAL(-1, compare_composite_keys(composite_keys, next++).first); - composite_keys.push_back(eosio::chain::backing_store::db_key_value_format::create_secondary_key(scope, table, max_f128, 0x0)); - BOOST_CHECK_EQUAL(-1, compare_composite_keys(composite_keys, next++).first); - composite_keys.push_back(eosio::chain::backing_store::db_key_value_format::create_table_key(scope, table)); - BOOST_CHECK_EQUAL(-1, compare_composite_keys(composite_keys, next++).first); -} - -template -void verify_primary_exception(const b1::chain_kv::bytes& composite_key, const std::string& starts_with) { - uint64_t decomposed_key; - name decomposed_scope; - name decomposed_table; - BOOST_CHECK_EXCEPTION( - eosio::chain::backing_store::db_key_value_format::get_primary_key(composite_key, decomposed_scope, decomposed_table, decomposed_key), - Exception, - eosio::testing::fc_exception_message_starts_with(starts_with) - ); -} - -template -void verify_secondary_exception(const b1::chain_kv::bytes& composite_key, const std::string& starts_with) { - name decomposed_scope; - name decomposed_table; - Key decomposed_key = helper().init; - uint64_t decomposed_primary_key = 0; - BOOST_CHECK_EXCEPTION( - eosio::chain::backing_store::db_key_value_format::get_secondary_key(composite_key, decomposed_scope, decomposed_table, decomposed_key, decomposed_primary_key), - Exception, - eosio::testing::fc_exception_message_starts_with(starts_with) - ); -} - -BOOST_AUTO_TEST_CASE(verify_no_type_in_key_test) { - const name scope = "myscope"_n; - const name table = "thisscope"_n; - const auto composite_key = eosio::chain::backing_store::db_key_value_format::create_prefix_key(scope, table); - const std::string starts_with = "DB intrinsic key-value store composite key is malformed, it does not contain an indication of the type of the db-key"; - verify_primary_exception(composite_key, starts_with); - verify_secondary_exception(composite_key, starts_with); - verify_secondary_exception(composite_key, starts_with); - verify_secondary_exception(composite_key, starts_with); - verify_secondary_exception(composite_key, starts_with); - verify_secondary_exception(composite_key, starts_with); -} - -using namespace eosio::chain::backing_store; - -template -void verify_table(const T& table_cache, const unique_table& orig_table, int table_itr) { - auto cached_table = table_cache.find_table_by_end_iterator(table_itr); - BOOST_CHECK(&orig_table != cached_table); // verifying that table cache is creating its own objects - BOOST_CHECK(orig_table.contract == cached_table->contract); - BOOST_CHECK(orig_table.scope == cached_table->scope); - BOOST_CHECK(orig_table.table == cached_table->table); - unique_table table_copy = orig_table; - auto table_copy_iter = table_cache.get_end_iterator_by_table(table_copy); - BOOST_CHECK_EQUAL(table_itr, table_copy_iter); -} - -BOOST_AUTO_TEST_CASE(table_cache_test) { - db_key_value_iter_store i64_cache; - - const unique_table t1 = { name{0}, name{0}, name{0} }; - const auto t1_itr = i64_cache.cache_table(t1); - BOOST_CHECK_LT(t1_itr, i64_cache.invalid_iterator()); - verify_table(i64_cache, t1, t1_itr); - - const unique_table t2 = { name{0}, name{0}, name{1} }; - const auto t2_itr = i64_cache.cache_table(t2); - BOOST_CHECK_LT(t2_itr, i64_cache.invalid_iterator()); - verify_table(i64_cache, t2, t2_itr); - - const unique_table t3 = { name{0}, name{1}, name{0} }; - const auto t3_itr = i64_cache.cache_table(t3); - BOOST_CHECK_LT(t3_itr, i64_cache.invalid_iterator()); - verify_table(i64_cache, t3, t3_itr); - - const unique_table t4 = { name{1}, name{0}, name{0} }; - const auto t4_itr = i64_cache.cache_table(t4); - BOOST_CHECK_LT(t4_itr, i64_cache.invalid_iterator()); - verify_table(i64_cache, t4, t4_itr); - - BOOST_CHECK_NE(t1_itr, t2_itr); - BOOST_CHECK_NE(t1_itr, t3_itr); - BOOST_CHECK_NE(t1_itr, t4_itr); - BOOST_CHECK_NE(t2_itr, t3_itr); - BOOST_CHECK_NE(t2_itr, t4_itr); - BOOST_CHECK_NE(t3_itr, t4_itr); - - const unique_table t5 = { name{1}, name{0}, name{1} }; - BOOST_CHECK_THROW(i64_cache.get_end_iterator_by_table(t5), eosio::chain::table_not_in_cache); - BOOST_CHECK_THROW(i64_cache.find_table_by_end_iterator(i64_cache.invalid_iterator()), eosio::chain::invalid_table_iterator); - const auto non_existent_itr = t4_itr - 1; - BOOST_CHECK_EQUAL(nullptr, i64_cache.find_table_by_end_iterator(non_existent_itr)); -} - -BOOST_AUTO_TEST_CASE(invalid_itr_cache_test) { - db_key_value_iter_store i128_cache; - BOOST_CHECK_EXCEPTION( - i128_cache.get( i128_cache.invalid_iterator() ), eosio::chain::invalid_table_iterator, - eosio::testing::fc_exception_message_is( "invalid iterator" ) - ); - BOOST_CHECK_EXCEPTION( - i128_cache.get( i128_cache.invalid_iterator() - 1 ), eosio::chain::table_operation_not_permitted, - eosio::testing::fc_exception_message_is( "dereference of end iterator" ) - ); - BOOST_CHECK_EXCEPTION( - i128_cache.get( 0 ), eosio::chain::invalid_table_iterator, - eosio::testing::fc_exception_message_is( "iterator out of range" ) - ); - - BOOST_CHECK_EXCEPTION( - i128_cache.remove( i128_cache.invalid_iterator() ), eosio::chain::invalid_table_iterator, - eosio::testing::fc_exception_message_is( "invalid iterator" ) - ); - BOOST_CHECK_EXCEPTION( - i128_cache.remove( i128_cache.invalid_iterator() - 1 ), eosio::chain::table_operation_not_permitted, - eosio::testing::fc_exception_message_is( "cannot call remove on end iterators" ) - ); - BOOST_CHECK_EXCEPTION( - i128_cache.remove( 0 ), eosio::chain::invalid_table_iterator, - eosio::testing::fc_exception_message_is( "iterator out of range" ) - ); - - eosio::chain::uint128_t key = 0; - eosio::chain::account_name payer; - BOOST_CHECK_EXCEPTION( - i128_cache.swap( i128_cache.invalid_iterator(), key, payer ), eosio::chain::invalid_table_iterator, - eosio::testing::fc_exception_message_is( "invalid iterator" ) - ); - BOOST_CHECK_EXCEPTION( - i128_cache.swap( i128_cache.invalid_iterator() - 1, key, payer ), eosio::chain::table_operation_not_permitted, - eosio::testing::fc_exception_message_is( "cannot call swap on end iterators" ) - ); - BOOST_CHECK_EXCEPTION( - i128_cache.swap( 0, key, payer ), eosio::chain::invalid_table_iterator, - eosio::testing::fc_exception_message_is( "iterator out of range" ) - ); - - using i128_type = secondary_key; - const i128_type i128_1 = { .table_ei = i128_cache.invalid_iterator(), .secondary = 0x0, .primary = 0x0, .payer = name{} }; - BOOST_CHECK_EXCEPTION( - i128_cache.add( i128_1 ), eosio::chain::invalid_table_iterator, - eosio::testing::fc_exception_message_is( "not an end iterator" ) - ); - const i128_type i128_2 = { .table_ei = i128_cache.invalid_iterator() - 1, .secondary = 0x0, .primary = 0x0, .payer = name{} }; - BOOST_CHECK_EXCEPTION( - i128_cache.add( i128_2 ), eosio::chain::invalid_table_iterator, - eosio::testing::fc_exception_message_is( "an invariant was broken, table should be in cache" ) - ); -} - -template -void validate_get(const db_key_value_iter_store& cache, const secondary_key& orig, int itr, bool check_payer = true) { - const auto& obj_in_cache = cache.get(itr); - BOOST_CHECK_NE(&orig, &obj_in_cache); - BOOST_CHECK_EQUAL(obj_in_cache.table_ei, orig.table_ei); - BOOST_CHECK(obj_in_cache.secondary == orig.secondary); - BOOST_CHECK_EQUAL(obj_in_cache.primary, orig.primary); - const int found_itr = cache.find(orig); - BOOST_CHECK_EQUAL(itr, found_itr); - if (check_payer) { - BOOST_CHECK_EQUAL(obj_in_cache.payer, orig.payer); - } - else { - // invariant needed to ensure payer not required, if this fails either that has changed, or check_payer was incorrectly set to false - BOOST_CHECK_NE(obj_in_cache.payer, orig.payer); - } -} - -struct i64_values { - using key = uint64_t; - key sec_val_1 = 0x0; - key sec_val_2 = 0x1; - key sec_val_3 = 0x2; - key sec_val_4 = 0x3; -}; - -struct i128_values { - using key = eosio::chain::uint128_t; - key sec_val_1 = create_uint128(0x0, 0x0); - key sec_val_2 = create_uint128(0x0, 0x1); - key sec_val_3 = create_uint128(0x1, 0x0); - key sec_val_4 = create_uint128(0x1, 0x1); -}; - -using key_suite = boost::mpl::list; - -BOOST_AUTO_TEST_CASE_TEMPLATE(itr_cache_test, KEY_SUITE, key_suite) { - db_key_value_iter_store key_cache; - KEY_SUITE keys; - using key_type = typename db_key_value_iter_store::secondary_obj_type; - const unique_table t1 = { name{0}, name{0}, name{0} }; - const unique_table t2 = { name{0}, name{0}, name{1} }; - const auto t1_itr = key_cache.cache_table(t1); - const auto t2_itr = key_cache.cache_table(t2); - const bool dont_check_payer = false; - - const key_type key_1 = { .table_ei = t1_itr, .secondary = keys.sec_val_1, .primary = 0x0, .payer = name{0x0} }; - const auto key_1_itr = key_cache.add(key_1); - BOOST_CHECK_LT(key_cache.invalid_iterator(), key_1_itr); - validate_get(key_cache, key_1, key_1_itr); - - // verify that payer isn't part of uniqueness - const key_type key_1_diff_payer = { .table_ei = t1_itr, .secondary = keys.sec_val_1, .primary = 0x0, .payer = name{0x1} }; - validate_get(key_cache, key_1_diff_payer, key_1_itr, dont_check_payer); - - const key_type key_2 = { .table_ei = t1_itr, .secondary = keys.sec_val_1, .primary = 0x1, .payer = name{0x0} }; - const auto key_2_itr = key_cache.add(key_2); - BOOST_CHECK_LT(key_cache.invalid_iterator(), key_2_itr); - validate_get(key_cache, key_2, key_2_itr); - BOOST_CHECK_LT(key_1_itr, key_2_itr); - - const key_type key_3 = { .table_ei = t1_itr, .secondary = keys.sec_val_2, .primary = 0x0, .payer = name{0x0} }; - const auto key_3_itr = key_cache.add(key_3); - BOOST_CHECK_LT(key_cache.invalid_iterator(), key_3_itr); - validate_get(key_cache, key_3, key_3_itr); - BOOST_CHECK_LT(key_2_itr, key_3_itr); - - const key_type key_4 = { .table_ei = t2_itr, .secondary = keys.sec_val_1, .primary = 0x0, .payer = name{0x0} }; - const auto key_4_itr = key_cache.add(key_4); - BOOST_CHECK_LT(key_cache.invalid_iterator(), key_4_itr); - validate_get(key_cache, key_4, key_4_itr); - BOOST_CHECK_LT(key_3_itr, key_4_itr); - - key_cache.remove(key_1_itr); - BOOST_CHECK_EXCEPTION( - key_cache.get(key_1_itr), eosio::chain::table_operation_not_permitted, - eosio::testing::fc_exception_message_is( "dereference of deleted object" ) - ); - // verify remove of an iterator that previously existed is fine - key_cache.remove(key_1_itr); - // verify swap of an iterator that previously existed is fine - key_cache.swap(key_1_itr, keys.sec_val_3, name{0x2}); - // validate that the repeat of key_1 w/ different payer can be added now - const auto key_1_diff_payer_itr = key_cache.add(key_1_diff_payer); - BOOST_CHECK_LT(key_4_itr, key_1_diff_payer_itr); - validate_get(key_cache, key_1_diff_payer, key_1_diff_payer_itr); - - const key_type key_2_swap_payer = { .table_ei = key_2.table_ei, .secondary = key_2.secondary, .primary = key_2.primary, .payer = name{0x2} }; - key_cache.swap(key_2_itr, key_2.secondary, key_2_swap_payer.payer); // same key, different payer - validate_get(key_cache, key_2_swap_payer, key_2_itr); - const auto key_2_swap_payer_itr = key_cache.add(key_2_swap_payer); - BOOST_CHECK_EQUAL(key_2_itr, key_2_swap_payer_itr); - const key_type key_2_swap_key = { .table_ei = key_2.table_ei, .secondary = keys.sec_val_3, .primary = key_2.primary, .payer = key_2_swap_payer.payer }; - key_cache.swap(key_2_itr, key_2_swap_key.secondary, key_2_swap_key.payer); // same key, different payer - validate_get(key_cache, key_2_swap_key, key_2_itr); - const key_type key_2_swap_both = { .table_ei = key_2.table_ei, .secondary = keys.sec_val_4, .primary = key_2.primary, .payer = name{0x3} }; - key_cache.swap(key_2_itr, key_2_swap_both.secondary, key_2_swap_both.payer); // same key, different payer - validate_get(key_cache, key_2_swap_both, key_2_itr); - const auto key_2_reload_itr = key_cache.add(key_2); - BOOST_CHECK_LT(key_1_diff_payer_itr, key_2_reload_itr); - key_cache.remove(key_2_itr); - key_cache.swap(key_2_itr, key_2.secondary, key_2_swap_payer.payer); // same key, different payer -} - - -BOOST_AUTO_TEST_SUITE_END() diff --git a/unittests/kv_tests.cpp b/unittests/kv_tests.cpp index 7f16bd1e1a..81e83cfdb7 100644 --- a/unittests/kv_tests.cpp +++ b/unittests/kv_tests.cpp @@ -621,7 +621,6 @@ class kv_tester : public tester { } // Make sure that a failed transaction correctly rolls back changes to the database, - // for both rocksdb and chainbase. void test_undo() { setmany("", "kvtest"_n, { @@ -695,11 +694,6 @@ class kv_chainbase_tester : public kv_tester { kv_chainbase_tester() : kv_tester(backing_store_type::CHAINBASE) { } }; -class kv_rocksdb_tester : public kv_tester { - public: - kv_rocksdb_tester() : kv_tester(backing_store_type::ROCKSDB) { } -}; - BOOST_AUTO_TEST_SUITE(kv_tests) BOOST_FIXTURE_TEST_CASE(kv_basic, kv_chainbase_tester) try { @@ -774,88 +768,6 @@ BOOST_FIXTURE_TEST_CASE(max_iterators, kv_chainbase_tester) try { // } FC_LOG_AND_RETHROW() -// RocksDB related test cases - -BOOST_FIXTURE_TEST_CASE(kv_basic_rocksdb, kv_rocksdb_tester) try { - test_kv_basic_common(); -} -FC_LOG_AND_RETHROW() - -BOOST_FIXTURE_TEST_CASE(kv_scan_rocksdb, kv_rocksdb_tester) try { - test_kv_scan_common(); -} -FC_LOG_AND_RETHROW() - -BOOST_FIXTURE_TEST_CASE(kv_scanrev_rocksdb, kv_rocksdb_tester) try { - test_kv_scanrev_common(); -} -FC_LOG_AND_RETHROW() - -BOOST_FIXTURE_TEST_CASE(kv_scanrev2_rocksdb, kv_rocksdb_tester) try { // - test_kv_scanrev2_common(); -} -FC_LOG_AND_RETHROW() - -BOOST_FIXTURE_TEST_CASE(kv_iterase_rocksdb, kv_rocksdb_tester) try { // - test_kv_iterase_common(); -} -FC_LOG_AND_RETHROW() - -BOOST_FIXTURE_TEST_CASE(kv_ram_usage_rocksdb, kv_rocksdb_tester) try { // - test_kv_ram_usage_common(); -} -FC_LOG_AND_RETHROW() - -BOOST_FIXTURE_TEST_CASE(kv_resource_limit_rocksdb, kv_rocksdb_tester) try { // - test_kv_resource_limit_common(); -} -FC_LOG_AND_RETHROW() - -BOOST_FIXTURE_TEST_CASE(kv_key_value_limit_rocksdb, kv_rocksdb_tester) try { // - test_kv_key_value_limit_common(); -} -FC_LOG_AND_RETHROW() - -BOOST_DATA_TEST_CASE_F(kv_rocksdb_tester, kv_inc_dec_usage_rocksdb, bdata::make(databases), db) try { // - test_kv_inc_dec_usage(); -} -FC_LOG_AND_RETHROW() - -BOOST_DATA_TEST_CASE_F(kv_rocksdb_tester, kv_inc_usage_and_limit_rocksdb, bdata::make(databases), db) try { // - test_kv_inc_usage_and_limit(); -} -FC_LOG_AND_RETHROW() - -BOOST_DATA_TEST_CASE_F(kv_rocksdb_tester, kv_dec_limit_and_usage_rocksdb, bdata::make(databases), db) try { // - test_kv_dec_limit_and_usage(); -} -FC_LOG_AND_RETHROW() - -BOOST_DATA_TEST_CASE_F(kv_rocksdb_tester, get_data_rocksdb, bdata::make(databases), db) try { // - test_get_data(); -} -FC_LOG_AND_RETHROW() - -BOOST_DATA_TEST_CASE_F(kv_rocksdb_tester, other_contract_rocksdb, bdata::make(databases), db) try { // - test_other_contract(); -} -FC_LOG_AND_RETHROW() - -BOOST_FIXTURE_TEST_CASE(max_iterators_rocksdb, kv_rocksdb_tester) try { // - test_max_iterators(); -} -FC_LOG_AND_RETHROW() - -BOOST_DATA_TEST_CASE_F(kv_chainbase_tester, undo, bdata::make(databases) * bdata::make({false, true}), db, rocks) try { // - test_undo(); -} -FC_LOG_AND_RETHROW() - -BOOST_DATA_TEST_CASE_F(kv_rocksdb_tester, rocksdb_undo, bdata::make(databases) * bdata::make({false, true}), db, rocks) try { // - test_undo(); -} -FC_LOG_AND_RETHROW() - // Initializes a kv_database configuration to usable values. // Must be set on a privileged account static const char kv_setup_wast[] = R"=====( diff --git a/unittests/snapshot_tests.cpp b/unittests/snapshot_tests.cpp index bed481db8f..5849434182 100644 --- a/unittests/snapshot_tests.cpp +++ b/unittests/snapshot_tests.cpp @@ -253,24 +253,6 @@ BOOST_AUTO_TEST_CASE_TEMPLATE(test_exhaustive_snapshot_cb_to_cb, SNAPSHOT_SUITE, eosio::chain::backing_store_type::CHAINBASE); } -BOOST_AUTO_TEST_CASE_TEMPLATE(test_exhaustive_snapshot_cb_to_rdb, SNAPSHOT_SUITE, snapshot_suites) -{ - exhaustive_snapshot(eosio::chain::backing_store_type::CHAINBASE, - eosio::chain::backing_store_type::ROCKSDB); -} - -BOOST_AUTO_TEST_CASE_TEMPLATE(test_exhaustive_snapshot_rdb_to_cb, SNAPSHOT_SUITE, snapshot_suites) -{ - exhaustive_snapshot(eosio::chain::backing_store_type::ROCKSDB, - eosio::chain::backing_store_type::CHAINBASE); -} - -BOOST_AUTO_TEST_CASE_TEMPLATE(test_exhaustive_snapshot_rdb_to_rdb, SNAPSHOT_SUITE, snapshot_suites) -{ - exhaustive_snapshot(eosio::chain::backing_store_type::ROCKSDB, - eosio::chain::backing_store_type::ROCKSDB); -} - BOOST_AUTO_TEST_CASE_TEMPLATE(test_replay_over_snapshot, SNAPSHOT_SUITE, snapshot_suites) { tester chain; @@ -656,8 +638,8 @@ static const char kv_snapshot_bios[] = R"=====( )====="; BOOST_AUTO_TEST_CASE_TEMPLATE(test_kv_snapshot, SNAPSHOT_SUITE, snapshot_suites) { - for (backing_store_type origin_backing_store : { backing_store_type::CHAINBASE, backing_store_type::ROCKSDB }) { - for (backing_store_type resulting_backing_store: { backing_store_type::CHAINBASE, backing_store_type::ROCKSDB }) { + for (backing_store_type origin_backing_store : { backing_store_type::CHAINBASE }) { + for (backing_store_type resulting_backing_store: { backing_store_type::CHAINBASE }) { tester chain {setup_policy::full, db_read_mode::SPECULATIVE, std::optional{}, std::optional{}, origin_backing_store}; chain.create_accounts({"manager"_n}); diff --git a/unittests/state_history_tests.cpp b/unittests/state_history_tests.cpp index 81ffcb2820..181d4b5ba0 100644 --- a/unittests/state_history_tests.cpp +++ b/unittests/state_history_tests.cpp @@ -677,7 +677,7 @@ class table_deltas_tester : public tester { }; BOOST_AUTO_TEST_CASE(test_deltas_not_empty) { - for (backing_store_type backing_store : { backing_store_type::CHAINBASE/* TODO: uncomment this , backing_store_type::ROCKSDB*/ } ) { + for (backing_store_type backing_store : { backing_store_type::CHAINBASE } ) { table_deltas_tester chain { backing_store }; auto deltas = eosio::state_history::create_deltas(chain.control->kv_db(), false); @@ -689,7 +689,7 @@ BOOST_AUTO_TEST_CASE(test_deltas_not_empty) { } BOOST_AUTO_TEST_CASE(test_deltas_account_creation) { - for (backing_store_type backing_store : { backing_store_type::CHAINBASE, backing_store_type::ROCKSDB }) { + for (backing_store_type backing_store : { backing_store_type::CHAINBASE }) { table_deltas_tester chain { backing_store }; chain.produce_block(); @@ -710,7 +710,7 @@ BOOST_AUTO_TEST_CASE(test_deltas_account_creation) { } BOOST_AUTO_TEST_CASE(test_deltas_account_metadata) { - for (backing_store_type backing_store : { backing_store_type::CHAINBASE, backing_store_type::ROCKSDB }) { + for (backing_store_type backing_store : { backing_store_type::CHAINBASE }) { table_deltas_tester chain { backing_store }; chain.produce_block(); @@ -729,7 +729,7 @@ BOOST_AUTO_TEST_CASE(test_deltas_account_metadata) { BOOST_AUTO_TEST_CASE(test_deltas_account_permission) { - for (backing_store_type backing_store : { backing_store_type::CHAINBASE, backing_store_type::ROCKSDB }) { + for (backing_store_type backing_store : { backing_store_type::CHAINBASE }) { table_deltas_tester chain { backing_store }; chain.produce_block(); @@ -751,7 +751,7 @@ BOOST_AUTO_TEST_CASE(test_deltas_account_permission) { } BOOST_AUTO_TEST_CASE(test_deltas_account_permission_creation_and_deletion) { - for (backing_store_type backing_store : { backing_store_type::CHAINBASE, backing_store_type::ROCKSDB }) { + for (backing_store_type backing_store : { backing_store_type::CHAINBASE }) { table_deltas_tester chain { backing_store }; chain.produce_block(); @@ -797,7 +797,7 @@ BOOST_AUTO_TEST_CASE(test_deltas_account_permission_creation_and_deletion) { } BOOST_AUTO_TEST_CASE(test_deltas_account_permission_modification) { - for (backing_store_type backing_store : { backing_store_type::CHAINBASE, backing_store_type::ROCKSDB }) { + for (backing_store_type backing_store : { backing_store_type::CHAINBASE }) { table_deltas_tester chain { backing_store }; chain.produce_block(); @@ -837,7 +837,7 @@ BOOST_AUTO_TEST_CASE(test_deltas_account_permission_modification) { } BOOST_AUTO_TEST_CASE(test_deltas_permission_link) { - for (backing_store_type backing_store : { backing_store_type::CHAINBASE, backing_store_type::ROCKSDB }) { + for (backing_store_type backing_store : { backing_store_type::CHAINBASE }) { table_deltas_tester chain { backing_store }; chain.produce_block(); @@ -863,7 +863,7 @@ BOOST_AUTO_TEST_CASE(test_deltas_permission_link) { } BOOST_AUTO_TEST_CASE(test_deltas_global_property_history) { - for (backing_store_type backing_store : { backing_store_type::CHAINBASE, backing_store_type::ROCKSDB }) { + for (backing_store_type backing_store : { backing_store_type::CHAINBASE }) { // Assuming max transaction delay is 45 days (default in config.hpp) table_deltas_tester chain { backing_store }; @@ -884,7 +884,7 @@ BOOST_AUTO_TEST_CASE(test_deltas_global_property_history) { } BOOST_AUTO_TEST_CASE(test_deltas_protocol_feature_history) { - for (backing_store_type backing_store : { backing_store_type::CHAINBASE, backing_store_type::ROCKSDB }) { + for (backing_store_type backing_store : { backing_store_type::CHAINBASE }) { table_deltas_tester chain { backing_store, setup_policy::none }; const auto &pfm = chain.control->get_protocol_feature_manager(); @@ -919,7 +919,7 @@ BOOST_AUTO_TEST_CASE(test_deltas_protocol_feature_history) { } BOOST_AUTO_TEST_CASE(test_deltas_kv) { - for (backing_store_type backing_store : { backing_store_type::CHAINBASE, backing_store_type::ROCKSDB }) { + for (backing_store_type backing_store : { backing_store_type::CHAINBASE }) { table_deltas_tester chain { backing_store }; chain.produce_blocks(2); @@ -980,7 +980,7 @@ BOOST_AUTO_TEST_CASE(test_deltas_kv) { } BOOST_AUTO_TEST_CASE(test_deltas_contract) { - for (backing_store_type backing_store : { backing_store_type::CHAINBASE, backing_store_type::ROCKSDB }) { + for (backing_store_type backing_store : { backing_store_type::CHAINBASE }) { table_deltas_tester chain { backing_store, setup_policy::none }; chain.produce_block(); @@ -1125,7 +1125,7 @@ BOOST_AUTO_TEST_CASE(test_deltas_contract) { } BOOST_AUTO_TEST_CASE(test_deltas_contract_several_rows){ - for (backing_store_type backing_store : { backing_store_type::CHAINBASE, backing_store_type::ROCKSDB }) { + for (backing_store_type backing_store : { backing_store_type::CHAINBASE }) { table_deltas_tester chain { backing_store, setup_policy::none }; chain.produce_block(); @@ -1302,7 +1302,7 @@ BOOST_AUTO_TEST_CASE(test_deltas_table_and_kv) { } BOOST_AUTO_TEST_CASE(test_deltas_resources_history) { - for (backing_store_type backing_store : { backing_store_type::CHAINBASE, backing_store_type::ROCKSDB }) { + for (backing_store_type backing_store : { backing_store_type::CHAINBASE }) { table_deltas_tester chain { backing_store }; chain.produce_block(); From 11e66ed374645024423ee5706c1c8a69fdfc0b23 Mon Sep 17 00:00:00 2001 From: Dmytro Sydorchenko Date: Mon, 24 Jan 2022 21:48:49 -0500 Subject: [PATCH 56/62] refactoring combined_database -> db_util --- libraries/chain/CMakeLists.txt | 2 +- libraries/chain/apply_context.cpp | 4 +- libraries/chain/controller.cpp | 45 ++-- .../chain/{kv_database.cpp => db_util.cpp} | 247 ++++++++---------- .../chain/include/eosio/chain/controller.hpp | 8 +- .../chain/include/eosio/chain/db_util.hpp | 97 +++++++ .../chain/include/eosio/chain/kv_database.hpp | 130 --------- .../eosio/chain/transaction_context.hpp | 4 +- libraries/chain/transaction_context.cpp | 4 +- libraries/state_history/create_deltas.cpp | 8 - .../eosio/state_history/create_deltas.hpp | 4 +- .../include/eosio/state_history/log.hpp | 4 +- libraries/state_history/log.cpp | 2 +- plugins/chain_plugin/chain_plugin.cpp | 5 +- .../eosio/chain_plugin/chain_plugin.hpp | 2 +- .../state_history_plugin.cpp | 2 +- .../eosio/trace_api/store_provider.hpp | 2 +- programs/eosio-tester/main.cpp | 2 +- unittests/state_history_tests.cpp | 10 +- 19 files changed, 251 insertions(+), 331 deletions(-) rename libraries/chain/{kv_database.cpp => db_util.cpp} (71%) create mode 100644 libraries/chain/include/eosio/chain/db_util.hpp delete mode 100644 libraries/chain/include/eosio/chain/kv_database.hpp diff --git a/libraries/chain/CMakeLists.txt b/libraries/chain/CMakeLists.txt index 79dda80c34..1710598a9f 100644 --- a/libraries/chain/CMakeLists.txt +++ b/libraries/chain/CMakeLists.txt @@ -93,7 +93,7 @@ add_library( eosio_chain block_header_state.cpp block_state.cpp fork_database.cpp - kv_database.cpp + db_util.cpp controller.cpp authorization_manager.cpp resource_limits.cpp diff --git a/libraries/chain/apply_context.cpp b/libraries/chain/apply_context.cpp index 741a5a5c55..37c14cc6cb 100644 --- a/libraries/chain/apply_context.cpp +++ b/libraries/chain/apply_context.cpp @@ -51,7 +51,7 @@ apply_context::apply_context(controller& con, transaction_context& trx_ctx, uint act = &trace.act; receiver = trace.receiver; context_free = trace.context_free; - _db_context = control.kv_db().create_db_context(*this, receiver); + _db_context = db_util::create_db_context(*this, receiver); } template @@ -104,7 +104,7 @@ void apply_context::exec_one() kv_iterators.resize(1); kv_destroyed_iterators.clear(); if (!context_free) { - kv_backing_store = control.kv_db().create_kv_context(receiver, create_kv_resource_manager(*this), control.get_global_properties().kv_configuration); + kv_backing_store = db_util::create_kv_context(control.mutable_db(), receiver, create_kv_resource_manager(*this), control.get_global_properties().kv_configuration); } receiver_account = &db.get( receiver ); if( !(context_free && control.skip_trx_checks()) ) { diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index a4b7202511..90193ccaf0 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -31,6 +31,8 @@ namespace eosio { namespace chain { using resource_limits::resource_limits_manager; +using namespace db_util; + struct building_block { building_block( const block_header_state& prev, block_timestamp_type when, @@ -68,7 +70,7 @@ struct completed_block { using block_stage_type = std::variant; struct pending_state { - pending_state( kv_session&& s, const block_header_state& prev, + pending_state( maybe_session&& s, const block_header_state& prev, block_timestamp_type when, uint16_t num_prev_blocks_to_confirm, const vector& new_protocol_feature_activations ) @@ -76,7 +78,7 @@ struct pending_state { ,_block_stage( building_block( prev, when, num_prev_blocks_to_confirm, new_protocol_feature_activations ) ) {} - kv_session _db_session; + maybe_session _db_session; block_stage_type _block_stage; controller::block_status _block_status = controller::block_status::incomplete; std::optional _producer_block_id; @@ -162,7 +164,6 @@ struct controller_impl { controller& self; std::function shutdown; chainbase::database db; - kv_database kv_db; block_log blog; std::optional pending; block_state_ptr head; @@ -204,7 +205,7 @@ struct controller_impl { head = prev; - kv_db.undo(); + db.undo(); protocol_features.popped_blocks_to( prev->block_num ); } @@ -234,7 +235,6 @@ struct controller_impl { db( cfg.state_dir, cfg.read_only ? database::read_only : database::read_write, cfg.state_size, false, cfg.db_map_mode ), - kv_db(kv_database(db)), blog( cfg.blog ), fork_db( cfg.blog.log_dir / config::reversible_blocks_dir_name ), wasmif( cfg.wasm_runtime, cfg.eosvmoc_tierup, db, cfg.state_dir, cfg.eosvmoc_config, !cfg.profile_accounts.empty() ), @@ -368,7 +368,7 @@ struct controller_impl { blog.append( std::move( *it ) ); ++it; - kv_db.commit( (*bitr)->block_num ); + db.commit( (*bitr)->block_num ); root_id = (*bitr)->id; } } catch( std::exception& ) { @@ -409,7 +409,7 @@ struct controller_impl { static_cast(*head) = genheader; head->activated_protocol_features = std::make_shared(); head->block = std::make_shared(genheader.header); - kv_db.set_revision( head->block_num ); + db.set_revision( head->block_num ); initialize_database(genesis); } @@ -464,7 +464,7 @@ struct controller_impl { // if the irreverible log is played without undo sessions enabled, we need to sync the // revision ordinal to the appropriate expected value here. if( self.skip_db_sessions( controller::block_status::irreversible ) ) - kv_db.set_revision( head->block_num ); + db.set_revision( head->block_num ); } else { ilog( "no irreversible blocks need to be replayed" ); } @@ -499,18 +499,16 @@ struct controller_impl { void startup(std::function shutdown, std::function check_shutdown, const snapshot_reader_ptr& snapshot) { EOS_ASSERT( snapshot, snapshot_exception, "No snapshot reader provided" ); - EOS_ASSERT( db.revision() == kv_db.revision(), database_revision_mismatch_exception, - "chainbase is at revision ${a}, but chain-kv is at revision ${b}", ("a", db.revision())("b", kv_db.revision()) ); this->shutdown = shutdown; ilog( "Starting initialization from snapshot, this may take a significant amount of time" ); try { snapshot->validate(); if( blog.head() ) { - kv_db.read_from_snapshot( snapshot, blog.first_block_num(), blog.head()->block_num(), + read_from_snapshot( db, snapshot, blog.first_block_num(), blog.head()->block_num(), authorization, resource_limits, head, snapshot_head_block, chain_id ); } else { - kv_db.read_from_snapshot( snapshot, 0, std::numeric_limits::max(), + read_from_snapshot( db, snapshot, 0, std::numeric_limits::max(), authorization, resource_limits, head, snapshot_head_block, chain_id ); const uint32_t lib_num = head->block_num; @@ -533,8 +531,6 @@ struct controller_impl { void startup(std::function shutdown, std::function check_shutdown, const genesis_state& genesis) { EOS_ASSERT( db.revision() < 1, database_exception, "This version of controller::startup only works with a fresh state database." ); - EOS_ASSERT( db.revision() == kv_db.revision(), database_revision_mismatch_exception, - "chainbase is at revision ${a}, but chain-kv is at revision ${b}", ("a", db.revision())("b", kv_db.revision()) ); const auto& genesis_chain_id = genesis.compute_chain_id(); EOS_ASSERT( genesis_chain_id == chain_id, chain_id_type_exception, "genesis state provided to startup corresponds to a chain ID (${genesis_chain_id}) that does not match the chain ID that controller was constructed with (${controller_chain_id})", @@ -568,8 +564,6 @@ struct controller_impl { void startup(std::function shutdown, std::function check_shutdown) { EOS_ASSERT( db.revision() >= 1, database_exception, "This version of controller::startup does not work with a fresh state database." ); - EOS_ASSERT( db.revision() == kv_db.revision(), database_revision_mismatch_exception, - "chainbase is at revision ${a}, but chain-kv is at revision ${b}", ("a", db.revision())("b", kv_db.revision()) ); EOS_ASSERT( fork_db.head(), fork_database_exception, "No existing fork database despite existing chain state. Replay required." ); this->shutdown = shutdown; @@ -635,7 +629,7 @@ struct controller_impl { }); } - kv_db.check_backing_store_setting( clean_startup ); + check_backing_store_setting( db, clean_startup ); // At this point head != nullptr EOS_ASSERT( db.revision() >= head->block_num, fork_database_exception, @@ -648,7 +642,7 @@ struct controller_impl { ("db",db.revision())("head",head->block_num) ); } while( db.revision() > head->block_num ) { - kv_db.undo(); + db.undo(); } protocol_features.init( db ); @@ -714,7 +708,7 @@ struct controller_impl { sha256 calculate_integrity_hash() const { sha256::encoder enc; auto hash_writer = std::make_shared(enc); - kv_db.add_to_snapshot(hash_writer, *head, authorization, resource_limits); + add_to_snapshot(db, hash_writer, *head, authorization, resource_limits); hash_writer->finalize(); return enc.result(); @@ -974,7 +968,7 @@ struct controller_impl { const bool validating = !self.is_producing_block(); EOS_ASSERT( !validating || explicit_billed_cpu_time, transaction_exception, "validating requires explicit billing" ); - kv_session undo_session = !self.skip_db_sessions() ? kv_db.make_session() : kv_db.make_no_op_session(); + maybe_session undo_session = !self.skip_db_sessions() ? maybe_session(db) : maybe_session(); auto gtrx = generated_transaction(gto); @@ -1347,9 +1341,9 @@ struct controller_impl { EOS_ASSERT( db.revision() == head->block_num, database_exception, "db revision is not on par with head block", ("db.revision()", db.revision())("controller_head_block", head->block_num)("fork_db_head_block", fork_db.head()->block_num) ); - pending.emplace( kv_db.make_session(), *head, when, confirm_block_count, new_protocol_feature_activations ); + pending.emplace( maybe_session(db), *head, when, confirm_block_count, new_protocol_feature_activations ); } else { - pending.emplace( kv_db.make_no_op_session(), *head, when, confirm_block_count, new_protocol_feature_activations ); + pending.emplace( maybe_session(), *head, when, confirm_block_count, new_protocol_feature_activations ); } pending->_block_status = s; @@ -1900,7 +1894,7 @@ struct controller_impl { emit( self.irreversible_block, bsp ); if (!self.skip_db_sessions(s)) { - kv_db.commit(bsp->block_num); + db.commit(bsp->block_num); } } else { @@ -2319,8 +2313,6 @@ controller::~controller() { //for that we need 'my' to be valid pointer pointing to valid controller_impl. my->fork_db.close(); */ - - my->kv_db.flush(); } void controller::add_indices() { @@ -2344,7 +2336,6 @@ const chainbase::database& controller::db()const { return my->db; } chainbase::database& controller::mutable_db()const { return my->db; } const fork_database& controller::fork_db()const { return my->fork_db; } -eosio::chain::kv_database& controller::kv_db() const { return my->kv_db; } void controller::preactivate_feature( uint32_t action_id, const digest_type& feature_digest ) { const auto& pfs = my->protocol_features.get_protocol_feature_set(); @@ -2792,7 +2783,7 @@ sha256 controller::calculate_integrity_hash()const { try { void controller::write_snapshot( const snapshot_writer_ptr& snapshot ) const { EOS_ASSERT( !my->pending, block_validate_exception, "cannot take a consistent snapshot with a pending block" ); - return my->kv_db.add_to_snapshot(snapshot, *my->fork_db.head(), my->authorization, my->resource_limits); + return add_to_snapshot(my->db, snapshot, *my->fork_db.head(), my->authorization, my->resource_limits); } int64_t controller::set_proposed_producers( vector producers ) { diff --git a/libraries/chain/kv_database.cpp b/libraries/chain/db_util.cpp similarity index 71% rename from libraries/chain/kv_database.cpp rename to libraries/chain/db_util.cpp index 0851352c31..2565689540 100644 --- a/libraries/chain/kv_database.cpp +++ b/libraries/chain/db_util.cpp @@ -1,32 +1,31 @@ #include -#include +#include #include #include -namespace eosio { namespace chain { - kv_session::kv_session(chainbase::database& cb_database) - { +namespace eosio { namespace chain { namespace db_util { + maybe_session::maybe_session(chainbase::database& cb_database) { cb_session = std::make_unique(cb_database.start_undo_session(true)); } - kv_session::kv_session(kv_session&& src) noexcept + maybe_session::maybe_session(maybe_session&& src) noexcept : cb_session(std::move(src.cb_session)) { } - void kv_session::push() { + void maybe_session::push() { if (cb_session) { cb_session->push(); cb_session = nullptr; } } - void kv_session::squash() { + void maybe_session::squash() { if (cb_session) { cb_session->squash(); cb_session = nullptr; } } - void kv_session::undo() { + void maybe_session::undo() { if (cb_session) { cb_session->undo(); cb_session = nullptr; @@ -37,16 +36,7 @@ namespace eosio { namespace chain { void walk_index(const Util& utils, const chainbase::database& db, F&& function) { utils.walk(db, std::forward(function)); } - - template - void walk_index(const index_utils& utils, const chainbase::database& db, F&& function) {} - - template - void walk_index(const index_utils& utils, const chainbase::database& db, F&& function) {} - - template - void walk_index(const index_utils& utils, const chainbase::database& db, F&& function) {} - + void add_kv_table_to_snapshot(const snapshot_writer_ptr& snapshot, const chainbase::database& db) { snapshot->write_section([&db](auto& section) { index_utils utils; @@ -65,18 +55,15 @@ namespace eosio { namespace chain { }); } - kv_database::kv_database(chainbase::database& chain_db) - : backing_store(backing_store_type::CHAINBASE), db(chain_db) {} - - void kv_database::check_backing_store_setting(bool clean_startup) { - if (backing_store != db.get().backing_store) { + void check_backing_store_setting(chainbase::database& db, bool clean_startup) { + if (backing_store_type::CHAINBASE != db.get().backing_store) { EOS_ASSERT(clean_startup, database_move_kv_disk_exception, "Existing state indicates a different backing store is in use; use resync, replay, or restore from snapshot to switch backing store"); - db.modify(db.get(), [this](auto& cfg) { cfg.backing_store = backing_store; }); + db.modify(db.get(), [](auto& cfg) { cfg.backing_store = backing_store_type::CHAINBASE; }); } } - void kv_database::destroy(const fc::path& p) { + void destroy(const fc::path& p) { if( !fc::is_directory( p ) ) return; @@ -84,88 +71,140 @@ namespace eosio { namespace chain { fc::remove( p / "shared_memory.meta" ); } - void kv_database::set_revision(uint64_t revision) { - db.set_revision(revision); + std::unique_ptr create_kv_context(chainbase::database& db, + name receiver, + kv_resource_manager resource_manager, + const kv_database_config& limits) { + return create_kv_chainbase_context(db, receiver, resource_manager, limits); } - int64_t kv_database::revision() { - return db.revision(); + std::unique_ptr create_db_context(apply_context& context, + name receiver) { + return backing_store::create_db_chainbase_context(context, receiver); } - void kv_database::undo() { - db.undo(); - } + template + void chainbase_add_contract_tables_to_snapshot(const chainbase::database& db, Section& section) { + index_utils::walk(db, [&db, §ion](const table_id_object& table_row) { + // add a row for the table + section.add_row(table_row, db); - void kv_database::commit(int64_t revision) { - db.commit(revision); - } + // followed by a size row and then N data rows for each type of table + contract_database_index_set::walk_indices([&db, §ion, &table_row](auto utils) { + using utils_t = decltype(utils); + using value_t = typename utils_t::index_t::value_type; + using by_table_id = object_to_table_id_tag_t; + + auto tid_key = std::make_tuple(table_row.id); + auto next_tid_key = std::make_tuple(table_id_object::id_type(table_row.id._id + 1)); + + unsigned_int size = utils_t::template size_range(db, tid_key, next_tid_key); + section.add_row(size, db); - void kv_database::flush() { + utils_t::template walk_range(db, tid_key, next_tid_key, [&db, §ion](const auto& row) { + section.add_row(row, db); + }); + }); + }); } - std::unique_ptr kv_database::create_kv_context(name receiver, kv_resource_manager resource_manager, - const kv_database_config& limits) const { - return create_kv_chainbase_context(db, receiver, resource_manager, limits); + template + void chainbase_read_contract_tables_from_snapshot(chainbase::database& db, Section& section) { + bool more = !section.empty(); + while (more) { + // read the row for the table + table_id_object::id_type t_id; + + index_utils::create(db, [&db, §ion, &t_id](auto& row) { + section.read_row(row, db); + t_id = row.id; + }); + + // read the size and data rows for each type of table + contract_database_index_set::walk_indices([&db, §ion, &t_id, &more](auto utils) { + using utils_t = decltype(utils); + + unsigned_int size; + more = section.read_row(size, db); + + for (size_t idx = 0; idx < size.value; ++idx) { + utils_t::create(db, [&db, §ion, &more, &t_id](auto& row) { + row.t_id = t_id; + more = section.read_row(row, db); + }); + } + }); + } } - std::unique_ptr kv_database::create_db_context(apply_context& context, name receiver) { - return backing_store::create_db_chainbase_context(context, receiver); + void add_contract_tables_to_snapshot(const chainbase::database& db, const snapshot_writer_ptr& snapshot) { + snapshot->write_section("contract_tables", [&](auto& section) { + chainbase_add_contract_tables_to_snapshot(db, section); + }); + } + void read_contract_tables_from_snapshot(chainbase::database& db, const snapshot_reader_ptr& snapshot) { + snapshot->read_section("contract_tables", [&](auto& section) { + chainbase_read_contract_tables_from_snapshot(db, section); + }); } - void kv_database::add_to_snapshot( - const eosio::chain::snapshot_writer_ptr& snapshot, const eosio::chain::block_state& head, - const eosio::chain::authorization_manager& authorization, - const eosio::chain::resource_limits::resource_limits_manager& resource_limits) const { + void add_to_snapshot(const chainbase::database& db, + const eosio::chain::snapshot_writer_ptr& snapshot, + const eosio::chain::block_state& head, + const eosio::chain::authorization_manager& authorization, + const eosio::chain::resource_limits::resource_limits_manager& resource_limits) { snapshot->write_section( - [this](auto& section) { section.add_row(chain_snapshot_header(), db); }); + [&](auto& section) { section.add_row(chain_snapshot_header(), db); }); snapshot->write_section( - [this, &head](auto& section) { section.template add_row(head, db); }); + [&](auto& section) { section.template add_row(head, db); }); - eosio::chain::controller_index_set::walk_indices([this, &snapshot](auto utils) { + controller_index_set::walk_indices([&](auto utils) { using value_t = typename decltype(utils)::index_t::value_type; - snapshot->write_section([utils, this](auto& section) { - walk_index(utils, db, [this, §ion](const auto& row) { section.add_row(row, db); }); + snapshot->write_section([utils, &db](auto& section) { + walk_index(utils, db, [&](const auto& row) { section.add_row(row, db); }); }); }); add_kv_table_to_snapshot(snapshot, db); - add_contract_tables_to_snapshot(snapshot); + add_contract_tables_to_snapshot(db, snapshot); authorization.add_to_snapshot(snapshot); resource_limits.add_to_snapshot(snapshot); } - void kv_database::read_from_snapshot(const snapshot_reader_ptr& snapshot, - uint32_t blog_start, - uint32_t blog_end, - eosio::chain::authorization_manager& authorization, - eosio::chain::resource_limits::resource_limits_manager& resource_limits, - eosio::chain::block_state_ptr& head, uint32_t& snapshot_head_block, - const eosio::chain::chain_id_type& chain_id) { + void read_from_snapshot(chainbase::database& db, + const snapshot_reader_ptr& snapshot, + uint32_t blog_start, + uint32_t blog_end, + eosio::chain::authorization_manager& authorization, + eosio::chain::resource_limits::resource_limits_manager& resource_limits, + eosio::chain::block_state_ptr& head, + uint32_t& snapshot_head_block, + const eosio::chain::chain_id_type& chain_id) { chain_snapshot_header header; - snapshot->read_section([this, &header](auto& section) { + snapshot->read_section([&](auto& section) { section.read_row(header, db); header.validate(); }); db.create([](auto&) {}); - check_backing_store_setting(true); + check_backing_store_setting(db, true); { /// load and upgrade the block header state block_header_state head_header_state; using v2 = legacy::snapshot_block_header_state_v2; if (std::clamp(header.version, v2::minimum_version, v2::maximum_version) == header.version) { - snapshot->read_section([this, &head_header_state](auto& section) { + snapshot->read_section([&](auto& section) { legacy::snapshot_block_header_state_v2 legacy_header_state; section.read_row(legacy_header_state, db); head_header_state = block_header_state(std::move(legacy_header_state)); }); } else { snapshot->read_section( - [this, &head_header_state](auto& section) { section.read_row(head_header_state, db); }); + [&](auto& section) { section.read_row(head_header_state, db); }); } snapshot_head_block = head_header_state.block_num; @@ -179,7 +218,7 @@ namespace eosio { namespace chain { static_cast(*head) = head_header_state; } - controller_index_set::walk_indices([this, &snapshot, &header](auto utils) { + controller_index_set::walk_indices([&](auto utils) { using value_t = typename decltype(utils)::index_t::value_type; // skip the table_id_object as its inlined with contract tables section @@ -209,7 +248,7 @@ namespace eosio { namespace chain { "Snapshot indicates chain_snapshot_header version 2, but does not contain a genesis_state. " "It must be corrupted."); snapshot->read_section( - [&db = this->db, gs_chain_id = genesis->compute_chain_id()](auto& section) { + [&db, gs_chain_id = genesis->compute_chain_id()](auto& section) { v2 legacy_global_properties; section.read_row(legacy_global_properties, db); @@ -222,7 +261,7 @@ namespace eosio { namespace chain { } if (std::clamp(header.version, v3::minimum_version, v3::maximum_version) == header.version) { - snapshot->read_section([&db = this->db](auto& section) { + snapshot->read_section([&](auto& section) { v3 legacy_global_properties; section.read_row(legacy_global_properties, db); @@ -235,7 +274,7 @@ namespace eosio { namespace chain { } if (std::clamp(header.version, v4::minimum_version, v4::maximum_version) == header.version) { - snapshot->read_section([&db = this->db](auto& section) { + snapshot->read_section([&](auto& section) { v4 legacy_global_properties; section.read_row(legacy_global_properties, db); @@ -248,21 +287,21 @@ namespace eosio { namespace chain { } - snapshot->read_section([this](auto& section) { + snapshot->read_section([&](auto& section) { bool more = !section.empty(); while (more) { - decltype(utils)::create(db, [this, §ion, &more](auto& row) { more = section.read_row(row, db); }); + decltype(utils)::create(db, [&](auto& row) { more = section.read_row(row, db); }); } }); }); read_kv_table_from_snapshot(snapshot, db, header.version); - read_contract_tables_from_snapshot(snapshot); + read_contract_tables_from_snapshot(db, snapshot); authorization.read_from_snapshot(snapshot); resource_limits.read_from_snapshot(snapshot, header.version); - set_revision(head->block_num); + db.set_revision(head->block_num); db.create([](const auto& header) { // nothing to do }); @@ -274,72 +313,6 @@ namespace eosio { namespace chain { ("snapshot_chain_id", gpo.chain_id)("controller_chain_id", chain_id)); } - template - void chainbase_add_contract_tables_to_snapshot(const chainbase::database& db, Section& section) { - index_utils::walk(db, [&db, §ion](const table_id_object& table_row) { - // add a row for the table - section.add_row(table_row, db); - - // followed by a size row and then N data rows for each type of table - contract_database_index_set::walk_indices([&db, §ion, &table_row](auto utils) { - using utils_t = decltype(utils); - using value_t = typename utils_t::index_t::value_type; - using by_table_id = object_to_table_id_tag_t; - - auto tid_key = std::make_tuple(table_row.id); - auto next_tid_key = std::make_tuple(table_id_object::id_type(table_row.id._id + 1)); - - unsigned_int size = utils_t::template size_range(db, tid_key, next_tid_key); - section.add_row(size, db); - - utils_t::template walk_range(db, tid_key, next_tid_key, [&db, §ion](const auto& row) { - section.add_row(row, db); - }); - }); - }); - } - - void kv_database::add_contract_tables_to_snapshot(const snapshot_writer_ptr& snapshot) const { - snapshot->write_section("contract_tables", [this](auto& section) { - chainbase_add_contract_tables_to_snapshot(db, section); - }); - } - - template - void chainbase_read_contract_tables_from_snapshot(chainbase::database& db, Section& section) { - bool more = !section.empty(); - while (more) { - // read the row for the table - table_id_object::id_type t_id; - - index_utils::create(db, [&db, §ion, &t_id](auto& row) { - section.read_row(row, db); - t_id = row.id; - }); - - // read the size and data rows for each type of table - contract_database_index_set::walk_indices([&db, §ion, &t_id, &more](auto utils) { - using utils_t = decltype(utils); - - unsigned_int size; - more = section.read_row(size, db); - - for (size_t idx = 0; idx < size.value; ++idx) { - utils_t::create(db, [&db, §ion, &more, &t_id](auto& row) { - row.t_id = t_id; - more = section.read_row(row, db); - }); - } - }); - } - } - - void kv_database::read_contract_tables_from_snapshot(const snapshot_reader_ptr& snapshot) { - snapshot->read_section("contract_tables", [this](auto& section) { - chainbase_read_contract_tables_from_snapshot(db, section); - }); - } - std::optional extract_legacy_genesis_state(snapshot_reader& snapshot, uint32_t version) { std::optional genesis; @@ -352,6 +325,6 @@ namespace eosio { namespace chain { } return genesis; } -}} // namespace eosio::chain +}}} // namespace eosio::chain::db_util diff --git a/libraries/chain/include/eosio/chain/controller.hpp b/libraries/chain/include/eosio/chain/controller.hpp index f691f1ac34..75e5789e0a 100644 --- a/libraries/chain/include/eosio/chain/controller.hpp +++ b/libraries/chain/include/eosio/chain/controller.hpp @@ -33,8 +33,6 @@ namespace eosio { namespace chain { class resource_limits_manager; }; - class kv_database; - struct controller_impl; using chainbase::database; using chainbase::pinnable_mapped_file; @@ -391,14 +389,12 @@ namespace eosio { namespace chain { void replace_producer_keys( const public_key_type& key ); void replace_account_keys( name account, name permission, const public_key_type& key ); - eosio::chain::kv_database& kv_db()const; - + chainbase::database& mutable_db()const; + private: friend class apply_context; friend class transaction_context; - chainbase::database& mutable_db()const; - std::unique_ptr my; }; diff --git a/libraries/chain/include/eosio/chain/db_util.hpp b/libraries/chain/include/eosio/chain/db_util.hpp new file mode 100644 index 0000000000..2977a795e8 --- /dev/null +++ b/libraries/chain/include/eosio/chain/db_util.hpp @@ -0,0 +1,97 @@ +#pragma once +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace eosio { namespace chain { + + class apply_context; + + namespace backing_store { + struct db_context; + } + +namespace db_util { + using controller_index_set = + index_set; + + using contract_database_index_set = index_set; + + using db_context = backing_store::db_context; + + class maybe_session { + public: + maybe_session() = default; + + maybe_session(chainbase::database& cb_database); + maybe_session(maybe_session&& src) noexcept; + + maybe_session& operator=(const maybe_session& src) = delete; + + void push(); + + void squash(); + + void undo(); + + private: + std::unique_ptr cb_session = {}; + }; + + // Save the backing_store setting to the chainbase + // Currently deprecated, left just for backward compatibility + void check_backing_store_setting(chainbase::database& db, bool clean_startup); + + void destroy(const fc::path& p); + + std::unique_ptr create_kv_context(chainbase::database& db, + name receiver, + kv_resource_manager resource_manager, + const kv_database_config& limits); + + std::unique_ptr create_db_context(apply_context& context, name receiver); + + void add_to_snapshot(const chainbase::database& db, + const eosio::chain::snapshot_writer_ptr& snapshot, + const eosio::chain::block_state& head, + const eosio::chain::authorization_manager& authorization, + const eosio::chain::resource_limits::resource_limits_manager& resource_limits); + + void read_from_snapshot(chainbase::database& db, + const snapshot_reader_ptr& snapshot, + uint32_t blog_start, + uint32_t blog_end, + eosio::chain::authorization_manager& authorization, + eosio::chain::resource_limits::resource_limits_manager& resource_limits, + eosio::chain::block_state_ptr& head, + uint32_t& snapshot_head_block, + const eosio::chain::chain_id_type& chain_id); + + std::optional extract_legacy_genesis_state(snapshot_reader& snapshot, uint32_t version); +}}} // namespace eosio::chain::db_util diff --git a/libraries/chain/include/eosio/chain/kv_database.hpp b/libraries/chain/include/eosio/chain/kv_database.hpp deleted file mode 100644 index 2af7a6f115..0000000000 --- a/libraries/chain/include/eosio/chain/kv_database.hpp +++ /dev/null @@ -1,130 +0,0 @@ -#pragma once -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -// It's a fatal condition if chainbase and chain_kv get out of sync with each -// other due to exceptions. -#define CATCH_AND_EXIT_DB_FAILURE() \ - catch (...) { \ - elog("Error while using database"); \ - std::abort(); \ - } - -namespace eosio { namespace chain { - - using controller_index_set = - index_set; - - using contract_database_index_set = index_set; - - class apply_context; - - namespace backing_store { - struct db_context; - } - using db_context = backing_store::db_context; - - class kv_session { - public: - kv_session() = default; - - kv_session(chainbase::database& cb_database); - kv_session(kv_session&& src) noexcept; - - ~kv_session() { undo(); } - - kv_session& operator=(const kv_session& src) = delete; - - void push(); - - void squash(); - - void undo(); - - private: - std::unique_ptr cb_session = {}; - }; - - class kv_database { - public: - explicit kv_database(chainbase::database& chain_db); - - kv_database(const kv_database& copy) = delete; - kv_database& operator=(const kv_database& copy) = delete; - - // Save the backing_store setting to the chainbase - // Currently deprecated, left just for backward compatibility - void check_backing_store_setting(bool clean_startup); - - static kv_session make_no_op_session() { return kv_session(); } - - kv_session make_session() { - return kv_session(db); - } - - void set_revision(uint64_t revision); - - int64_t revision(); - - void undo(); - - void commit(int64_t revision); - - void flush(); - - static void destroy(const fc::path& p); - - std::unique_ptr create_kv_context(name receiver, kv_resource_manager resource_manager, - const kv_database_config& limits)const; - - std::unique_ptr create_db_context(apply_context& context, name receiver); - - void add_to_snapshot(const eosio::chain::snapshot_writer_ptr& snapshot, const eosio::chain::block_state& head, - const eosio::chain::authorization_manager& authorization, - const eosio::chain::resource_limits::resource_limits_manager& resource_limits) const; - - void read_from_snapshot(const snapshot_reader_ptr& snapshot, uint32_t blog_start, uint32_t blog_end, - eosio::chain::authorization_manager& authorization, - eosio::chain::resource_limits::resource_limits_manager& resource_limits, - eosio::chain::block_state_ptr& head, uint32_t& snapshot_head_block, - const eosio::chain::chain_id_type& chain_id); - - auto &get_db(void) const { return db; } - - private: - void add_contract_tables_to_snapshot(const snapshot_writer_ptr& snapshot) const; - void read_contract_tables_from_snapshot(const snapshot_reader_ptr& snapshot); - - backing_store_type backing_store; - chainbase::database& db; - }; - - std::optional extract_legacy_genesis_state(snapshot_reader& snapshot, uint32_t version); -}} // namespace eosio::chain diff --git a/libraries/chain/include/eosio/chain/transaction_context.hpp b/libraries/chain/include/eosio/chain/transaction_context.hpp index d97da3e881..d4758df184 100644 --- a/libraries/chain/include/eosio/chain/transaction_context.hpp +++ b/libraries/chain/include/eosio/chain/transaction_context.hpp @@ -1,5 +1,5 @@ #pragma once -#include +#include #include #include #include @@ -157,7 +157,7 @@ namespace eosio { namespace chain { controller& control; const packed_transaction& packed_trx; - kv_session undo_session; + db_util::maybe_session undo_session; transaction_trace_ptr trace; fc::time_point start; diff --git a/libraries/chain/transaction_context.cpp b/libraries/chain/transaction_context.cpp index 426df1eed8..7a06a71da9 100644 --- a/libraries/chain/transaction_context.cpp +++ b/libraries/chain/transaction_context.cpp @@ -21,6 +21,8 @@ namespace eosio { namespace chain { + using namespace db_util; + transaction_checktime_timer::transaction_checktime_timer(platform_timer& timer) : expired(timer.expired), _timer(timer) { expired = 0; @@ -49,7 +51,7 @@ namespace eosio { namespace chain { fc::time_point s ) :control(c) ,packed_trx(t) - ,undo_session(!c.skip_db_sessions() ? c.kv_db().make_session() : c.kv_db().make_no_op_session()) + ,undo_session(!c.skip_db_sessions() ? maybe_session(c.mutable_db()) : maybe_session()) ,trace(std::make_shared()) ,start(s) ,transaction_timer(std::move(tmr)) diff --git a/libraries/state_history/create_deltas.cpp b/libraries/state_history/create_deltas.cpp index 0b55aff599..1d729ded91 100644 --- a/libraries/state_history/create_deltas.cpp +++ b/libraries/state_history/create_deltas.cpp @@ -140,13 +140,5 @@ std::vector create_deltas(const chainbase::database& db, bool full_ return deltas; } -std::vector create_deltas(const chain::kv_database& db, bool full_snapshot){ - auto &chainbase_db = db.get_db(); - - std::vector deltas = create_deltas(chainbase_db, full_snapshot); - - return deltas; -} - } // namespace state_history } // namespace eosio diff --git a/libraries/state_history/include/eosio/state_history/create_deltas.hpp b/libraries/state_history/include/eosio/state_history/create_deltas.hpp index 177703d3cd..4378cbf2f8 100644 --- a/libraries/state_history/include/eosio/state_history/create_deltas.hpp +++ b/libraries/state_history/include/eosio/state_history/create_deltas.hpp @@ -1,12 +1,12 @@ #pragma once #include -#include +#include namespace eosio { namespace state_history { -std::vector create_deltas(const chain::kv_database& db, bool full_snapshot); +std::vector create_deltas(const chainbase::database& db, bool full_snapshot); } // namespace state_history } // namespace eosio diff --git a/libraries/state_history/include/eosio/state_history/log.hpp b/libraries/state_history/include/eosio/state_history/log.hpp index 344bee7e29..6bd29dd092 100644 --- a/libraries/state_history/include/eosio/state_history/log.hpp +++ b/libraries/state_history/include/eosio/state_history/log.hpp @@ -6,7 +6,7 @@ #include #include -#include +#include #include #include #include @@ -241,7 +241,7 @@ class state_history_chain_state_log : public state_history_log { std::shared_ptr> get_log_entry(block_num_type block_num); - void store(const chain::kv_database& db, const chain::block_state_ptr& block_state); + void store(const chainbase::database& db, const chain::block_state_ptr& block_state); private: void write_payload(cfile_stream& stream, const std::vector& data) override; }; diff --git a/libraries/state_history/log.cpp b/libraries/state_history/log.cpp index 3b4a4d3ed0..d399279b4b 100644 --- a/libraries/state_history/log.cpp +++ b/libraries/state_history/log.cpp @@ -504,7 +504,7 @@ std::shared_ptr> state_history_chain_state_log::get_log_entry( return std::make_shared>(); } -void state_history_chain_state_log::store(const chain::kv_database& db, +void state_history_chain_state_log::store(const chainbase::database& db, const chain::block_state_ptr& block_state) { auto [begin, end] = begin_end_block_nums(); bool fresh = begin == end; diff --git a/plugins/chain_plugin/chain_plugin.cpp b/plugins/chain_plugin/chain_plugin.cpp index a4c80021d9..b6ac5012a2 100644 --- a/plugins/chain_plugin/chain_plugin.cpp +++ b/plugins/chain_plugin/chain_plugin.cpp @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include @@ -931,7 +930,7 @@ void chain_plugin::plugin_initialize(const variables_map& options) { ilog( "Replay requested: deleting state database" ); if( options.at( "truncate-at-block" ).as() > 0 ) wlog( "The --truncate-at-block option does not work for a regular replay of the blockchain." ); - eosio::chain::kv_database::destroy( my->chain_config->state_dir ); + eosio::chain::db_util::destroy( my->chain_config->state_dir ); } else if( options.at( "truncate-at-block" ).as() > 0 ) { wlog( "The --truncate-at-block option can only be used with --hard-replay-blockchain." ); } @@ -1970,7 +1969,7 @@ struct kv_table_rows_context { kv_table_rows_context(const controller& db, const read_only::get_kv_table_rows_params& param, const fc::microseconds abi_serializer_max_time, bool shorten_error) - : kv_context(db.kv_db().create_kv_context( + : kv_context(db_util::create_kv_context(db.mutable_db(), param.code, {}, db.get_global_properties().kv_configuration)) // To do: provide kv_resource_manmager to create_kv_context , p(param) diff --git a/plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp b/plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp index 4facfb009e..20e30bdb30 100644 --- a/plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp +++ b/plugins/chain_plugin/include/eosio/chain_plugin/chain_plugin.hpp @@ -14,7 +14,7 @@ #include #include #include -#include +#include #include #include diff --git a/plugins/state_history_plugin/state_history_plugin.cpp b/plugins/state_history_plugin/state_history_plugin.cpp index 4106801c8a..cf4c5e73c7 100644 --- a/plugins/state_history_plugin/state_history_plugin.cpp +++ b/plugins/state_history_plugin/state_history_plugin.cpp @@ -550,7 +550,7 @@ struct state_history_plugin_impl : std::enable_shared_from_thisstore(chain_plug->chain().kv_db(), block_state); + chain_state_log->store(chain_plug->chain().db(), block_state); } return; } diff --git a/plugins/trace_api_plugin/include/eosio/trace_api/store_provider.hpp b/plugins/trace_api_plugin/include/eosio/trace_api/store_provider.hpp index 113cbfbd0e..508cec647b 100644 --- a/plugins/trace_api_plugin/include/eosio/trace_api/store_provider.hpp +++ b/plugins/trace_api_plugin/include/eosio/trace_api/store_provider.hpp @@ -11,7 +11,7 @@ #include #include #include -#include +#include #include namespace eosio::trace_api { diff --git a/programs/eosio-tester/main.cpp b/programs/eosio-tester/main.cpp index 4638f53bc2..299b98f58b 100644 --- a/programs/eosio-tester/main.cpp +++ b/programs/eosio-tester/main.cpp @@ -253,7 +253,7 @@ struct test_chain { std::vector traces; state_history::trace_converter::unpack(strm, traces); message.traces = traces; - message.deltas = fc::raw::pack(create_deltas(control->kv_db(), !prev_block)); + message.deltas = fc::raw::pack(create_deltas(control->db(), !prev_block)); prev_block = message.this_block; history[control->head_block_num()] = fc::raw::pack(state_result{ message }); diff --git a/unittests/state_history_tests.cpp b/unittests/state_history_tests.cpp index 181d4b5ba0..2dd71fdaa9 100644 --- a/unittests/state_history_tests.cpp +++ b/unittests/state_history_tests.cpp @@ -255,7 +255,7 @@ BOOST_AUTO_TEST_CASE(test_chain_state_log) { uint32_t last_accepted_block_num = 0; chain.control->accepted_block.connect([&](const block_state_ptr& block_state) { - log.store(chain.control->kv_db(), block_state); + log.store(chain.control->db(), block_state); last_accepted_block_num = block_state->block_num; }); @@ -292,7 +292,7 @@ struct state_history_tester : state_history_tester_logs, tester { control.accepted_block.connect([&](const block_state_ptr& bs) { traces_log.store(control.db(), bs); - chain_state_log.store(control.kv_db(), bs); + chain_state_log.store(control.db(), bs); }); control.block_start.connect([&](uint32_t block_num) { traces_log.block_start(block_num); } ); }) {} @@ -539,7 +539,7 @@ BOOST_AUTO_TEST_CASE(test_state_result_abi) { std::vector traces; trace_converter::unpack(strm, traces); message.traces = traces; - message.deltas = fc::raw::pack(create_deltas(control->kv_db(), !prev_block)); + message.deltas = fc::raw::pack(create_deltas(control->db(), !prev_block)); prev_block = message.this_block; history[control->head_block_num()] = fc::raw::pack(state_result{message}); @@ -651,7 +651,7 @@ class table_deltas_tester : public tester { table_deltas_tester(backing_store_type backing_store, setup_policy policy=setup_policy::full) : tester(policy, db_read_mode::SPECULATIVE, std::optional{}, std::optional{}, backing_store) { }; pair find_table_delta(const std::string &name, bool full_snapshot = false) { - v = eosio::state_history::create_deltas(control->kv_db(), full_snapshot); + v = eosio::state_history::create_deltas(control->db(), full_snapshot); auto find_by_name = [&name](const auto& x) { return x.name == name; @@ -680,7 +680,7 @@ BOOST_AUTO_TEST_CASE(test_deltas_not_empty) { for (backing_store_type backing_store : { backing_store_type::CHAINBASE } ) { table_deltas_tester chain { backing_store }; - auto deltas = eosio::state_history::create_deltas(chain.control->kv_db(), false); + auto deltas = eosio::state_history::create_deltas(chain.control->db(), false); for(const auto &delta: deltas) { BOOST_REQUIRE(!delta.rows.obj.empty()); From cf4cf8574b4a5fe077052c270eddfb399d6b27cb Mon Sep 17 00:00:00 2001 From: Dmytro Sydorchenko Date: Tue, 25 Jan 2022 23:00:33 -0500 Subject: [PATCH 57/62] #11028 concerns addressed --- libraries/chain/CMakeLists.txt | 1 - libraries/chain/apply_context.cpp | 33 +- libraries/chain/backing_store/db_context.cpp | 44 ++- .../backing_store/db_context_chainbase.cpp | 297 ------------------ libraries/chain/controller.cpp | 1 + libraries/chain/db_util.cpp | 10 +- .../include/eosio/chain/apply_context.hpp | 24 +- .../eosio/chain/backing_store/db_context.hpp | 219 ++----------- .../chain/include/eosio/chain/controller.hpp | 12 +- .../chain/include/eosio/chain/db_util.hpp | 22 +- libraries/chain/webassembly/database.cpp | 120 +++---- plugins/chain_plugin/chain_plugin.cpp | 2 +- programs/eosio-tester/main.cpp | 43 +-- 13 files changed, 165 insertions(+), 663 deletions(-) delete mode 100644 libraries/chain/backing_store/db_context_chainbase.cpp diff --git a/libraries/chain/CMakeLists.txt b/libraries/chain/CMakeLists.txt index 1710598a9f..165f34ec5d 100644 --- a/libraries/chain/CMakeLists.txt +++ b/libraries/chain/CMakeLists.txt @@ -134,7 +134,6 @@ add_library( eosio_chain platform_timer_accuracy.cpp backing_store/kv_context.cpp backing_store/db_context.cpp - backing_store/db_context_chainbase.cpp ${PLATFORM_TIMER_IMPL} ${HEADERS} ) diff --git a/libraries/chain/apply_context.cpp b/libraries/chain/apply_context.cpp index 37c14cc6cb..9005a5a7c4 100644 --- a/libraries/chain/apply_context.cpp +++ b/libraries/chain/apply_context.cpp @@ -14,11 +14,10 @@ #include using boost::container::flat_set; +using namespace eosio::chain::backing_store; namespace eosio { namespace chain { -using db_context = backing_store::db_context; - static inline void print_debug(account_name receiver, const action_trace& ar) { if (!ar.console.empty()) { auto prefix = fc::format_string( @@ -51,7 +50,6 @@ apply_context::apply_context(controller& con, transaction_context& trx_ctx, uint act = &trace.act; receiver = trace.receiver; context_free = trace.context_free; - _db_context = db_util::create_db_context(*this, receiver); } template @@ -104,7 +102,7 @@ void apply_context::exec_one() kv_iterators.resize(1); kv_destroyed_iterators.clear(); if (!context_free) { - kv_backing_store = db_util::create_kv_context(control.mutable_db(), receiver, create_kv_resource_manager(*this), control.get_global_properties().kv_configuration); + kv_backing_store = db_util::create_kv_context(control, receiver, create_kv_resource_manager(*this), control.get_global_properties().kv_configuration); } receiver_account = &db.get( receiver ); if( !(context_free && control.skip_trx_checks()) ) { @@ -222,7 +220,6 @@ void apply_context::exec() increment_action_id(); for( uint32_t i = 1; i < _notified.size(); ++i ) { std::tie( receiver, action_ordinal ) = _notified[i]; - _db_context->receiver = receiver; exec_one(); increment_action_id(); } @@ -846,7 +843,7 @@ int apply_context::get_context_free_data( uint32_t index, char* buffer, size_t b return copy_size; } -int apply_context::db_store_i64_chainbase( name scope, name table, const account_name& payer, uint64_t id, const char* buffer, size_t buffer_size ) { +int apply_context::db_store_i64( name scope, name table, const account_name& payer, uint64_t id, const char* buffer, size_t buffer_size ) { // require_write_lock( scope ); const auto& tab = find_or_create_table( receiver, scope, table, payer ); auto tableid = tab.id; @@ -881,7 +878,7 @@ int apply_context::db_store_i64_chainbase( name scope, name table, const account return db_iter_store.add( obj ); } -void apply_context::db_update_i64_chainbase( int iterator, account_name payer, const char* buffer, size_t buffer_size ) { +void apply_context::db_update_i64( int iterator, account_name payer, const char* buffer, size_t buffer_size ) { const key_value_object& obj = db_iter_store.get( iterator ); const auto& table_obj = db_iter_store.get_table( obj.t_id ); @@ -922,7 +919,7 @@ void apply_context::db_update_i64_chainbase( int iterator, account_name payer, c }); } -void apply_context::db_remove_i64_chainbase( int iterator ) { +void apply_context::db_remove_i64( int iterator ) { const key_value_object& obj = db_iter_store.get( iterator ); const auto& table_obj = db_iter_store.get_table( obj.t_id ); @@ -953,7 +950,7 @@ void apply_context::db_remove_i64_chainbase( int iterator ) { db_iter_store.remove( iterator ); } -int apply_context::db_get_i64_chainbase( int iterator, char* buffer, size_t buffer_size ) { +int apply_context::db_get_i64( int iterator, char* buffer, size_t buffer_size ) { const key_value_object& obj = db_iter_store.get( iterator ); auto s = obj.value.size(); @@ -965,7 +962,7 @@ int apply_context::db_get_i64_chainbase( int iterator, char* buffer, size_t buff return copy_size; } -int apply_context::db_next_i64_chainbase( int iterator, uint64_t& primary ) { +int apply_context::db_next_i64( int iterator, uint64_t& primary ) { if( iterator < -1 ) return -1; // cannot increment past end iterator of table const auto& obj = db_iter_store.get( iterator ); // Check for iterator != -1 happens in this call @@ -980,7 +977,7 @@ int apply_context::db_next_i64_chainbase( int iterator, uint64_t& primary ) { return db_iter_store.add( *itr ); } -int apply_context::db_previous_i64_chainbase( int iterator, uint64_t& primary ) { +int apply_context::db_previous_i64( int iterator, uint64_t& primary ) { const auto& idx = db.get_index(); if( iterator < -1 ) // is end iterator @@ -1012,7 +1009,7 @@ int apply_context::db_previous_i64_chainbase( int iterator, uint64_t& primary ) return db_iter_store.add(*itr); } -int apply_context::db_find_i64_chainbase( name code, name scope, name table, uint64_t id ) { +int apply_context::db_find_i64( name code, name scope, name table, uint64_t id ) { //require_read_lock( code, scope ); // redundant? const auto* tab = find_table( code, scope, table ); @@ -1026,7 +1023,7 @@ int apply_context::db_find_i64_chainbase( name code, name scope, name table, uin return db_iter_store.add( *obj ); } -int apply_context::db_lowerbound_i64_chainbase( name code, name scope, name table, uint64_t id ) { +int apply_context::db_lowerbound_i64( name code, name scope, name table, uint64_t id ) { //require_read_lock( code, scope ); // redundant? const auto* tab = find_table( code, scope, table ); @@ -1042,7 +1039,7 @@ int apply_context::db_lowerbound_i64_chainbase( name code, name scope, name tabl return db_iter_store.add( *itr ); } -int apply_context::db_upperbound_i64_chainbase( name code, name scope, name table, uint64_t id ) { +int apply_context::db_upperbound_i64( name code, name scope, name table, uint64_t id ) { //require_read_lock( code, scope ); // redundant? const auto* tab = find_table( code, scope, table ); @@ -1058,7 +1055,7 @@ int apply_context::db_upperbound_i64_chainbase( name code, name scope, name tabl return db_iter_store.add( *itr ); } -int apply_context::db_end_i64_chainbase( name code, name scope, name table ) { +int apply_context::db_end_i64( name code, name scope, name table ) { //require_read_lock( code, scope ); // redundant? const auto* tab = find_table( code, scope, table ); @@ -1201,10 +1198,4 @@ uint32_t apply_context::get_action_id() const { void apply_context::increment_action_id() { trx_context.action_id.increment(); } - -db_context& apply_context::db_get_context() { - EOS_ASSERT( _db_context, action_validate_exception, - "context-free actions cannot access state" ); - return *_db_context; -} } } /// eosio::chain diff --git a/libraries/chain/backing_store/db_context.cpp b/libraries/chain/backing_store/db_context.cpp index f5a0c60352..08be87a2c8 100644 --- a/libraries/chain/backing_store/db_context.cpp +++ b/libraries/chain/backing_store/db_context.cpp @@ -2,9 +2,9 @@ #include #include -namespace eosio { namespace chain { namespace backing_store { +namespace eosio { namespace chain { namespace backing_store { namespace db_context { -std::string db_context::table_event(name code, name scope, name table) { +std::string table_event(name code, name scope, name table) { return STORAGE_EVENT_ID("${code}:${scope}:${table}", ("code", code) ("scope", scope) @@ -12,7 +12,7 @@ std::string db_context::table_event(name code, name scope, name table) { ); } -std::string db_context::table_event(name code, name scope, name table, name qualifier) { +std::string table_event(name code, name scope, name table, name qualifier) { return STORAGE_EVENT_ID("${code}:${scope}:${table}:${qualifier}", ("code", code) ("scope", scope) @@ -21,7 +21,7 @@ std::string db_context::table_event(name code, name scope, name table, name qual ); } -void db_context::log_insert_table(fc::logger& deep_mind_logger, uint32_t action_id, name code, name scope, name table, account_name payer) { +void log_insert_table(fc::logger& deep_mind_logger, uint32_t action_id, name code, name scope, name table, account_name payer) { fc_dlog(deep_mind_logger, "TBL_OP INS ${action_id} ${code} ${scope} ${table} ${payer}", ("action_id", action_id) ("code", code) @@ -31,7 +31,7 @@ void db_context::log_insert_table(fc::logger& deep_mind_logger, uint32_t action_ ); } -void db_context::log_remove_table(fc::logger& deep_mind_logger, uint32_t action_id, name code, name scope, name table, account_name payer) { +void log_remove_table(fc::logger& deep_mind_logger, uint32_t action_id, name code, name scope, name table, account_name payer) { fc_dlog(deep_mind_logger, "TBL_OP REM ${action_id} ${code} ${scope} ${table} ${payer}", ("action_id", action_id) ("code", code) @@ -41,7 +41,7 @@ void db_context::log_remove_table(fc::logger& deep_mind_logger, uint32_t action_ ); } -void db_context::log_row_insert(fc::logger& deep_mind_logger, uint32_t action_id, name code, name scope, name table, +void log_row_insert(fc::logger& deep_mind_logger, uint32_t action_id, name code, name scope, name table, account_name payer, account_name primkey, const char* buffer, size_t buffer_size) { fc_dlog(deep_mind_logger, "DB_OP INS ${action_id} ${payer} ${table_code} ${scope} ${table_name} ${primkey} ${ndata}", ("action_id", action_id) @@ -54,7 +54,7 @@ void db_context::log_row_insert(fc::logger& deep_mind_logger, uint32_t action_id ); } -void db_context::log_row_update(fc::logger& deep_mind_logger, uint32_t action_id, name code, name scope, name table, +void log_row_update(fc::logger& deep_mind_logger, uint32_t action_id, name code, name scope, name table, account_name old_payer, account_name new_payer, account_name primkey, const char* old_buffer, size_t old_buffer_size, const char* new_buffer, size_t new_buffer_size) { fc_dlog(deep_mind_logger, "DB_OP UPD ${action_id} ${opayer}:${npayer} ${table_code} ${scope} ${table_name} ${primkey} ${odata}:${ndata}", @@ -70,7 +70,7 @@ void db_context::log_row_update(fc::logger& deep_mind_logger, uint32_t action_id ); } -void db_context::log_row_remove(fc::logger& deep_mind_logger, uint32_t action_id, name code, name scope, name table, +void log_row_remove(fc::logger& deep_mind_logger, uint32_t action_id, name code, name scope, name table, account_name payer, account_name primkey, const char* buffer, size_t buffer_size) { fc_dlog(deep_mind_logger, "DB_OP REM ${action_id} ${payer} ${table_code} ${scope} ${table_name} ${primkey} ${odata}", ("action_id", action_id) @@ -83,52 +83,48 @@ void db_context::log_row_remove(fc::logger& deep_mind_logger, uint32_t action_id ); } -storage_usage_trace db_context::add_table_trace(uint32_t action_id, std::string&& event_id) { +storage_usage_trace add_table_trace(uint32_t action_id, std::string&& event_id) { return storage_usage_trace(action_id, std::move(event_id), "table", "add", "create_table"); } -storage_usage_trace db_context::rem_table_trace(uint32_t action_id, std::string&& event_id) { +storage_usage_trace rem_table_trace(uint32_t action_id, std::string&& event_id) { return storage_usage_trace(action_id, std::move(event_id), "table", "remove", "remove_table"); } -storage_usage_trace db_context::row_add_trace(uint32_t action_id, std::string&& event_id) { +storage_usage_trace row_add_trace(uint32_t action_id, std::string&& event_id) { return storage_usage_trace(action_id, std::move(event_id), "table_row", "add", "primary_index_add"); } -storage_usage_trace db_context::row_update_trace(uint32_t action_id, std::string&& event_id) { +storage_usage_trace row_update_trace(uint32_t action_id, std::string&& event_id) { return storage_usage_trace(action_id, std::move(event_id), "table_row", "update", "primary_index_update"); } -storage_usage_trace db_context::row_update_add_trace(uint32_t action_id, std::string&& event_id) { +storage_usage_trace row_update_add_trace(uint32_t action_id, std::string&& event_id) { return storage_usage_trace(action_id, std::move(event_id), "table_row", "add", "primary_index_update_add_new_payer"); } -storage_usage_trace db_context::row_update_rem_trace(uint32_t action_id, std::string&& event_id) { +storage_usage_trace row_update_rem_trace(uint32_t action_id, std::string&& event_id) { return storage_usage_trace(action_id, std::move(event_id), "table_row", "remove", "primary_index_update_remove_old_payer"); } -storage_usage_trace db_context::row_rem_trace(uint32_t action_id, std::string&& event_id) { +storage_usage_trace row_rem_trace(uint32_t action_id, std::string&& event_id) { return storage_usage_trace(action_id, std::move(event_id), "table_row", "remove", "primary_index_remove"); } -storage_usage_trace db_context::secondary_add_trace(uint32_t action_id, std::string&& event_id) { +storage_usage_trace secondary_add_trace(uint32_t action_id, std::string&& event_id) { return storage_usage_trace(action_id, std::move(event_id), "secondary_index", "add", "secondary_index_add"); } -storage_usage_trace db_context::secondary_rem_trace(uint32_t action_id, std::string&& event_id) { +storage_usage_trace secondary_rem_trace(uint32_t action_id, std::string&& event_id) { return storage_usage_trace(action_id, std::move(event_id), "secondary_index", "remove", "secondary_index_remove"); } -storage_usage_trace db_context::secondary_update_add_trace(uint32_t action_id, std::string&& event_id) { +storage_usage_trace secondary_update_add_trace(uint32_t action_id, std::string&& event_id) { return storage_usage_trace(action_id, std::move(event_id), "secondary_index", "add", "secondary_index_update_add_new_payer"); } -storage_usage_trace db_context::secondary_update_rem_trace(uint32_t action_id, std::string&& event_id) { +storage_usage_trace secondary_update_rem_trace(uint32_t action_id, std::string&& event_id) { return storage_usage_trace(action_id, std::move(event_id), "secondary_index", "remove", "secondary_index_update_remove_old_payer"); } -void db_context::update_db_usage( const account_name& payer, int64_t delta, const storage_usage_trace& trace ) { - context.update_db_usage(payer, delta, trace); -} - -}}} // namespace eosio::chain::backing_store +}}}} // namespace eosio::chain::backing_store::db_context diff --git a/libraries/chain/backing_store/db_context_chainbase.cpp b/libraries/chain/backing_store/db_context_chainbase.cpp deleted file mode 100644 index b5a15aca8e..0000000000 --- a/libraries/chain/backing_store/db_context_chainbase.cpp +++ /dev/null @@ -1,297 +0,0 @@ -#include -#include -#include - -namespace eosio { namespace chain { namespace backing_store { - - struct db_context_chainbase : db_context { - db_context_chainbase(apply_context& context, name receiver) - : db_context( context, receiver ) {} - - int32_t db_store_i64(uint64_t scope, uint64_t table, account_name payer, uint64_t id, const char* buffer , size_t buffer_size) override { - return context.db_store_i64_chainbase(name(scope), name(table), payer, id, buffer, buffer_size); - } - - void db_update_i64(int32_t itr, account_name payer, const char* buffer , size_t buffer_size) override { - context.db_update_i64_chainbase(itr, payer, buffer, buffer_size); - } - - void db_remove_i64(int32_t itr) override { - context.db_remove_i64_chainbase(itr); - } - - int32_t db_get_i64(int32_t itr, char* buffer , size_t buffer_size) override { - return context.db_get_i64_chainbase(itr, buffer, buffer_size); - } - - int32_t db_next_i64(int32_t itr, uint64_t& primary) override { - return context.db_next_i64_chainbase(itr, primary); - } - - int32_t db_previous_i64(int32_t itr, uint64_t& primary) override { - return context.db_previous_i64_chainbase(itr, primary); - } - - int32_t db_find_i64(uint64_t code, uint64_t scope, uint64_t table, uint64_t id) override { - return context.db_find_i64_chainbase(name(code), name(scope), name(table), id); - } - - int32_t db_lowerbound_i64(uint64_t code, uint64_t scope, uint64_t table, uint64_t id) override { - return context.db_lowerbound_i64_chainbase(name(code), name(scope), name(table), id); - } - - int32_t db_upperbound_i64(uint64_t code, uint64_t scope, uint64_t table, uint64_t id) override { - return context.db_upperbound_i64_chainbase(name(code), name(scope), name(table), id); - } - - int32_t db_end_i64(uint64_t code, uint64_t scope, uint64_t table) override { - return context.db_end_i64_chainbase(name(code), name(scope), name(table)); - } - - /** - * interface for uint64_t secondary - */ - int32_t db_idx64_store(uint64_t scope, uint64_t table, account_name payer, uint64_t id, - const uint64_t& secondary) override { - return context.idx64.store( scope, table, payer, id, secondary ); - } - - void db_idx64_update(int32_t iterator, account_name payer, const uint64_t& secondary) override { - context.idx64.update( iterator, payer, secondary ); - } - - void db_idx64_remove(int32_t iterator) override { - return context.idx64.remove(iterator); - } - - int32_t db_idx64_find_secondary(uint64_t code, uint64_t scope, uint64_t table, const uint64_t& secondary, - uint64_t& primary) override { - return context.idx64.find_secondary(code, scope, table, secondary, primary); - } - - int32_t db_idx64_find_primary(uint64_t code, uint64_t scope, uint64_t table, uint64_t& secondary, - uint64_t primary) override { - return context.idx64.find_primary(code, scope, table, secondary, primary); - } - - int32_t db_idx64_lowerbound(uint64_t code, uint64_t scope, uint64_t table, uint64_t& secondary, - uint64_t& primary) override { - return context.idx64.lowerbound_secondary(code, scope, table, secondary, primary); - } - - int32_t db_idx64_upperbound(uint64_t code, uint64_t scope, uint64_t table, uint64_t& secondary, - uint64_t& primary) override { - return context.idx64.upperbound_secondary(code, scope, table, secondary, primary); - } - - int32_t db_idx64_end(uint64_t code, uint64_t scope, uint64_t table) override { - return context.idx64.end_secondary(code, scope, table); - } - - int32_t db_idx64_next(int32_t iterator, uint64_t& primary) override { - return context.idx64.next_secondary(iterator, primary); - } - - int32_t db_idx64_previous(int32_t iterator, uint64_t& primary) override { - return context.idx64.previous_secondary(iterator, primary); - } - - /** - * interface for uint128_t secondary - */ - int32_t db_idx128_store(uint64_t scope, uint64_t table, account_name payer, uint64_t id, - const uint128_t& secondary) override { - return context.idx128.store(scope, table, payer, id, secondary); - } - - void db_idx128_update(int32_t iterator, account_name payer, const uint128_t& secondary) override { - return context.idx128.update(iterator, payer, secondary); - } - - void db_idx128_remove(int32_t iterator) override { - context.idx128.remove(iterator); - } - - int32_t db_idx128_find_secondary(uint64_t code, uint64_t scope, uint64_t table, const uint128_t& secondary, - uint64_t& primary) override { - return context.idx128.find_secondary(code, scope, table, secondary, primary); - } - - int32_t db_idx128_find_primary(uint64_t code, uint64_t scope, uint64_t table, uint128_t& secondary, - uint64_t primary) override { - return context.idx128.find_primary(code, scope, table, secondary, primary); - } - - int32_t db_idx128_lowerbound(uint64_t code, uint64_t scope, uint64_t table, uint128_t& secondary, - uint64_t& primary) override { - return context.idx128.lowerbound_secondary(code, scope, table, secondary, primary); - } - - int32_t db_idx128_upperbound(uint64_t code, uint64_t scope, uint64_t table, uint128_t& secondary, - uint64_t& primary) override { - return context.idx128.upperbound_secondary(code, scope, table, secondary, primary); - } - - int32_t db_idx128_end(uint64_t code, uint64_t scope, uint64_t table) override { - return context.idx128.end_secondary(code, scope, table); - } - - int32_t db_idx128_next(int32_t iterator, uint64_t& primary) override { - return context.idx128.next_secondary(iterator, primary); - } - - int32_t db_idx128_previous(int32_t iterator, uint64_t& primary) override { - return context.idx128.previous_secondary(iterator, primary); - } - - /** - * interface for 256-bit interger secondary - */ - int32_t db_idx256_store(uint64_t scope, uint64_t table, account_name payer, uint64_t id, - const uint128_t* data) override { - return context.idx256.store(scope, table, payer, id, data); - } - - void db_idx256_update(int32_t iterator, account_name payer, const uint128_t* data) override { - context.idx256.update(iterator, payer, data); - } - - void db_idx256_remove(int32_t iterator) override { - context.idx256.remove(iterator); - } - - int32_t db_idx256_find_secondary(uint64_t code, uint64_t scope, uint64_t table, const uint128_t* data, - uint64_t& primary) override { - return context.idx256.find_secondary(code, scope, table, data, primary); - } - - int32_t db_idx256_find_primary(uint64_t code, uint64_t scope, uint64_t table, uint128_t* data, - uint64_t primary) override { - return context.idx256.find_primary(code, scope, table, data, primary); - } - - int32_t db_idx256_lowerbound(uint64_t code, uint64_t scope, uint64_t table, uint128_t* data, - uint64_t& primary) override { - return context.idx256.lowerbound_secondary(code, scope, table, data, primary); - } - - int32_t db_idx256_upperbound(uint64_t code, uint64_t scope, uint64_t table, uint128_t* data, - uint64_t& primary) override { - return context.idx256.upperbound_secondary(code, scope, table, data, primary); - } - - int32_t db_idx256_end(uint64_t code, uint64_t scope, uint64_t table) override { - return context.idx256.end_secondary(code, scope, table); - } - - int32_t db_idx256_next(int32_t iterator, uint64_t& primary) override { - return context.idx256.next_secondary(iterator, primary); - } - - int32_t db_idx256_previous(int32_t iterator, uint64_t& primary) override { - return context.idx256.previous_secondary(iterator, primary); - } - - /** - * interface for double secondary - */ - int32_t db_idx_double_store(uint64_t scope, uint64_t table, account_name payer, uint64_t id, - const float64_t& secondary) override { - return context.idx_double.store(scope, table, payer, id, secondary); - } - - void db_idx_double_update(int32_t iterator, account_name payer, const float64_t& secondary) override { - context.idx_double.update(iterator, payer, secondary); - } - - void db_idx_double_remove(int32_t iterator) override { - context.idx_double.remove(iterator); - } - - int32_t db_idx_double_find_secondary(uint64_t code, uint64_t scope, uint64_t table, - const float64_t& secondary, uint64_t& primary) override { - return context.idx_double.find_secondary(code, scope, table, secondary, primary); - } - - int32_t db_idx_double_find_primary(uint64_t code, uint64_t scope, uint64_t table, float64_t& secondary, - uint64_t primary) override { - return context.idx_double.find_primary(code, scope, table, secondary, primary); - } - - int32_t db_idx_double_lowerbound(uint64_t code, uint64_t scope, uint64_t table, float64_t& secondary, - uint64_t& primary) override { - return context.idx_double.lowerbound_secondary(code, scope, table, secondary, primary); - } - - int32_t db_idx_double_upperbound(uint64_t code, uint64_t scope, uint64_t table, float64_t& secondary, - uint64_t& primary) override { - return context.idx_double.upperbound_secondary(code, scope, table, secondary, primary); - } - - int32_t db_idx_double_end(uint64_t code, uint64_t scope, uint64_t table) override { - return context.idx_double.end_secondary(code, scope, table); - } - - int32_t db_idx_double_next(int32_t iterator, uint64_t& primary) override { - return context.idx_double.next_secondary(iterator, primary); - } - - int32_t db_idx_double_previous(int32_t iterator, uint64_t& primary) override { - return context.idx_double.previous_secondary(iterator, primary); - } - - /** - * interface for long double secondary - */ - int32_t db_idx_long_double_store(uint64_t scope, uint64_t table, account_name payer, uint64_t id, - const float128_t& secondary) override { - return context.idx_long_double.store(scope, table, payer, id, secondary); - } - - void db_idx_long_double_update(int32_t iterator, account_name payer, const float128_t& secondary) override { - context.idx_long_double.update(iterator, payer, secondary); - } - - void db_idx_long_double_remove(int32_t iterator) override { - context.idx_long_double.remove(iterator); - } - - int32_t db_idx_long_double_find_secondary(uint64_t code, uint64_t scope, uint64_t table, - const float128_t& secondary, uint64_t& primary) override { - return context.idx_long_double.find_secondary(code, scope, table, secondary, primary); - } - - int32_t db_idx_long_double_find_primary(uint64_t code, uint64_t scope, uint64_t table, - float128_t& secondary, uint64_t primary) override { - return context.idx_long_double.find_primary(code, scope, table, secondary, primary); - } - - int32_t db_idx_long_double_lowerbound(uint64_t code, uint64_t scope, uint64_t table, float128_t& secondary, - uint64_t& primary) override { - return context.idx_long_double.lowerbound_secondary(code, scope, table, secondary, primary); - } - - int32_t db_idx_long_double_upperbound(uint64_t code, uint64_t scope, uint64_t table, float128_t& secondary, - uint64_t& primary) override { - return context.idx_long_double.upperbound_secondary(code, scope, table, secondary, primary); - } - - int32_t db_idx_long_double_end(uint64_t code, uint64_t scope, uint64_t table) override { - return context.idx_long_double.end_secondary(code, scope, table); - } - - int32_t db_idx_long_double_next(int32_t iterator, uint64_t& primary) override { - return context.idx_long_double.next_secondary(iterator, primary); - } - - int32_t db_idx_long_double_previous(int32_t iterator, uint64_t& primary) override { - return context.idx_long_double.previous_secondary(iterator, primary); - } - }; // db_context_chainbase - - std::unique_ptr create_db_chainbase_context(apply_context& context, name receiver) - { - return std::make_unique(context, receiver); - } - -}}} // namespace eosio::chain::backing_store diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index fcb805744f..a68f13a5ce 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include diff --git a/libraries/chain/db_util.cpp b/libraries/chain/db_util.cpp index 2565689540..ba1bd2e88c 100644 --- a/libraries/chain/db_util.cpp +++ b/libraries/chain/db_util.cpp @@ -1,7 +1,6 @@ #include #include #include -#include namespace eosio { namespace chain { namespace db_util { maybe_session::maybe_session(chainbase::database& cb_database) { @@ -71,16 +70,11 @@ namespace eosio { namespace chain { namespace db_util { fc::remove( p / "shared_memory.meta" ); } - std::unique_ptr create_kv_context(chainbase::database& db, + std::unique_ptr create_kv_context(const controller& c, name receiver, kv_resource_manager resource_manager, const kv_database_config& limits) { - return create_kv_chainbase_context(db, receiver, resource_manager, limits); - } - - std::unique_ptr create_db_context(apply_context& context, - name receiver) { - return backing_store::create_db_chainbase_context(context, receiver); + return create_kv_chainbase_context(c.mutable_db(), receiver, resource_manager, limits); } template diff --git a/libraries/chain/include/eosio/chain/apply_context.hpp b/libraries/chain/include/eosio/chain/apply_context.hpp index fab1a3e32b..9450083f45 100644 --- a/libraries/chain/include/eosio/chain/apply_context.hpp +++ b/libraries/chain/include/eosio/chain/apply_context.hpp @@ -391,18 +391,16 @@ class apply_context { void update_db_usage( const account_name& payer, int64_t delta, const storage_usage_trace& trace ); - int db_store_i64_chainbase( name scope, name table, const account_name& payer, uint64_t id, const char* buffer, size_t buffer_size ); - void db_update_i64_chainbase( int iterator, account_name payer, const char* buffer, size_t buffer_size ); - void db_remove_i64_chainbase( int iterator ); - int db_get_i64_chainbase( int iterator, char* buffer, size_t buffer_size ); - int db_next_i64_chainbase( int iterator, uint64_t& primary ); - int db_previous_i64_chainbase( int iterator, uint64_t& primary ); - int db_find_i64_chainbase( name code, name scope, name table, uint64_t id ); - int db_lowerbound_i64_chainbase( name code, name scope, name table, uint64_t id ); - int db_upperbound_i64_chainbase( name code, name scope, name table, uint64_t id ); - int db_end_i64_chainbase( name code, name scope, name table ); - - backing_store::db_context& db_get_context(); + int db_store_i64( name scope, name table, const account_name& payer, uint64_t id, const char* buffer, size_t buffer_size ); + void db_update_i64( int iterator, account_name payer, const char* buffer, size_t buffer_size ); + void db_remove_i64( int iterator ); + int db_get_i64( int iterator, char* buffer, size_t buffer_size ); + int db_next_i64( int iterator, uint64_t& primary ); + int db_previous_i64( int iterator, uint64_t& primary ); + int db_find_i64( name code, name scope, name table, uint64_t id ); + int db_lowerbound_i64( name code, name scope, name table, uint64_t id ); + int db_upperbound_i64( name code, name scope, name table, uint64_t id ); + int db_end_i64( name code, name scope, name table ); private: @@ -498,8 +496,6 @@ class apply_context { vector _cfa_inline_actions; ///< action_ordinals of queued inline context-free actions std::string _pending_console_output; flat_set _account_ram_deltas; ///< flat_set of account_delta so json is an array of objects - - std::unique_ptr _db_context; }; using apply_handler = std::function; diff --git a/libraries/chain/include/eosio/chain/backing_store/db_context.hpp b/libraries/chain/include/eosio/chain/backing_store/db_context.hpp index c458bb4278..d95186e531 100644 --- a/libraries/chain/include/eosio/chain/backing_store/db_context.hpp +++ b/libraries/chain/include/eosio/chain/backing_store/db_context.hpp @@ -20,198 +20,27 @@ namespace chain { class apply_context; -namespace backing_store { - struct db_context { - db_context(apply_context& c, const name& recv) : context(c), receiver(recv) {} - - virtual ~db_context() {} - - virtual int32_t db_store_i64(uint64_t scope, uint64_t table, account_name payer, uint64_t id, const char* buffer , size_t buffer_size) = 0; - - virtual void db_update_i64(int32_t itr, account_name payer, const char* buffer , size_t buffer_size) = 0; - - virtual void db_remove_i64(int32_t itr) = 0; - - virtual int32_t db_get_i64(int32_t itr, char* buffer , size_t buffer_size) = 0; - - virtual int32_t db_next_i64(int32_t itr, uint64_t& primary) = 0; - - virtual int32_t db_previous_i64(int32_t itr, uint64_t& primary) = 0; - - virtual int32_t db_find_i64(uint64_t code, uint64_t scope, uint64_t table, uint64_t id) = 0; - - virtual int32_t db_lowerbound_i64(uint64_t code, uint64_t scope, uint64_t table, uint64_t id) = 0; - - virtual int32_t db_upperbound_i64(uint64_t code, uint64_t scope, uint64_t table, uint64_t id) = 0; - - virtual int32_t db_end_i64(uint64_t code, uint64_t scope, uint64_t table) = 0; - - /** - * interface for uint64_t secondary - */ - virtual int32_t db_idx64_store(uint64_t scope, uint64_t table, account_name payer, uint64_t id, - const uint64_t& secondary) = 0; - - virtual void db_idx64_update(int32_t iterator, account_name payer, const uint64_t& secondary) = 0; - - virtual void db_idx64_remove(int32_t iterator) = 0; - - virtual int32_t db_idx64_find_secondary(uint64_t code, uint64_t scope, uint64_t table, - const uint64_t& secondary, uint64_t& primary) = 0; - - virtual int32_t db_idx64_find_primary(uint64_t code, uint64_t scope, uint64_t table, uint64_t& secondary, - uint64_t primary) = 0; - - virtual int32_t db_idx64_lowerbound(uint64_t code, uint64_t scope, uint64_t table, uint64_t& secondary, - uint64_t& primary) = 0; - - virtual int32_t db_idx64_upperbound(uint64_t code, uint64_t scope, uint64_t table, uint64_t& secondary, - uint64_t& primary) = 0; - - virtual int32_t db_idx64_end(uint64_t code, uint64_t scope, uint64_t table) = 0; - - virtual int32_t db_idx64_next(int32_t iterator, uint64_t& primary) = 0; - - virtual int32_t db_idx64_previous(int32_t iterator, uint64_t& primary) = 0; - - /** - * interface for uint128_t secondary - */ - virtual int32_t db_idx128_store(uint64_t scope, uint64_t table, account_name payer, uint64_t id, - const uint128_t& secondary) = 0; - - virtual void db_idx128_update(int32_t iterator, account_name payer, const uint128_t& secondary) = 0; - - virtual void db_idx128_remove(int32_t iterator) = 0; - - virtual int32_t db_idx128_find_secondary(uint64_t code, uint64_t scope, uint64_t table, - const uint128_t& secondary, uint64_t& primary) = 0; - - virtual int32_t db_idx128_find_primary(uint64_t code, uint64_t scope, uint64_t table, uint128_t& secondary, - uint64_t primary) = 0; - - virtual int32_t db_idx128_lowerbound(uint64_t code, uint64_t scope, uint64_t table, uint128_t& secondary, - uint64_t& primary) = 0; - - virtual int32_t db_idx128_upperbound(uint64_t code, uint64_t scope, uint64_t table, uint128_t& secondary, - uint64_t& primary) = 0; - - virtual int32_t db_idx128_end(uint64_t code, uint64_t scope, uint64_t table) = 0; - - virtual int32_t db_idx128_next(int32_t iterator, uint64_t& primary) = 0; - - virtual int32_t db_idx128_previous(int32_t iterator, uint64_t& primary) = 0; - - /** - * interface for 256-bit interger secondary - */ - virtual int32_t db_idx256_store(uint64_t scope, uint64_t table, account_name payer, uint64_t id, - const uint128_t* data) = 0; - - virtual void db_idx256_update(int32_t iterator, account_name payer, const uint128_t* data) = 0; - - virtual void db_idx256_remove(int32_t iterator) = 0; - - virtual int32_t db_idx256_find_secondary(uint64_t code, uint64_t scope, uint64_t table, const uint128_t* data, - uint64_t& primary) = 0; - - virtual int32_t db_idx256_find_primary(uint64_t code, uint64_t scope, uint64_t table, uint128_t* data, - uint64_t primary) = 0; - - virtual int32_t db_idx256_lowerbound(uint64_t code, uint64_t scope, uint64_t table, uint128_t* data, - uint64_t& primary) = 0; - - virtual int32_t db_idx256_upperbound(uint64_t code, uint64_t scope, uint64_t table, uint128_t* data, - uint64_t& primary) = 0; - - virtual int32_t db_idx256_end(uint64_t code, uint64_t scope, uint64_t table) = 0; - - virtual int32_t db_idx256_next(int32_t iterator, uint64_t& primary) = 0; - - virtual int32_t db_idx256_previous(int32_t iterator, uint64_t& primary) = 0; - - /** - * interface for double secondary - */ - virtual int32_t db_idx_double_store(uint64_t scope, uint64_t table, account_name payer, uint64_t id, - const float64_t& secondary) = 0; - - virtual void db_idx_double_update(int32_t iterator, account_name payer, const float64_t& secondary) = 0; - - virtual void db_idx_double_remove(int32_t iterator) = 0; - - virtual int32_t db_idx_double_find_secondary(uint64_t code, uint64_t scope, uint64_t table, - const float64_t& secondary, uint64_t& primary) = 0; - - virtual int32_t db_idx_double_find_primary(uint64_t code, uint64_t scope, uint64_t table, - float64_t& secondary, uint64_t primary) = 0; - - virtual int32_t db_idx_double_lowerbound(uint64_t code, uint64_t scope, uint64_t table, float64_t& secondary, - uint64_t& primary) = 0; - - virtual int32_t db_idx_double_upperbound(uint64_t code, uint64_t scope, uint64_t table, float64_t& secondary, - uint64_t& primary) = 0; - - virtual int32_t db_idx_double_end(uint64_t code, uint64_t scope, uint64_t table) = 0; - - virtual int32_t db_idx_double_next(int32_t iterator, uint64_t& primary) = 0; - - virtual int32_t db_idx_double_previous(int32_t iterator, uint64_t& primary) = 0; - - /** - * interface for long double secondary - */ - virtual int32_t db_idx_long_double_store(uint64_t scope, uint64_t table, account_name payer, uint64_t id, - const float128_t& secondary) = 0; - - virtual void db_idx_long_double_update(int32_t iterator, account_name payer, const float128_t& secondary) = 0; - - virtual void db_idx_long_double_remove(int32_t iterator) = 0; - - virtual int32_t db_idx_long_double_find_secondary(uint64_t code, uint64_t scope, uint64_t table, - const float128_t& secondary, uint64_t& primary) = 0; - - virtual int32_t db_idx_long_double_find_primary(uint64_t code, uint64_t scope, uint64_t table, - float128_t& secondary, uint64_t primary) = 0; - - virtual int32_t db_idx_long_double_lowerbound(uint64_t code, uint64_t scope, uint64_t table, - float128_t& secondary, uint64_t& primary) = 0; - - virtual int32_t db_idx_long_double_upperbound(uint64_t code, uint64_t scope, uint64_t table, - float128_t& secondary, uint64_t& primary) = 0; - - virtual int32_t db_idx_long_double_end(uint64_t code, uint64_t scope, uint64_t table) = 0; - - virtual int32_t db_idx_long_double_next(int32_t iterator, uint64_t& primary) = 0; - - virtual int32_t db_idx_long_double_previous(int32_t iterator, uint64_t& primary) = 0; - - static std::string table_event(name code, name scope, name table); - static std::string table_event(name code, name scope, name table, name qualifier); - static void log_insert_table(fc::logger& deep_mind_logger, uint32_t action_id, name code, name scope, name table, account_name payer); - static void log_remove_table(fc::logger& deep_mind_logger, uint32_t action_id, name code, name scope, name table, account_name payer); - static void log_row_insert(fc::logger& deep_mind_logger, uint32_t action_id, name code, name scope, name table, - account_name payer, account_name primkey, const char* buffer, size_t buffer_size); - static void log_row_update(fc::logger& deep_mind_logger, uint32_t action_id, name code, name scope, name table, - account_name old_payer, account_name new_payer, account_name primkey, - const char* old_buffer, size_t old_buffer_size, const char* new_buffer, size_t new_buffer_size); - static void log_row_remove(fc::logger& deep_mind_logger, uint32_t action_id, name code, name scope, name table, - account_name payer, account_name primkey, const char* buffer, size_t buffer_size); - static storage_usage_trace add_table_trace(uint32_t action_id, std::string&& event_id); - static storage_usage_trace rem_table_trace(uint32_t action_id, std::string&& event_id); - static storage_usage_trace row_add_trace(uint32_t action_id, std::string&& event_id); - static storage_usage_trace row_update_trace(uint32_t action_id, std::string&& event_id); - static storage_usage_trace row_update_add_trace(uint32_t action_id, std::string&& event_id); - static storage_usage_trace row_update_rem_trace(uint32_t action_id, std::string&& event_id); - static storage_usage_trace row_rem_trace(uint32_t action_id, std::string&& event_id); - static storage_usage_trace secondary_add_trace(uint32_t action_id, std::string&& event_id); - static storage_usage_trace secondary_rem_trace(uint32_t action_id, std::string&& event_id); - static storage_usage_trace secondary_update_add_trace(uint32_t action_id, std::string&& event_id); - static storage_usage_trace secondary_update_rem_trace(uint32_t action_id, std::string&& event_id); - void update_db_usage(const account_name& payer, int64_t delta, const storage_usage_trace& trace); - apply_context& context; - name receiver; - }; - - std::unique_ptr create_db_chainbase_context(apply_context& context, name receiver); -}}} // ns eosio::chain::backing_store +namespace backing_store { namespace db_context { + std::string table_event(name code, name scope, name table); + std::string table_event(name code, name scope, name table, name qualifier); + void log_insert_table(fc::logger& deep_mind_logger, uint32_t action_id, name code, name scope, name table, account_name payer); + void log_remove_table(fc::logger& deep_mind_logger, uint32_t action_id, name code, name scope, name table, account_name payer); + void log_row_insert(fc::logger& deep_mind_logger, uint32_t action_id, name code, name scope, name table, + account_name payer, account_name primkey, const char* buffer, size_t buffer_size); + void log_row_update(fc::logger& deep_mind_logger, uint32_t action_id, name code, name scope, name table, + account_name old_payer, account_name new_payer, account_name primkey, + const char* old_buffer, size_t old_buffer_size, const char* new_buffer, size_t new_buffer_size); + void log_row_remove(fc::logger& deep_mind_logger, uint32_t action_id, name code, name scope, name table, + account_name payer, account_name primkey, const char* buffer, size_t buffer_size); + storage_usage_trace add_table_trace(uint32_t action_id, std::string&& event_id); + storage_usage_trace rem_table_trace(uint32_t action_id, std::string&& event_id); + storage_usage_trace row_add_trace(uint32_t action_id, std::string&& event_id); + storage_usage_trace row_update_trace(uint32_t action_id, std::string&& event_id); + storage_usage_trace row_update_add_trace(uint32_t action_id, std::string&& event_id); + storage_usage_trace row_update_rem_trace(uint32_t action_id, std::string&& event_id); + storage_usage_trace row_rem_trace(uint32_t action_id, std::string&& event_id); + storage_usage_trace secondary_add_trace(uint32_t action_id, std::string&& event_id); + storage_usage_trace secondary_rem_trace(uint32_t action_id, std::string&& event_id); + storage_usage_trace secondary_update_add_trace(uint32_t action_id, std::string&& event_id); + storage_usage_trace secondary_update_rem_trace(uint32_t action_id, std::string&& event_id); +}}}} // ns eosio::chain::backing_store::db_context diff --git a/libraries/chain/include/eosio/chain/controller.hpp b/libraries/chain/include/eosio/chain/controller.hpp index c93ac8f87f..0004003d0a 100644 --- a/libraries/chain/include/eosio/chain/controller.hpp +++ b/libraries/chain/include/eosio/chain/controller.hpp @@ -50,6 +50,13 @@ namespace eosio { namespace chain { class fork_database; + struct kv_context; + struct kv_resource_manager; + struct kv_database_config; + namespace db_util{ + std::unique_ptr create_kv_context(const controller&, name, kv_resource_manager, const kv_database_config&); + }; + enum class db_read_mode { SPECULATIVE, HEAD, @@ -388,11 +395,12 @@ namespace eosio { namespace chain { void replace_producer_keys( const public_key_type& key ); void replace_account_keys( name account, name permission, const public_key_type& key ); - chainbase::database& mutable_db()const; - private: friend class apply_context; friend class transaction_context; + friend std::unique_ptr db_util::create_kv_context(const controller&, name, kv_resource_manager, const kv_database_config&); + + chainbase::database& mutable_db()const; std::unique_ptr my; diff --git a/libraries/chain/include/eosio/chain/db_util.hpp b/libraries/chain/include/eosio/chain/db_util.hpp index 2977a795e8..a4c1295e27 100644 --- a/libraries/chain/include/eosio/chain/db_util.hpp +++ b/libraries/chain/include/eosio/chain/db_util.hpp @@ -1,5 +1,4 @@ #pragma once -#include #include #include #include @@ -11,29 +10,18 @@ #include #include #include -#include #include #include #include -#include #include #include #include #include -#include #include #include -namespace eosio { namespace chain { - - class apply_context; - - namespace backing_store { - struct db_context; - } - -namespace db_util { +namespace eosio { namespace chain { namespace db_util { using controller_index_set = index_set; - using db_context = backing_store::db_context; - class maybe_session { public: maybe_session() = default; - maybe_session(chainbase::database& cb_database); + explicit maybe_session(chainbase::database& cb_database); maybe_session(maybe_session&& src) noexcept; maybe_session& operator=(const maybe_session& src) = delete; @@ -70,13 +56,11 @@ namespace db_util { void destroy(const fc::path& p); - std::unique_ptr create_kv_context(chainbase::database& db, + std::unique_ptr create_kv_context(const controller& c, name receiver, kv_resource_manager resource_manager, const kv_database_config& limits); - std::unique_ptr create_db_context(apply_context& context, name receiver); - void add_to_snapshot(const chainbase::database& db, const eosio::chain::snapshot_writer_ptr& snapshot, const eosio::chain::block_state& head, diff --git a/libraries/chain/webassembly/database.cpp b/libraries/chain/webassembly/database.cpp index 45afe0ed3f..a5fde9a542 100644 --- a/libraries/chain/webassembly/database.cpp +++ b/libraries/chain/webassembly/database.cpp @@ -6,74 +6,74 @@ namespace eosio { namespace chain { namespace webassembly { * interface for primary index */ int32_t interface::db_store_i64( uint64_t scope, uint64_t table, uint64_t payer, uint64_t id, legacy_span buffer ) { - return context.db_get_context().db_store_i64( scope, table, account_name(payer), id, buffer.data(), buffer.size() ); + return context.db_store_i64( name(scope), name(table), account_name(payer), id, buffer.data(), buffer.size() ); } void interface::db_update_i64( int32_t itr, uint64_t payer, legacy_span buffer ) { - context.db_get_context().db_update_i64( itr, account_name(payer), buffer.data(), buffer.size() ); + context.db_update_i64( itr, account_name(payer), buffer.data(), buffer.size() ); } void interface::db_remove_i64( int32_t itr ) { - context.db_get_context().db_remove_i64( itr ); + context.db_remove_i64( itr ); } int32_t interface::db_get_i64( int32_t itr, legacy_span buffer ) { - return context.db_get_context().db_get_i64( itr, buffer.data(), buffer.size() ); + return context.db_get_i64( itr, buffer.data(), buffer.size() ); } int32_t interface::db_next_i64( int32_t itr, legacy_ptr primary ) { - return context.db_get_context().db_next_i64(itr, *primary); + return context.db_next_i64(itr, *primary); } int32_t interface::db_previous_i64( int32_t itr, legacy_ptr primary ) { - return context.db_get_context().db_previous_i64(itr, *primary); + return context.db_previous_i64(itr, *primary); } int32_t interface::db_find_i64( uint64_t code, uint64_t scope, uint64_t table, uint64_t id ) { - return context.db_get_context().db_find_i64( code, scope, table, id ); + return context.db_find_i64( name(code), name(scope), name(table), id ); } int32_t interface::db_lowerbound_i64( uint64_t code, uint64_t scope, uint64_t table, uint64_t id ) { - return context.db_get_context().db_lowerbound_i64( code, scope, table, id ); + return context.db_lowerbound_i64( name(code), name(scope), name(table), id ); } int32_t interface::db_upperbound_i64( uint64_t code, uint64_t scope, uint64_t table, uint64_t id ) { - return context.db_get_context().db_upperbound_i64( code, scope, table, id ); + return context.db_upperbound_i64( name(code), name(scope), name(table), id ); } int32_t interface::db_end_i64( uint64_t code, uint64_t scope, uint64_t table ) { - return context.db_get_context().db_end_i64( code, scope, table ); + return context.db_end_i64( name(code), name(scope), name(table) ); } /** * interface for uint64_t secondary */ int32_t interface::db_idx64_store( uint64_t scope, uint64_t table, uint64_t payer, uint64_t id, legacy_ptr secondary ) { - return context.db_get_context().db_idx64_store( scope, table, account_name(payer), id, *secondary ); + return context.idx64.store( scope, table, account_name(payer), id, *secondary ); } void interface::db_idx64_update( int32_t iterator, uint64_t payer, legacy_ptr secondary ) { - context.db_get_context().db_idx64_update( iterator, account_name(payer), *secondary ); + context.idx64.update( iterator, account_name(payer), *secondary ); } void interface::db_idx64_remove( int32_t iterator ) { - context.db_get_context().db_idx64_remove( iterator ); + context.idx64.remove( iterator ); } int32_t interface::db_idx64_find_secondary( uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, legacy_ptr primary ) { - return context.db_get_context().db_idx64_find_secondary(code, scope, table, *secondary, *primary); + return context.idx64.find_secondary(code, scope, table, *secondary, *primary); } int32_t interface::db_idx64_find_primary( uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, uint64_t primary ) { - return context.db_get_context().db_idx64_find_primary(code, scope, table, *secondary, primary); + return context.idx64.find_primary(code, scope, table, *secondary, primary); } int32_t interface::db_idx64_lowerbound( uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, legacy_ptr primary ) { - const int32_t ret = context.db_get_context().db_idx64_lowerbound(code, scope, table, *secondary, *primary); + const int32_t ret = context.idx64.lowerbound_secondary(code, scope, table, *secondary, *primary); (void)legacy_ptr(std::move(secondary)); (void)legacy_ptr(std::move(primary)); return ret; } int32_t interface::db_idx64_upperbound( uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, legacy_ptr primary ) { - const int32_t ret = context.db_get_context().db_idx64_upperbound(code, scope, table, *secondary, *primary); + const int32_t ret = context.idx64.upperbound_secondary(code, scope, table, *secondary, *primary); (void)legacy_ptr(std::move(secondary)); (void)legacy_ptr(std::move(primary)); return ret; } int32_t interface::db_idx64_end( uint64_t code, uint64_t scope, uint64_t table ) { - return context.db_get_context().db_idx64_end(code, scope, table); + return context.idx64.end_secondary(code, scope, table); } int32_t interface::db_idx64_next( int32_t iterator, legacy_ptr primary ) { - return context.db_get_context().db_idx64_next(iterator, *primary); + return context.idx64.next_secondary(iterator, *primary); } int32_t interface::db_idx64_previous( int32_t iterator, legacy_ptr primary ) { - return context.db_get_context().db_idx64_previous(iterator, *primary); + return context.idx64.previous_secondary(iterator, *primary); } @@ -81,40 +81,40 @@ namespace eosio { namespace chain { namespace webassembly { * interface for uint128_t secondary */ int32_t interface::db_idx128_store( uint64_t scope, uint64_t table, uint64_t payer, uint64_t id, legacy_ptr secondary ) { - return context.db_get_context().db_idx128_store( scope, table, account_name(payer), id, *secondary ); + return context.idx128.store( scope, table, account_name(payer), id, *secondary ); } void interface::db_idx128_update( int32_t iterator, uint64_t payer, legacy_ptr secondary ) { - return context.db_get_context().db_idx128_update( iterator, account_name(payer), *secondary ); + return context.idx128.update( iterator, account_name(payer), *secondary ); } void interface::db_idx128_remove( int32_t iterator ) { - return context.db_get_context().db_idx128_remove( iterator ); + return context.idx128.remove( iterator ); } int32_t interface::db_idx128_find_secondary( uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, legacy_ptr primary ) { - return context.db_get_context().db_idx128_find_secondary(code, scope, table, *secondary, *primary); + return context.idx128.find_secondary(code, scope, table, *secondary, *primary); } int32_t interface::db_idx128_find_primary( uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, uint64_t primary ) { - return context.db_get_context().db_idx128_find_primary(code, scope, table, *secondary, primary); + return context.idx128.find_primary(code, scope, table, *secondary, primary); } int32_t interface::db_idx128_lowerbound( uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, legacy_ptr primary ) { - int32_t result = context.db_get_context().db_idx128_lowerbound(code, scope, table, *secondary, *primary); + int32_t result = context.idx128.lowerbound_secondary(code, scope, table, *secondary, *primary); (void)legacy_ptr(std::move(secondary)); (void)legacy_ptr(std::move(primary)); return result; } int32_t interface::db_idx128_upperbound( uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, legacy_ptr primary ) { - int32_t result = context.db_get_context().db_idx128_upperbound(code, scope, table, *secondary, *primary); + int32_t result = context.idx128.upperbound_secondary(code, scope, table, *secondary, *primary); (void)legacy_ptr(std::move(secondary)); (void)legacy_ptr(std::move(primary)); return result; } int32_t interface::db_idx128_end( uint64_t code, uint64_t scope, uint64_t table ) { - return context.db_get_context().db_idx128_end(code, scope, table); + return context.idx128.end_secondary(code, scope, table); } int32_t interface::db_idx128_next( int32_t iterator, legacy_ptr primary ) { - return context.db_get_context().db_idx128_next(iterator, *primary); + return context.idx128.next_secondary(iterator, *primary); } int32_t interface::db_idx128_previous( int32_t iterator, legacy_ptr primary ) { - return context.db_get_context().db_idx128_previous(iterator, *primary); + return context.idx128.previous_secondary(iterator, *primary); } /** @@ -126,38 +126,38 @@ namespace eosio { namespace chain { namespace webassembly { db_api_exception, "invalid size of secondary key array for idx256: given ${given} bytes but expected ${expected} bytes", ("given",data.size())("expected", idx256_array_size) ); - return context.db_get_context().db_idx256_store(scope, table, account_name(payer), id, data.data()); + return context.idx256.store(scope, table, account_name(payer), id, data.data()); } void interface::db_idx256_update( int32_t iterator, uint64_t payer, legacy_span data ) { EOS_ASSERT( data.size() == idx256_array_size, db_api_exception, "invalid size of secondary key array for idx256: given ${given} bytes but expected ${expected} bytes", ("given",data.size())("expected", idx256_array_size) ); - return context.db_get_context().db_idx256_update(iterator, account_name(payer), data.data()); + return context.idx256.update(iterator, account_name(payer), data.data()); } void interface::db_idx256_remove( int32_t iterator ) { - return context.db_get_context().db_idx256_remove(iterator); + return context.idx256.remove(iterator); } int32_t interface::db_idx256_find_secondary( uint64_t code, uint64_t scope, uint64_t table, legacy_span data, legacy_ptr primary ) { EOS_ASSERT( data.size() == idx256_array_size, db_api_exception, "invalid size of secondary key array for idx256: given ${given} bytes but expected ${expected} bytes", ("given",data.size())("expected", idx256_array_size) ); - return context.db_get_context().db_idx256_find_secondary(code, scope, table, data.data(), *primary); + return context.idx256.find_secondary(code, scope, table, data.data(), *primary); } int32_t interface::db_idx256_find_primary( uint64_t code, uint64_t scope, uint64_t table, legacy_span data, uint64_t primary ) { EOS_ASSERT( data.size() == idx256_array_size, db_api_exception, "invalid size of secondary key array for idx256: given ${given} bytes but expected ${expected} bytes", ("given",data.size())("expected", idx256_array_size) ); - return context.db_get_context().db_idx256_find_primary(code, scope, table, data.data(), primary); + return context.idx256.find_primary(code, scope, table, data.data(), primary); } int32_t interface::db_idx256_lowerbound( uint64_t code, uint64_t scope, uint64_t table, legacy_span data, legacy_ptr primary ) { EOS_ASSERT( data.size() == idx256_array_size, db_api_exception, "invalid size of secondary key array for idx256: given ${given} bytes but expected ${expected} bytes", ("given",data.size())("expected", idx256_array_size) ); - int32_t result = context.db_get_context().db_idx256_lowerbound(code, scope, table, data.data(), *primary); + int32_t result = context.idx256.lowerbound_secondary(code, scope, table, data.data(), *primary); (void)legacy_span(std::move(data)); (void)legacy_ptr(std::move(primary)); return result; @@ -167,98 +167,98 @@ namespace eosio { namespace chain { namespace webassembly { db_api_exception, "invalid size of secondary key array for idx256: given ${given} bytes but expected ${expected} bytes", ("given",data.size())("expected", idx256_array_size) ); - int32_t result = context.db_get_context().db_idx256_upperbound(code, scope, table, data.data(), *primary); + int32_t result = context.idx256.upperbound_secondary(code, scope, table, data.data(), *primary); (void)legacy_span(std::move(data)); (void)legacy_ptr(std::move(primary)); return result; } int32_t interface::db_idx256_end( uint64_t code, uint64_t scope, uint64_t table ) { - return context.db_get_context().db_idx256_end(code, scope, table); + return context.idx256.end_secondary(code, scope, table); } int32_t interface::db_idx256_next( int32_t iterator, legacy_ptr primary ) { - return context.db_get_context().db_idx256_next(iterator, *primary); + return context.idx256.next_secondary(iterator, *primary); } int32_t interface::db_idx256_previous( int32_t iterator, legacy_ptr primary ) { - return context.db_get_context().db_idx256_previous(iterator, *primary); + return context.idx256.previous_secondary(iterator, *primary); } /** * interface for double secondary */ int32_t interface::db_idx_double_store( uint64_t scope, uint64_t table, uint64_t payer, uint64_t id, legacy_ptr secondary ) { - return context.db_get_context().db_idx_double_store( scope, table, account_name(payer), id, *secondary ); + return context.idx_double.store( scope, table, account_name(payer), id, *secondary ); } void interface::db_idx_double_update( int32_t iterator, uint64_t payer, legacy_ptr secondary ) { - return context.db_get_context().db_idx_double_update( iterator, account_name(payer), *secondary ); + return context.idx_double.update( iterator, account_name(payer), *secondary ); } void interface::db_idx_double_remove( int32_t iterator ) { - return context.db_get_context().db_idx_double_remove( iterator ); + return context.idx_double.remove( iterator ); } int32_t interface::db_idx_double_find_secondary( uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, legacy_ptr primary ) { - return context.db_get_context().db_idx_double_find_secondary(code, scope, table, *secondary, *primary); + return context.idx_double.find_secondary(code, scope, table, *secondary, *primary); } int32_t interface::db_idx_double_find_primary( uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, uint64_t primary ) { - return context.db_get_context().db_idx_double_find_primary(code, scope, table, *secondary, primary); + return context.idx_double.find_primary(code, scope, table, *secondary, primary); } int32_t interface::db_idx_double_lowerbound( uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, legacy_ptr primary ) { - int32_t result = context.db_get_context().db_idx_double_lowerbound(code, scope, table, *secondary, *primary); + int32_t result = context.idx_double.lowerbound_secondary(code, scope, table, *secondary, *primary); (void)legacy_ptr(std::move(secondary)); (void)legacy_ptr(std::move(primary)); return result; } int32_t interface::db_idx_double_upperbound( uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, legacy_ptr primary ) { - int32_t result = context.db_get_context().db_idx_double_upperbound(code, scope, table, *secondary, *primary); + int32_t result = context.idx_double.upperbound_secondary(code, scope, table, *secondary, *primary); (void)legacy_ptr(std::move(secondary)); (void)legacy_ptr(std::move(primary)); return result; } int32_t interface::db_idx_double_end( uint64_t code, uint64_t scope, uint64_t table ) { - return context.db_get_context().db_idx_double_end(code, scope, table); + return context.idx_double.end_secondary(code, scope, table); } int32_t interface::db_idx_double_next( int32_t iterator, legacy_ptr primary ) { - return context.db_get_context().db_idx_double_next(iterator, *primary); + return context.idx_double.next_secondary(iterator, *primary); } int32_t interface::db_idx_double_previous( int32_t iterator, legacy_ptr primary ) { - return context.db_get_context().db_idx_double_previous(iterator, *primary); + return context.idx_double.previous_secondary(iterator, *primary); } /** * interface for long double secondary */ int32_t interface::db_idx_long_double_store( uint64_t scope, uint64_t table, uint64_t payer, uint64_t id, legacy_ptr secondary ) { - return context.db_get_context().db_idx_long_double_store( scope, table, account_name(payer), id, *secondary ); + return context.idx_long_double.store( scope, table, account_name(payer), id, *secondary ); } void interface::db_idx_long_double_update( int32_t iterator, uint64_t payer, legacy_ptr secondary ) { - return context.db_get_context().db_idx_long_double_update( iterator, account_name(payer), *secondary ); + return context.idx_long_double.update( iterator, account_name(payer), *secondary ); } void interface::db_idx_long_double_remove( int32_t iterator ) { - return context.db_get_context().db_idx_long_double_remove( iterator ); + return context.idx_long_double.remove( iterator ); } int32_t interface::db_idx_long_double_find_secondary( uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, legacy_ptr primary ) { - return context.db_get_context().db_idx_long_double_find_secondary(code, scope, table, *secondary, *primary); + return context.idx_long_double.find_secondary(code, scope, table, *secondary, *primary); } int32_t interface::db_idx_long_double_find_primary( uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, uint64_t primary ) { - return context.db_get_context().db_idx_long_double_find_primary(code, scope, table, *secondary, primary); + return context.idx_long_double.find_primary(code, scope, table, *secondary, primary); } int32_t interface::db_idx_long_double_lowerbound( uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, legacy_ptr primary ) { - int32_t result = context.db_get_context().db_idx_long_double_lowerbound(code, scope, table, *secondary, *primary); + int32_t result = context.idx_long_double.lowerbound_secondary(code, scope, table, *secondary, *primary); (void)legacy_ptr(std::move(secondary)); (void)legacy_ptr(std::move(primary)); return result; } int32_t interface::db_idx_long_double_upperbound( uint64_t code, uint64_t scope, uint64_t table, legacy_ptr secondary, legacy_ptr primary ) { - int32_t result = context.db_get_context().db_idx_long_double_upperbound(code, scope, table, *secondary, *primary); + int32_t result = context.idx_long_double.upperbound_secondary(code, scope, table, *secondary, *primary); (void)legacy_ptr(std::move(secondary)); (void)legacy_ptr(std::move(primary)); return result; } int32_t interface::db_idx_long_double_end( uint64_t code, uint64_t scope, uint64_t table ) { - return context.db_get_context().db_idx_long_double_end(code, scope, table); + return context.idx_long_double.end_secondary(code, scope, table); } int32_t interface::db_idx_long_double_next( int32_t iterator, legacy_ptr primary ) { - return context.db_get_context().db_idx_long_double_next(iterator, *primary); + return context.idx_long_double.next_secondary(iterator, *primary); } int32_t interface::db_idx_long_double_previous( int32_t iterator, legacy_ptr primary ) { - return context.db_get_context().db_idx_long_double_previous(iterator, *primary); + return context.idx_long_double.previous_secondary(iterator, *primary); } }}} // ns eosio::chain::webassembly diff --git a/plugins/chain_plugin/chain_plugin.cpp b/plugins/chain_plugin/chain_plugin.cpp index 32700662c4..967d149486 100644 --- a/plugins/chain_plugin/chain_plugin.cpp +++ b/plugins/chain_plugin/chain_plugin.cpp @@ -1969,7 +1969,7 @@ struct kv_table_rows_context { kv_table_rows_context(const controller& db, const read_only::get_kv_table_rows_params& param, const fc::microseconds abi_serializer_max_time, bool shorten_error) - : kv_context(db_util::create_kv_context(db.mutable_db(), + : kv_context(db_util::create_kv_context(db, param.code, {}, db.get_global_properties().kv_configuration)) // To do: provide kv_resource_manmager to create_kv_context , p(param) diff --git a/programs/eosio-tester/main.cpp b/programs/eosio-tester/main.cpp index 98e66cad32..36a9bb0e0c 100644 --- a/programs/eosio-tester/main.cpp +++ b/programs/eosio-tester/main.cpp @@ -48,6 +48,7 @@ using eosio::state_history::create_deltas; using eosio::state_history::get_blocks_result_v1; using eosio::state_history::state_result; using eosio::vm::span; +using eosio::chain::name; struct callbacks; using rhf_t = eosio::vm::registered_host_functions; @@ -861,64 +862,64 @@ struct callbacks { } // clang-format off - int32_t db_get_i64(int32_t iterator, span buffer) {return selected().db_get_context().db_get_i64(iterator, buffer.data(), buffer.size());} - int32_t db_next_i64(int32_t iterator, wasm_ptr primary) {return selected().db_get_context().db_next_i64(iterator, *primary);} - int32_t db_previous_i64(int32_t iterator, wasm_ptr primary) {return selected().db_get_context().db_previous_i64(iterator, *primary);} - int32_t db_find_i64(uint64_t code, uint64_t scope, uint64_t table, uint64_t id) {return selected().db_get_context().db_find_i64(code, scope, table, id);} - int32_t db_lowerbound_i64(uint64_t code, uint64_t scope, uint64_t table, uint64_t id) {return selected().db_get_context().db_lowerbound_i64(code, scope, table, id);} - int32_t db_upperbound_i64(uint64_t code, uint64_t scope, uint64_t table, uint64_t id) {return selected().db_get_context().db_upperbound_i64(code, scope, table, id);} - int32_t db_end_i64(uint64_t code, uint64_t scope, uint64_t table) {return selected().db_get_context().db_end_i64(code, scope, table);} + int32_t db_get_i64(int32_t iterator, span buffer) {return selected().db_get_i64(iterator, buffer.data(), buffer.size());} + int32_t db_next_i64(int32_t iterator, wasm_ptr primary) {return selected().db_next_i64(iterator, *primary);} + int32_t db_previous_i64(int32_t iterator, wasm_ptr primary) {return selected().db_previous_i64(iterator, *primary);} + int32_t db_find_i64(uint64_t code, uint64_t scope, uint64_t table, uint64_t id) {return selected().db_find_i64(name(code), name(scope), name(table), id);} + int32_t db_lowerbound_i64(uint64_t code, uint64_t scope, uint64_t table, uint64_t id) {return selected().db_lowerbound_i64(name(code), name(scope), name(table), id);} + int32_t db_upperbound_i64(uint64_t code, uint64_t scope, uint64_t table, uint64_t id) {return selected().db_upperbound_i64(name(code), name(scope), name(table), id);} + int32_t db_end_i64(uint64_t code, uint64_t scope, uint64_t table) {return selected().db_end_i64(name(code), name(scope), name(table));} int32_t db_idx64_find_secondary(uint64_t code, uint64_t scope, uint64_t table, wasm_ptr secondary, wasm_ptr primary) { - return selected().db_get_context().db_idx64_find_secondary(code, scope, table, *secondary, *primary); + return selected().idx64.find_secondary(code, scope, table, *secondary, *primary); } int32_t db_idx64_find_primary(uint64_t code, uint64_t scope, uint64_t table, wasm_ptr secondary, uint64_t primary) { - return selected().db_get_context().db_idx64_find_primary(code, scope, table, *secondary, primary); + return selected().idx64.find_primary(code, scope, table, *secondary, primary); } int32_t db_idx64_lowerbound(uint64_t code, uint64_t scope, uint64_t table, wasm_ptr secondary, wasm_ptr primary) { - return selected().db_get_context().db_idx64_lowerbound(code, scope, table, *secondary, *primary); + return selected().idx64.lowerbound_secondary(code, scope, table, *secondary, *primary); } int32_t db_idx64_upperbound(uint64_t code, uint64_t scope, uint64_t table, wasm_ptr secondary, wasm_ptr primary) { - return selected().db_get_context().db_idx64_upperbound(code, scope, table, *secondary, *primary); + return selected().idx64.upperbound_secondary(code, scope, table, *secondary, *primary); } int32_t db_idx64_end(uint64_t code, uint64_t scope, uint64_t table) { - return selected().db_get_context().db_idx64_end(code, scope, table); + return selected().idx64.end_secondary(code, scope, table); } int32_t db_idx64_next(int32_t iterator, wasm_ptr primary) { - return selected().db_get_context().db_idx64_next(iterator, *primary); + return selected().idx64.next_secondary(iterator, *primary); } int32_t db_idx64_previous(int32_t iterator, wasm_ptr primary) { - return selected().db_get_context().db_idx64_previous(iterator, *primary); + return selected().idx64.previous_secondary(iterator, *primary); } int32_t db_idx128_find_secondary(uint64_t code, uint64_t scope, uint64_t table, wasm_ptr secondary, wasm_ptr primary) { - return selected().db_get_context().db_idx128_find_secondary(code, scope, table, *secondary, *primary); + return selected().idx128.find_secondary(code, scope, table, *secondary, *primary); } int32_t db_idx128_find_primary(uint64_t code, uint64_t scope, uint64_t table, wasm_ptr secondary, uint64_t primary) { - return selected().db_get_context().db_idx128_find_primary(code, scope, table, *secondary, primary); + return selected().idx128.find_primary(code, scope, table, *secondary, primary); } int32_t db_idx128_lowerbound(uint64_t code, uint64_t scope, uint64_t table, wasm_ptr secondary, wasm_ptr primary) { - return selected().db_get_context().db_idx128_lowerbound(code, scope, table, *secondary, *primary); + return selected().idx128.lowerbound_secondary(code, scope, table, *secondary, *primary); } int32_t db_idx128_upperbound(uint64_t code, uint64_t scope, uint64_t table, wasm_ptr secondary, wasm_ptr primary) { - return selected().db_get_context().db_idx128_upperbound(code, scope, table, *secondary, *primary); + return selected().idx128.upperbound_secondary(code, scope, table, *secondary, *primary); } int32_t db_idx128_end(uint64_t code, uint64_t scope, uint64_t table) { - return selected().db_get_context().db_idx128_end(code, scope, table); + return selected().idx128.end_secondary(code, scope, table); } int32_t db_idx128_next(int32_t iterator, wasm_ptr primary) { - return selected().db_get_context().db_idx128_next(iterator, *primary); + return selected().idx128.next_secondary(iterator, *primary); } int32_t db_idx128_previous(int32_t iterator, wasm_ptr primary) { - return selected().db_get_context().db_idx128_previous(iterator, *primary); + return selected().idx128.previous_secondary(iterator, *primary); } // DB_WRAPPERS_ARRAY_SECONDARY(idx256, 2, unsigned __int128) // DB_WRAPPERS_FLOAT_SECONDARY(idx_double, float64_t) From 4c813185cd4f4ad2a8cca7c5dd7e197551d3320e Mon Sep 17 00:00:00 2001 From: Dmytro Sydorchenko Date: Tue, 1 Feb 2022 20:47:20 -0500 Subject: [PATCH 58/62] fixing snapshot difference with previous version --- libraries/chain/db_util.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/libraries/chain/db_util.cpp b/libraries/chain/db_util.cpp index ba1bd2e88c..cb69ae2d34 100644 --- a/libraries/chain/db_util.cpp +++ b/libraries/chain/db_util.cpp @@ -35,6 +35,15 @@ namespace eosio { namespace chain { namespace db_util { void walk_index(const Util& utils, const chainbase::database& db, F&& function) { utils.walk(db, std::forward(function)); } + + template + void walk_index(const index_utils& utils, const chainbase::database& db, F&& function) {} + + template + void walk_index(const index_utils& utils, const chainbase::database& db, F&& function) {} + + template + void walk_index(const index_utils& utils, const chainbase::database& db, F&& function) {} void add_kv_table_to_snapshot(const snapshot_writer_ptr& snapshot, const chainbase::database& db) { snapshot->write_section([&db](auto& section) { From 9acabba1f831a55e7a5fa85540ff017dcd30bf28 Mon Sep 17 00:00:00 2001 From: Dmytro Sydorchenko Date: Thu, 3 Feb 2022 23:37:26 -0500 Subject: [PATCH 59/62] #11028 review concerns addressed --- libraries/chain/apply_context.cpp | 10 +- libraries/chain/backing_store/kv_context.cpp | 249 +++++++++++++++- libraries/chain/db_util.cpp | 6 +- .../include/eosio/chain/apply_context.hpp | 2 +- .../eosio/chain/backing_store/kv_context.hpp | 111 ++++--- .../backing_store/kv_context_chainbase.hpp | 276 ------------------ .../chain/include/eosio/chain/config.hpp | 4 - .../chain/include/eosio/chain/controller.hpp | 4 +- .../chain/include/eosio/chain/db_util.hpp | 8 +- .../eosio/chain/kv_chainbase_objects.hpp | 1 - .../chain/include/eosio/chain/kv_config.hpp | 3 +- .../rodeos/include/b1/rodeos/callbacks/kv.hpp | 3 +- .../include/b1/rodeos/rodeos_tables.hpp | 18 +- libraries/rodeos/include/eosio/key_value.hpp | 8 +- programs/eosio-tester/main.cpp | 3 +- tests/rodeos_utils.py | 2 +- unittests/kv_tests.cpp | 90 +++--- unittests/snapshot_tests.cpp | 2 +- unittests/test-contracts/CMakeLists.txt | 3 +- .../test-contracts/kvload/CMakeLists.txt | 6 - unittests/test-contracts/kvload/kvload.abi | 28 -- unittests/test-contracts/kvload/kvload.cpp | 78 ----- unittests/test-contracts/kvload/kvload.wasm | Bin 13896 -> 0 bytes 23 files changed, 364 insertions(+), 551 deletions(-) delete mode 100644 libraries/chain/include/eosio/chain/backing_store/kv_context_chainbase.hpp delete mode 100644 unittests/test-contracts/kvload/CMakeLists.txt delete mode 100644 unittests/test-contracts/kvload/kvload.abi delete mode 100644 unittests/test-contracts/kvload/kvload.cpp delete mode 100755 unittests/test-contracts/kvload/kvload.wasm diff --git a/libraries/chain/apply_context.cpp b/libraries/chain/apply_context.cpp index 9005a5a7c4..980059376d 100644 --- a/libraries/chain/apply_context.cpp +++ b/libraries/chain/apply_context.cpp @@ -1065,19 +1065,19 @@ int apply_context::db_end_i64( name code, name scope, name table ) { } int64_t apply_context::kv_erase(uint64_t contract, const char* key, uint32_t key_size) { - return kv_get_backing_store().kv_erase(contract, key, key_size); + return get_kv_context().kv_erase(contract, key, key_size); } int64_t apply_context::kv_set(uint64_t contract, const char* key, uint32_t key_size, const char* value, uint32_t value_size, account_name payer) { - return kv_get_backing_store().kv_set(contract, key, key_size, value, value_size, payer); + return get_kv_context().kv_set(contract, key, key_size, value, value_size, payer); } bool apply_context::kv_get(uint64_t contract, const char* key, uint32_t key_size, uint32_t& value_size) { - return kv_get_backing_store().kv_get(contract, key, key_size, value_size); + return get_kv_context().kv_get(contract, key, key_size, value_size); } uint32_t apply_context::kv_get_data(uint32_t offset, char* data, uint32_t data_size) { - return kv_get_backing_store().kv_get_data(offset, data, data_size); + return get_kv_context().kv_get_data(offset, data, data_size); } uint32_t apply_context::kv_it_create(uint64_t contract, const char* prefix, uint32_t size) { @@ -1091,7 +1091,7 @@ uint32_t apply_context::kv_it_create(uint64_t contract, const char* prefix, uint itr = kv_iterators.size(); kv_iterators.emplace_back(); } - kv_iterators[itr] = kv_get_backing_store().kv_it_create(contract, prefix, size); + kv_iterators[itr] = get_kv_context().kv_it_create(contract, prefix, size); return itr; } diff --git a/libraries/chain/backing_store/kv_context.cpp b/libraries/chain/backing_store/kv_context.cpp index 88f8f9fa74..bf99057579 100644 --- a/libraries/chain/backing_store/kv_context.cpp +++ b/libraries/chain/backing_store/kv_context.cpp @@ -5,21 +5,237 @@ namespace eosio { namespace chain { - namespace { - void kv_resource_manager_update_ram(apply_context& context, int64_t delta, const kv_resource_trace& trace, account_name payer) { - std::string event_id; - if (context.control.get_deep_mind_logger() != nullptr) { - event_id = STORAGE_EVENT_ID("${id}", ("id", fc::to_hex(trace.key.data(), trace.key.size()))); + kv_iterator::kv_iterator(chainbase::database& db, tracker_type& tracker, uint32_t& itr_count, name contract, std::vector prefix) + : db{ db }, tracker{ tracker }, itr_count(itr_count), contract{ contract }, prefix{ std::move(prefix) } { + + ++itr_count; + } + + kv_iterator::~kv_iterator() { --itr_count; } + + template + kv_it_stat kv_iterator::move_to(const It& it, uint32_t* found_key_size, uint32_t* found_value_size) { + if (it != idx.end() && it->contract == contract && + it->kv_key.size() >= prefix.size() && !memcmp(it->kv_key.data(), prefix.data(), prefix.size())) { + current = &*it; + *found_key_size = current->kv_key.size(); + *found_value_size = current->kv_value.size(); + return kv_it_stat::iterator_ok; + } else { + current = nullptr; + *found_key_size = 0; + *found_value_size = 0; + return kv_it_stat::iterator_end; + } + } + + kv_it_stat kv_iterator::kv_it_status() { + if (!current) + return kv_it_stat::iterator_end; + else if (!tracker.is_removed(*current)) + return kv_it_stat::iterator_ok; + else + return kv_it_stat::iterator_erased; + } + + int32_t kv_iterator::kv_it_compare(const kv_iterator& rhs) { + EOS_ASSERT(contract == rhs.contract, kv_bad_iter, "Incompatible key-value iterators"); + EOS_ASSERT(!current || !tracker.is_removed(*current), kv_bad_iter, "Iterator to erased element"); + EOS_ASSERT(!rhs.current || !tracker.is_removed(*rhs.current), kv_bad_iter, "Iterator to erased element"); + if (!rhs.current) { + if (!current) + return 0; + else { + return -1; } + } + if (!current) { + return 1; + } + + return compare_blob(current->kv_key, rhs.current->kv_key); + } - context.update_db_usage(payer, delta, storage_usage_trace(context.get_action_id(), std::move(event_id), "kv", trace.op_to_string())); + int32_t kv_iterator::kv_it_key_compare(const char* key, uint32_t size) { + if (!current) + return 1; + EOS_ASSERT(!tracker.is_removed(*current), kv_bad_iter, "Iterator to erased element"); + return compare_blob(current->kv_key, std::string_view{ key, size }); + } + + kv_it_stat kv_iterator::kv_it_move_to_end() { + current = nullptr; + return kv_it_stat::iterator_end; + } + + kv_it_stat kv_iterator::kv_it_next(uint32_t* found_key_size, uint32_t* found_value_size) { + if (current) { + EOS_ASSERT(!tracker.is_removed(*current), kv_bad_iter, "Iterator to erased element"); + auto it = idx.iterator_to(*current); + ++it; + return move_to(it, found_key_size, found_value_size); } + return move_to(idx.lower_bound(boost::make_tuple(contract, prefix)), found_key_size, found_value_size); } - kv_resource_manager create_kv_resource_manager(apply_context& context) { - return {&context, config::billable_size_v, &kv_resource_manager_update_ram}; + + kv_it_stat kv_iterator::kv_it_prev(uint32_t* found_key_size, uint32_t* found_value_size) { + std::decay_t it; + if (current) { + EOS_ASSERT(!tracker.is_removed(*current), kv_bad_iter, "Iterator to erased element"); + it = idx.iterator_to(*current); + } else + it = idx.upper_bound(boost::make_tuple(contract, blob_prefix{{prefix.data(), prefix.size()}})); + if (it != idx.begin()) + return move_to(--it, found_key_size, found_value_size); + current = nullptr; + *found_key_size = 0; + *found_value_size = 0; + return kv_it_stat::iterator_end; + } + + kv_it_stat kv_iterator::kv_it_lower_bound(const char* key, uint32_t size, uint32_t* found_key_size, uint32_t* found_value_size) { + auto clamped_key = std::max(std::string_view{ key, size }, std::string_view{ prefix.data(), prefix.size() }, unsigned_blob_less{}); + return move_to(idx.lower_bound(boost::make_tuple(contract, clamped_key)), found_key_size, found_value_size); + } + + kv_it_stat kv_iterator::kv_it_key(uint32_t offset, char* dest, uint32_t size, uint32_t& actual_size) { + if (!current) { + actual_size = 0; + return kv_it_stat::iterator_end; + } + EOS_ASSERT(!tracker.is_removed(*current), kv_bad_iter, "Iterator to erased element"); + if (offset < current->kv_key.size()) + memcpy(dest, current->kv_key.data() + offset, std::min((size_t)size, current->kv_key.size() - offset)); + actual_size = current->kv_key.size(); + return kv_it_stat::iterator_ok; + } + + kv_it_stat kv_iterator::kv_it_value(uint32_t offset, char* dest, uint32_t size, uint32_t& actual_size) { + if (!current) { + actual_size = 0; + return kv_it_stat::iterator_end; + } + EOS_ASSERT(!tracker.is_removed(*current), kv_bad_iter, "Iterator to erased element"); + if (offset < current->kv_value.size()) + memcpy(dest, current->kv_value.data() + offset, std::min((size_t)size, current->kv_value.size() - offset)); + actual_size = current->kv_value.size(); + return kv_it_stat::iterator_ok; + } + + std::optional kv_iterator::kv_it_payer() { + if (!current) return {}; + return current->payer; + } + + kv_context::kv_context(chainbase::database& db, name receiver, + const kv_resource_manager& resource_manager, const kv_database_config& limits) + : db{ db }, receiver{ receiver }, resource_manager{ resource_manager }, limits{limits} {} + + int64_t kv_context::kv_erase(uint64_t contract, const char* key, uint32_t key_size) { + EOS_ASSERT(name{ contract } == receiver, table_operation_not_permitted, "Can not write to this key"); + temp_data_buffer.reset(); + auto* kv = db.find(boost::make_tuple(name{ contract }, std::string_view{ key, key_size })); + if (!kv) + return 0; + const int64_t resource_delta = erase_table_usage(resource_manager, kv->payer, key, kv->kv_key.size(), kv->kv_value.size()); + + if (auto dm_logger = resource_manager._context->control.get_deep_mind_logger()) { + fc_dlog(*dm_logger, "KV_OP REM ${action_id} ${db} ${payer} ${key} ${odata}", + ("action_id", resource_manager._context->get_action_id()) + ("contract", name{ contract }) + ("payer", kv->payer) + ("key", fc::to_hex(kv->kv_key.data(), kv->kv_key.size())) + ("odata", fc::to_hex(kv->kv_value.data(), kv->kv_value.size())) + ); + } + + tracker.remove(*kv); + return resource_delta; + } + + int64_t kv_context::kv_set(uint64_t contract, const char* key, uint32_t key_size, const char* value, + uint32_t value_size, account_name payer) { + EOS_ASSERT(name{ contract } == receiver, table_operation_not_permitted, "Can not write to this key"); + EOS_ASSERT(key_size <= limits.max_key_size, kv_limit_exceeded, "Key too large"); + EOS_ASSERT(value_size <= limits.max_value_size, kv_limit_exceeded, "Value too large"); + + temp_data_buffer.reset(); + auto* kv = db.find( + boost::make_tuple(name{ contract }, std::string_view{ key, key_size })); + if (kv) { + const auto resource_delta = update_table_usage(resource_manager, kv->payer, payer, key, key_size, kv->kv_value.size(), value_size); + + if (auto dm_logger = resource_manager._context->control.get_deep_mind_logger()) { + fc_dlog(*dm_logger, "KV_OP UPD ${action_id} ${db} ${payer} ${key} ${odata}:${ndata}", + ("action_id", resource_manager._context->get_action_id()) + ("contract", name{ contract }) + ("payer", payer) + ("key", fc::to_hex(kv->kv_key.data(), kv->kv_key.size())) + ("odata", fc::to_hex(kv->kv_value.data(), kv->kv_value.size())) + ("ndata", fc::to_hex(value, value_size)) + ); + } + + db.modify(*kv, [&](auto& obj) { + obj.kv_value.assign(value, value_size); + obj.payer = payer; + }); + return resource_delta; + } else { + const int64_t resource_delta = create_table_usage(resource_manager, payer, key, key_size, value_size); + db.create([&](auto& obj) { + obj.contract = name{ contract }; + obj.kv_key.assign(key, key_size); + obj.kv_value.assign(value, value_size); + obj.payer = payer; + }); + + if (auto dm_logger = resource_manager._context->control.get_deep_mind_logger()) { + fc_dlog(*dm_logger, "KV_OP INS ${action_id} ${db} ${payer} ${key} ${ndata}", + ("action_id", resource_manager._context->get_action_id()) + ("contract", name{ contract }) + ("payer", payer) + ("key", fc::to_hex(key, key_size)) + ("ndata", fc::to_hex(value, value_size)) + ); + } + + return resource_delta; + } + } + + bool kv_context::kv_get(uint64_t contract, const char* key, uint32_t key_size, uint32_t& value_size) { + auto* kv = db.find(boost::make_tuple(name{ contract }, std::string_view{ key, key_size })); + if (kv) { + temp_data_buffer.emplace(kv->kv_value); + value_size = temp_data_buffer->size(); + return true; + } else { + temp_data_buffer.reset(); + value_size = 0; + return false; + } + } + + uint32_t kv_context::kv_get_data(uint32_t offset, char* data, uint32_t data_size) { + const char* temp = nullptr; + uint32_t temp_size = 0; + if (temp_data_buffer) { + temp = temp_data_buffer->data(); + temp_size = temp_data_buffer->size(); + } + if (offset < temp_size) + memcpy(data, temp + offset, std::min(data_size, temp_size - offset)); + return temp_size; + } + + std::unique_ptr kv_context::kv_it_create(uint64_t contract, const char* prefix, uint32_t size) { + EOS_ASSERT(num_iterators < limits.max_iterators, kv_bad_iter, "Too many iterators"); + EOS_ASSERT(size <= limits.max_key_size, kv_bad_iter, "Prefix too large"); + return std::make_unique(db, tracker, num_iterators, name{ contract }, + std::vector{ prefix, prefix + size }); } - /* int64_t kv_context::create_table_usage(kv_resource_manager& resource_manager, const account_name& payer, const char* key, const uint32_t key_size, const uint32_t value_size) { const int64_t resource_delta = (static_cast(resource_manager.billable_size) + key_size + value_size); resource_manager.update_table_usage(payer, resource_delta, kv_resource_trace(key, key_size, kv_resource_trace::operation::create)); @@ -50,5 +266,18 @@ namespace eosio { namespace chain { return resource_delta; } - */ + + namespace { + void kv_resource_manager_update_ram(apply_context& context, int64_t delta, const kv_resource_trace& trace, account_name payer) { + std::string event_id; + if (context.control.get_deep_mind_logger() != nullptr) { + event_id = STORAGE_EVENT_ID("${id}", ("id", fc::to_hex(trace.key.data(), trace.key.size()))); + } + + context.update_db_usage(payer, delta, storage_usage_trace(context.get_action_id(), std::move(event_id), "kv", trace.op_to_string())); + } + } + kv_resource_manager create_kv_resource_manager(apply_context& context) { + return {&context, config::billable_size_v, &kv_resource_manager_update_ram}; + } }} diff --git a/libraries/chain/db_util.cpp b/libraries/chain/db_util.cpp index cb69ae2d34..84452c689c 100644 --- a/libraries/chain/db_util.cpp +++ b/libraries/chain/db_util.cpp @@ -1,6 +1,6 @@ -#include #include #include +#include namespace eosio { namespace chain { namespace db_util { maybe_session::maybe_session(chainbase::database& cb_database) { @@ -81,9 +81,9 @@ namespace eosio { namespace chain { namespace db_util { std::unique_ptr create_kv_context(const controller& c, name receiver, - kv_resource_manager resource_manager, + const kv_resource_manager& resource_manager, const kv_database_config& limits) { - return create_kv_chainbase_context(c.mutable_db(), receiver, resource_manager, limits); + return std::make_unique(c.mutable_db(), receiver, resource_manager, limits); } template diff --git a/libraries/chain/include/eosio/chain/apply_context.hpp b/libraries/chain/include/eosio/chain/apply_context.hpp index 9450083f45..187341ca3c 100644 --- a/libraries/chain/include/eosio/chain/apply_context.hpp +++ b/libraries/chain/include/eosio/chain/apply_context.hpp @@ -425,7 +425,7 @@ class apply_context { int32_t kv_it_lower_bound(uint32_t itr, const char* key, uint32_t size, uint32_t* found_key_size, uint32_t* found_value_size); int32_t kv_it_key(uint32_t itr, uint32_t offset, char* dest, uint32_t size, uint32_t& actual_size); int32_t kv_it_value(uint32_t itr, uint32_t offset, char* dest, uint32_t size, uint32_t& actual_size); - kv_context& kv_get_backing_store() { + kv_context& get_kv_context() { EOS_ASSERT( kv_backing_store, action_validate_exception, "KV APIs cannot access state (null backing_store)" ); return *kv_backing_store; } diff --git a/libraries/chain/include/eosio/chain/backing_store/kv_context.hpp b/libraries/chain/include/eosio/chain/backing_store/kv_context.hpp index dacb83d087..fc351cfb28 100644 --- a/libraries/chain/include/eosio/chain/backing_store/kv_context.hpp +++ b/libraries/chain/include/eosio/chain/backing_store/kv_context.hpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -14,8 +15,6 @@ namespace eosio { namespace chain { class apply_context; - inline constexpr name kvram_id = "eosio.kvram"_n; - enum class kv_it_stat { iterator_ok = 0, // Iterator is positioned at a key-value pair iterator_erased = -1, // The key-value pair that the iterator used to be positioned at was erased @@ -23,20 +22,34 @@ namespace eosio { namespace chain { }; struct kv_iterator { - virtual ~kv_iterator() {} - - virtual bool is_kv_chainbase_context_iterator() const = 0; - virtual kv_it_stat kv_it_status() = 0; - virtual int32_t kv_it_compare(const kv_iterator& rhs) = 0; - virtual int32_t kv_it_key_compare(const char* key, uint32_t size) = 0; - virtual kv_it_stat kv_it_move_to_end() = 0; - virtual kv_it_stat kv_it_next(uint32_t* found_key_size, uint32_t* found_value_size) = 0; - virtual kv_it_stat kv_it_prev(uint32_t* found_key_size, uint32_t* found_value_size) = 0; - virtual kv_it_stat kv_it_lower_bound(const char* key, uint32_t size, - uint32_t* found_key_size, uint32_t* found_value_size) = 0; - virtual kv_it_stat kv_it_key(uint32_t offset, char* dest, uint32_t size, uint32_t& actual_size) = 0; - virtual kv_it_stat kv_it_value(uint32_t offset, char* dest, uint32_t size, uint32_t& actual_size) = 0; - virtual std::optional kv_it_payer() = 0; + using index_type = std::decay_t().get_index())>; + using tracker_type = std::decay_t().get_mutable_index().track_removed())>; + + chainbase::database& db; + const index_type& idx = db.get_index(); + tracker_type& tracker; + uint32_t& itr_count; + name contract; + std::vector prefix; + const kv_object* current = nullptr; + + kv_iterator(chainbase::database& db, tracker_type& tracker, uint32_t& itr_count, name contract, std::vector prefix); + + ~kv_iterator(); + + template + kv_it_stat move_to(const It& it, uint32_t* found_key_size, uint32_t* found_value_size); + kv_it_stat kv_it_status(); + int32_t kv_it_compare(const kv_iterator& rhs); + int32_t kv_it_key_compare(const char* key, uint32_t size); + kv_it_stat kv_it_move_to_end(); + kv_it_stat kv_it_next(uint32_t* found_key_size, uint32_t* found_value_size); + kv_it_stat kv_it_prev(uint32_t* found_key_size, uint32_t* found_value_size); + kv_it_stat kv_it_lower_bound(const char* key, uint32_t size, uint32_t* found_key_size, uint32_t* found_value_size); + kv_it_stat kv_it_key(uint32_t offset, char* dest, uint32_t size, uint32_t& actual_size); + kv_it_stat kv_it_value(uint32_t offset, char* dest, uint32_t size, uint32_t& actual_size); + + std::optional kv_it_payer(); }; struct kv_resource_trace { @@ -70,54 +83,28 @@ namespace eosio { namespace chain { kv_resource_manager create_kv_resource_manager(apply_context& context); struct kv_context { - virtual ~kv_context() {} - - virtual int64_t kv_erase(uint64_t contract, const char* key, uint32_t key_size) = 0; - virtual int64_t kv_set(uint64_t contract, const char* key, uint32_t key_size, const char* value, - uint32_t value_size, account_name payer) = 0; - virtual bool kv_get(uint64_t contract, const char* key, uint32_t key_size, uint32_t& value_size) = 0; - virtual uint32_t kv_get_data(uint32_t offset, char* data, uint32_t data_size) = 0; - - virtual std::unique_ptr kv_it_create(uint64_t contract, const char* prefix, uint32_t size) = 0; + using tracker_type = std::decay_t().get_mutable_index().track_removed())>; + chainbase::database& db; + tracker_type tracker = db.get_mutable_index().track_removed(); + name receiver; + kv_resource_manager resource_manager; + const kv_database_config& limits; + uint32_t num_iterators = 0; + std::optional temp_data_buffer; + + kv_context(chainbase::database& db, name receiver, + const kv_resource_manager& resource_manager, const kv_database_config& limits); + int64_t kv_erase(uint64_t contract, const char* key, uint32_t key_size); + int64_t kv_set(uint64_t contract, const char* key, uint32_t key_size, const char* value, + uint32_t value_size, account_name payer); + bool kv_get(uint64_t contract, const char* key, uint32_t key_size, uint32_t& value_size); + uint32_t kv_get_data(uint32_t offset, char* data, uint32_t data_size); + std::unique_ptr kv_it_create(uint64_t contract, const char* prefix, uint32_t size); protected: // Updates resource usage for payer and returns resource delta - template - int64_t create_table_usage(Resource_manager& resource_manager, const account_name& payer, const char* key, const uint32_t key_size, const uint32_t value_size) { - const int64_t resource_delta = (static_cast(resource_manager.billable_size) + key_size + value_size); - resource_manager.update_table_usage(payer, resource_delta, kv_resource_trace(key, key_size, kv_resource_trace::operation::create)); - return resource_delta; - } - - template - int64_t erase_table_usage(Resource_manager& resource_manager, const account_name& payer, const char* key, const uint32_t key_size, const uint32_t value_size) { - const int64_t resource_delta = -(static_cast(resource_manager.billable_size) + key_size + value_size); - resource_manager.update_table_usage(payer, resource_delta, kv_resource_trace(key, key_size, kv_resource_trace::operation::erase)); - return resource_delta; - } - - template - int64_t update_table_usage(Resource_manager& resource_manager, const account_name& old_payer, const account_name& new_payer, const char* key, const uint32_t key_size, const uint32_t old_value_size, const uint32_t new_value_size) { - // 64-bit arithmetic cannot overflow, because both the key and value are limited to 32-bits - const int64_t old_size = key_size + old_value_size; - const int64_t new_size = key_size + new_value_size; - const int64_t resource_delta = new_size - old_size; - - if (old_payer != new_payer) { - // refund the existing payer - resource_manager.update_table_usage(old_payer, -(old_size + resource_manager.billable_size), kv_resource_trace(key, key_size, kv_resource_trace::operation::update)); - // charge the new payer for full amount - resource_manager.update_table_usage(new_payer, new_size + resource_manager.billable_size, kv_resource_trace(key, key_size, kv_resource_trace::operation::update)); - } else if (old_size != new_size) { - // adjust delta for the existing payer - resource_manager.update_table_usage(new_payer, resource_delta, kv_resource_trace(key, key_size, kv_resource_trace::operation::update)); - } // No need for a final "else" as usage does not change - - return resource_delta; - } + int64_t create_table_usage(kv_resource_manager& resource_manager, const account_name& payer, const char* key, const uint32_t key_size, const uint32_t value_size); + int64_t erase_table_usage(kv_resource_manager& resource_manager, const account_name& payer, const char* key, const uint32_t key_size, const uint32_t value_size); + int64_t update_table_usage(kv_resource_manager& resource_manager, const account_name& old_payer, const account_name& new_payer, const char* key, const uint32_t key_size, const uint32_t old_value_size, const uint32_t new_value_size); }; - - std::unique_ptr create_kv_chainbase_context(chainbase::database& db, name receiver, - kv_resource_manager resource_manager, const kv_database_config& limits); - }} // namespace eosio::chain diff --git a/libraries/chain/include/eosio/chain/backing_store/kv_context_chainbase.hpp b/libraries/chain/include/eosio/chain/backing_store/kv_context_chainbase.hpp deleted file mode 100644 index 2e35f5fe03..0000000000 --- a/libraries/chain/include/eosio/chain/backing_store/kv_context_chainbase.hpp +++ /dev/null @@ -1,276 +0,0 @@ -#include -#include -#include -#include - -namespace eosio { namespace chain { - - struct kv_iterator_chainbase : kv_iterator { - using index_type = std::decay_t().get_index())>; - using tracker_type = std::decay_t().get_mutable_index().track_removed())>; - - chainbase::database& db; - const index_type& idx = db.get_index(); - tracker_type& tracker; - uint32_t& itr_count; - name contract; - std::vector prefix; - const kv_object* current = nullptr; - - kv_iterator_chainbase(chainbase::database& db, tracker_type& tracker, uint32_t& itr_count, name contract, std::vector prefix) - : db{ db }, tracker{ tracker }, itr_count(itr_count), contract{ contract }, prefix{ std::move(prefix) } { - - ++itr_count; - } - - ~kv_iterator_chainbase() override { --itr_count; } - - bool is_kv_chainbase_context_iterator() const override { return true; } - - template - kv_it_stat move_to(const It& it, uint32_t* found_key_size, uint32_t* found_value_size) { - if (it != idx.end() && it->contract == contract && - it->kv_key.size() >= prefix.size() && !memcmp(it->kv_key.data(), prefix.data(), prefix.size())) { - current = &*it; - *found_key_size = current->kv_key.size(); - *found_value_size = current->kv_value.size(); - return kv_it_stat::iterator_ok; - } else { - current = nullptr; - *found_key_size = 0; - *found_value_size = 0; - return kv_it_stat::iterator_end; - } - } - - kv_it_stat kv_it_status() override { - if (!current) - return kv_it_stat::iterator_end; - else if (!tracker.is_removed(*current)) - return kv_it_stat::iterator_ok; - else - return kv_it_stat::iterator_erased; - } - - int32_t kv_it_compare(const kv_iterator& rhs) override { - EOS_ASSERT(rhs.is_kv_chainbase_context_iterator(), kv_bad_iter, "Incompatible key-value iterators"); - auto& r = static_cast(rhs); - - EOS_ASSERT(contract == r.contract, kv_bad_iter, "Incompatible key-value iterators"); - EOS_ASSERT(!current || !tracker.is_removed(*current), kv_bad_iter, "Iterator to erased element"); - EOS_ASSERT(!r.current || !tracker.is_removed(*r.current), kv_bad_iter, "Iterator to erased element"); - if (!r.current) { - if (!current) - return 0; - else { - return -1; - } - } - if (!current) { - return 1; - } - - return compare_blob(current->kv_key, r.current->kv_key); - } - - int32_t kv_it_key_compare(const char* key, uint32_t size) override { - if (!current) - return 1; - EOS_ASSERT(!tracker.is_removed(*current), kv_bad_iter, "Iterator to erased element"); - return compare_blob(current->kv_key, std::string_view{ key, size }); - } - - kv_it_stat kv_it_move_to_end() override { - current = nullptr; - return kv_it_stat::iterator_end; - } - - kv_it_stat kv_it_next(uint32_t* found_key_size, uint32_t* found_value_size) override { - if (current) { - EOS_ASSERT(!tracker.is_removed(*current), kv_bad_iter, "Iterator to erased element"); - auto it = idx.iterator_to(*current); - ++it; - return move_to(it, found_key_size, found_value_size); - } - return move_to(idx.lower_bound(boost::make_tuple(contract, prefix)), found_key_size, found_value_size); - } - - kv_it_stat kv_it_prev(uint32_t* found_key_size, uint32_t* found_value_size) override { - std::decay_t it; - if (current) { - EOS_ASSERT(!tracker.is_removed(*current), kv_bad_iter, "Iterator to erased element"); - it = idx.iterator_to(*current); - } else - it = idx.upper_bound(boost::make_tuple(contract, blob_prefix{{prefix.data(), prefix.size()}})); - if (it != idx.begin()) - return move_to(--it, found_key_size, found_value_size); - current = nullptr; - *found_key_size = 0; - *found_value_size = 0; - return kv_it_stat::iterator_end; - } - - kv_it_stat kv_it_lower_bound(const char* key, uint32_t size, uint32_t* found_key_size, uint32_t* found_value_size) override { - auto clamped_key = std::max(std::string_view{ key, size }, std::string_view{ prefix.data(), prefix.size() }, unsigned_blob_less{}); - return move_to(idx.lower_bound(boost::make_tuple(contract, clamped_key)), found_key_size, found_value_size); - } - - kv_it_stat kv_it_key(uint32_t offset, char* dest, uint32_t size, uint32_t& actual_size) override { - if (!current) { - actual_size = 0; - return kv_it_stat::iterator_end; - } - EOS_ASSERT(!tracker.is_removed(*current), kv_bad_iter, "Iterator to erased element"); - if (offset < current->kv_key.size()) - memcpy(dest, current->kv_key.data() + offset, std::min((size_t)size, current->kv_key.size() - offset)); - actual_size = current->kv_key.size(); - return kv_it_stat::iterator_ok; - } - - kv_it_stat kv_it_value(uint32_t offset, char* dest, uint32_t size, uint32_t& actual_size) override { - if (!current) { - actual_size = 0; - return kv_it_stat::iterator_end; - } - EOS_ASSERT(!tracker.is_removed(*current), kv_bad_iter, "Iterator to erased element"); - if (offset < current->kv_value.size()) - memcpy(dest, current->kv_value.data() + offset, std::min((size_t)size, current->kv_value.size() - offset)); - actual_size = current->kv_value.size(); - return kv_it_stat::iterator_ok; - } - - std::optional kv_it_payer() override { - if (!current) return {}; - return current->payer; - } - }; // kv_iterator_chainbase - - template - struct kv_context_chainbase : kv_context { - using tracker_type = std::decay_t().get_mutable_index().track_removed())>; - chainbase::database& db; - tracker_type tracker = db.get_mutable_index().track_removed(); - name receiver; - Resource_manager resource_manager; - const kv_database_config& limits; - uint32_t num_iterators = 0; - std::optional temp_data_buffer; - - kv_context_chainbase(chainbase::database& db, name receiver, - Resource_manager resource_manager, const kv_database_config& limits) - : db{ db }, receiver{ receiver }, resource_manager{ resource_manager }, limits{limits} {} - - int64_t kv_erase(uint64_t contract, const char* key, uint32_t key_size) override { - EOS_ASSERT(name{ contract } == receiver, table_operation_not_permitted, "Can not write to this key"); - temp_data_buffer.reset(); - auto* kv = db.find(boost::make_tuple(name{ contract }, std::string_view{ key, key_size })); - if (!kv) - return 0; - const int64_t resource_delta = erase_table_usage(resource_manager, kv->payer, key, kv->kv_key.size(), kv->kv_value.size()); - - if (auto dm_logger = resource_manager._context->control.get_deep_mind_logger()) { - fc_dlog(*dm_logger, "KV_OP REM ${action_id} ${db} ${payer} ${key} ${odata}", - ("action_id", resource_manager._context->get_action_id()) - ("contract", name{ contract }) - ("payer", kv->payer) - ("key", fc::to_hex(kv->kv_key.data(), kv->kv_key.size())) - ("odata", fc::to_hex(kv->kv_value.data(), kv->kv_value.size())) - ); - } - - tracker.remove(*kv); - return resource_delta; - } - - int64_t kv_set(uint64_t contract, const char* key, uint32_t key_size, const char* value, - uint32_t value_size, account_name payer) override { - EOS_ASSERT(name{ contract } == receiver, table_operation_not_permitted, "Can not write to this key"); - EOS_ASSERT(key_size <= limits.max_key_size, kv_limit_exceeded, "Key too large"); - EOS_ASSERT(value_size <= limits.max_value_size, kv_limit_exceeded, "Value too large"); - - temp_data_buffer.reset(); - auto* kv = db.find( - boost::make_tuple(name{ contract }, std::string_view{ key, key_size })); - if (kv) { - const auto resource_delta = update_table_usage(resource_manager, kv->payer, payer, key, key_size, kv->kv_value.size(), value_size); - - if (auto dm_logger = resource_manager._context->control.get_deep_mind_logger()) { - fc_dlog(*dm_logger, "KV_OP UPD ${action_id} ${db} ${payer} ${key} ${odata}:${ndata}", - ("action_id", resource_manager._context->get_action_id()) - ("contract", name{ contract }) - ("payer", payer) - ("key", fc::to_hex(kv->kv_key.data(), kv->kv_key.size())) - ("odata", fc::to_hex(kv->kv_value.data(), kv->kv_value.size())) - ("ndata", fc::to_hex(value, value_size)) - ); - } - - db.modify(*kv, [&](auto& obj) { - obj.kv_value.assign(value, value_size); - obj.payer = payer; - }); - return resource_delta; - } else { - const int64_t resource_delta = create_table_usage(resource_manager, payer, key, key_size, value_size); - db.create([&](auto& obj) { - obj.contract = name{ contract }; - obj.kv_key.assign(key, key_size); - obj.kv_value.assign(value, value_size); - obj.payer = payer; - }); - - if (auto dm_logger = resource_manager._context->control.get_deep_mind_logger()) { - fc_dlog(*dm_logger, "KV_OP INS ${action_id} ${db} ${payer} ${key} ${ndata}", - ("action_id", resource_manager._context->get_action_id()) - ("contract", name{ contract }) - ("payer", payer) - ("key", fc::to_hex(key, key_size)) - ("ndata", fc::to_hex(value, value_size)) - ); - } - - return resource_delta; - } - } - - bool kv_get(uint64_t contract, const char* key, uint32_t key_size, uint32_t& value_size) override { - auto* kv = db.find(boost::make_tuple(name{ contract }, std::string_view{ key, key_size })); - if (kv) { - temp_data_buffer.emplace(kv->kv_value); - value_size = temp_data_buffer->size(); - return true; - } else { - temp_data_buffer.reset(); - value_size = 0; - return false; - } - } - - uint32_t kv_get_data(uint32_t offset, char* data, uint32_t data_size) override { - const char* temp = nullptr; - uint32_t temp_size = 0; - if (temp_data_buffer) { - temp = temp_data_buffer->data(); - temp_size = temp_data_buffer->size(); - } - if (offset < temp_size) - memcpy(data, temp + offset, std::min(data_size, temp_size - offset)); - return temp_size; - } - - std::unique_ptr kv_it_create(uint64_t contract, const char* prefix, uint32_t size) override { - EOS_ASSERT(num_iterators < limits.max_iterators, kv_bad_iter, "Too many iterators"); - EOS_ASSERT(size <= limits.max_key_size, kv_bad_iter, "Prefix too large"); - return std::make_unique(db, tracker, num_iterators, name{ contract }, - std::vector{ prefix, prefix + size }); - } - }; // kv_context_chainbase - - template - std::unique_ptr create_kv_chainbase_context(chainbase::database& db, name receiver, - Resource_manager resource_manager, const kv_database_config& limits) - { - return std::make_unique>(db,receiver, resource_manager, limits); - } - -}} // namespace eosio::chain diff --git a/libraries/chain/include/eosio/chain/config.hpp b/libraries/chain/include/eosio/chain/config.hpp index e8e1a75129..703144f671 100644 --- a/libraries/chain/include/eosio/chain/config.hpp +++ b/libraries/chain/include/eosio/chain/config.hpp @@ -90,10 +90,6 @@ const static uint32_t default_max_action_return_value_size = 256; static_assert(MAX_SIZE_OF_BYTE_ARRAYS == 20*1024*1024, "Changing MAX_SIZE_OF_BYTE_ARRAYS breaks consensus. Make sure this is expected"); -const static uint32_t default_max_kv_key_size = 1024; -const static uint32_t default_max_kv_value_size = 1024*1024; // Large enough to hold most contracts -const static uint32_t default_max_kv_iterators = 1024; - const static uint32_t default_max_wasm_mutable_global_bytes = 1024; const static uint32_t default_max_wasm_table_elements = 1024; const static uint32_t default_max_wasm_section_elements = 8192; diff --git a/libraries/chain/include/eosio/chain/controller.hpp b/libraries/chain/include/eosio/chain/controller.hpp index 0004003d0a..611f7fd52c 100644 --- a/libraries/chain/include/eosio/chain/controller.hpp +++ b/libraries/chain/include/eosio/chain/controller.hpp @@ -54,7 +54,7 @@ namespace eosio { namespace chain { struct kv_resource_manager; struct kv_database_config; namespace db_util{ - std::unique_ptr create_kv_context(const controller&, name, kv_resource_manager, const kv_database_config&); + std::unique_ptr create_kv_context(const controller&, name, const kv_resource_manager&, const kv_database_config&); }; enum class db_read_mode { @@ -398,7 +398,7 @@ namespace eosio { namespace chain { private: friend class apply_context; friend class transaction_context; - friend std::unique_ptr db_util::create_kv_context(const controller&, name, kv_resource_manager, const kv_database_config&); + friend std::unique_ptr db_util::create_kv_context(const controller&, name,const kv_resource_manager&, const kv_database_config&); chainbase::database& mutable_db()const; diff --git a/libraries/chain/include/eosio/chain/db_util.hpp b/libraries/chain/include/eosio/chain/db_util.hpp index a4c1295e27..cc3237a5ce 100644 --- a/libraries/chain/include/eosio/chain/db_util.hpp +++ b/libraries/chain/include/eosio/chain/db_util.hpp @@ -56,10 +56,10 @@ namespace eosio { namespace chain { namespace db_util { void destroy(const fc::path& p); - std::unique_ptr create_kv_context(const controller& c, - name receiver, - kv_resource_manager resource_manager, - const kv_database_config& limits); + std::unique_ptr create_kv_context(const controller& c, + name receiver, + const kv_resource_manager& resource_manager, + const kv_database_config& limits); void add_to_snapshot(const chainbase::database& db, const eosio::chain::snapshot_writer_ptr& snapshot, diff --git a/libraries/chain/include/eosio/chain/kv_chainbase_objects.hpp b/libraries/chain/include/eosio/chain/kv_chainbase_objects.hpp index eb285239a2..87cb8402b2 100644 --- a/libraries/chain/include/eosio/chain/kv_chainbase_objects.hpp +++ b/libraries/chain/include/eosio/chain/kv_chainbase_objects.hpp @@ -2,7 +2,6 @@ #include #include -#include #include #include #include diff --git a/libraries/chain/include/eosio/chain/kv_config.hpp b/libraries/chain/include/eosio/chain/kv_config.hpp index 7330fa09ac..01f722a16f 100644 --- a/libraries/chain/include/eosio/chain/kv_config.hpp +++ b/libraries/chain/include/eosio/chain/kv_config.hpp @@ -7,8 +7,7 @@ namespace eosio { namespace chain { /** * @brief limits for a kv database. - * - * Each database (ram or disk, currently) has its own limits for these parameters. + * * The key and value limits apply when adding or modifying elements. They may be reduced * below existing database entries. */ diff --git a/libraries/rodeos/include/b1/rodeos/callbacks/kv.hpp b/libraries/rodeos/include/b1/rodeos/callbacks/kv.hpp index 6705774cea..29db2ca3b7 100644 --- a/libraries/rodeos/include/b1/rodeos/callbacks/kv.hpp +++ b/libraries/rodeos/include/b1/rodeos/callbacks/kv.hpp @@ -30,8 +30,7 @@ struct kv_iterator_rocksdb { } ~kv_iterator_rocksdb() { --num_iterators; } - - bool is_kv_chainbase_context_iterator() const { return false; } + bool is_kv_rocksdb_context_iterator() const { return true; } kv_it_stat kv_it_status() { diff --git a/libraries/rodeos/include/b1/rodeos/rodeos_tables.hpp b/libraries/rodeos/include/b1/rodeos/rodeos_tables.hpp index 425a8cc41c..acb08282dd 100644 --- a/libraries/rodeos/include/b1/rodeos/rodeos_tables.hpp +++ b/libraries/rodeos/include/b1/rodeos/rodeos_tables.hpp @@ -79,7 +79,7 @@ struct block_info_kv : eosio::kv_table { } }; block_info_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { - init(state_account, eosio::name{ "block.info" }, state_database, primary_index, id_index); + init(state_account, state_database, primary_index, id_index); } }; @@ -88,7 +88,7 @@ struct global_property_kv : eosio::kv_table { [](const auto& var) { return std::vector{}; } }; global_property_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { - init(state_account, eosio::name{ "global.prop" }, state_database, primary_index); + init(state_account, state_database, primary_index); } }; @@ -98,7 +98,7 @@ struct account_kv : eosio::kv_table { } }; account_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { - init(state_account, eosio::name{ "account" }, state_database, primary_index); + init(state_account, state_database, primary_index); } }; @@ -109,7 +109,7 @@ struct account_metadata_kv : eosio::kv_table { account_metadata_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { - init(state_account, eosio::name{ "account.meta" }, state_database, primary_index); + init(state_account, state_database, primary_index); } }; @@ -122,7 +122,7 @@ struct code_kv : eosio::kv_table { }; code_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { - init(state_account, eosio::name{ "code" }, state_database, primary_index); + init(state_account, state_database, primary_index); } }; @@ -135,7 +135,7 @@ struct contract_table_kv : eosio::kv_table { }; contract_table_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { - init(state_account, eosio::name{ "contract.tab" }, state_database, primary_index); + init(state_account, state_database, primary_index); } }; @@ -150,7 +150,7 @@ struct contract_row_kv : eosio::kv_table { } }; contract_row_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { - init(state_account, eosio::name{ "contract.row" }, state_database, primary_index); + init(state_account, state_database, primary_index); } }; @@ -176,7 +176,7 @@ struct contract_index64_kv : eosio::kv_table { contract_index64_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { - init(state_account, eosio::name{ "contract.i1" }, state_database, primary_index, secondary_index); + init(state_account, state_database, primary_index, secondary_index); } }; @@ -202,7 +202,7 @@ struct contract_index128_kv : eosio::kv_table { contract_index128_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { - init(state_account, eosio::name{ "contract.i2" }, state_database, primary_index, secondary_index); + init(state_account, state_database, primary_index, secondary_index); } }; diff --git a/libraries/rodeos/include/eosio/key_value.hpp b/libraries/rodeos/include/eosio/key_value.hpp index 2f2082b85c..00f80a5b98 100644 --- a/libraries/rodeos/include/eosio/key_value.hpp +++ b/libraries/rodeos/include/eosio/key_value.hpp @@ -233,8 +233,6 @@ inline key_type make_key(T val) { } #endif -static constexpr eosio::name kv_ram = "eosio.kvram"_n; - struct default_constructor_tag; /** @@ -952,13 +950,12 @@ class kv_table { } template - void init(eosio::name contract, eosio::name table, eosio::name db, PrimaryIndex& prim_index, SecondaryIndices&... indices) { + void init(eosio::name contract, eosio::name table, PrimaryIndex& prim_index, SecondaryIndices&... indices) { validate_types(prim_index); (validate_types(indices), ...); contract_name = contract; table_name = table; - db_name = db.value; primary_index = &prim_index; primary_index->contract_name = contract_name; @@ -975,7 +972,6 @@ class kv_table { private: eosio::name contract_name; eosio::name table_name; - uint64_t db_name; eosio::name primary_index_name; @@ -992,7 +988,7 @@ class kv_table { }; -template +template class kv_singleton { kv_environment environment; diff --git a/programs/eosio-tester/main.cpp b/programs/eosio-tester/main.cpp index 36a9bb0e0c..a0b594eddd 100644 --- a/programs/eosio-tester/main.cpp +++ b/programs/eosio-tester/main.cpp @@ -37,7 +37,6 @@ using eosio::chain::digest_type; using eosio::chain::kv_bad_db_id; using eosio::chain::kv_bad_iter; using eosio::chain::kv_context; -using eosio::chain::kvram_id; using eosio::chain::protocol_feature_exception; using eosio::chain::protocol_feature_set; using eosio::chain::signed_transaction; @@ -1015,7 +1014,7 @@ struct callbacks { } kv_context& kv_get_db() { - return selected().kv_get_backing_store(); + return selected().get_kv_context(); } void kv_check_iterator(uint32_t itr) { diff --git a/tests/rodeos_utils.py b/tests/rodeos_utils.py index 47122ffefc..b5986b8dd1 100755 --- a/tests/rodeos_utils.py +++ b/tests/rodeos_utils.py @@ -324,7 +324,7 @@ def prepareLoad(self): # due to empty JSON response, which is expected pass - self.prodNode.pushMessage("eosio", "setkvparam", '[\"eosio.kvram\"]', "--permission eosio") + self.prodNode.pushMessage("eosio", "setkvparam", '[]', "--permission eosio") cmd="curl --data-binary '[\"eosio\",\"5KQwrPbwdL6PhXujxW37FSSQZ1JiwsST4cqQzDeyXtP79zkvFD3\"]' %s/v1/txn_test_gen/create_test_accounts" % (self.prodNode.endpointHttp) Utils.Print("{}".format(cmd)) try: diff --git a/unittests/kv_tests.cpp b/unittests/kv_tests.cpp index 81e83cfdb7..d8a008c23a 100644 --- a/unittests/kv_tests.cpp +++ b/unittests/kv_tests.cpp @@ -604,20 +604,20 @@ class kv_tester : public tester { void test_max_iterators() { BOOST_TEST_REQUIRE(set_kv_limits(1024, 1024, 7) == ""); // individual limits - BOOST_TEST(iterlimit({{"eosio.kvram"_n, 7, false}}) == ""); - BOOST_TEST(iterlimit({{"eosio.kvram"_n, 2000, false}}) == "Too many iterators"); + BOOST_TEST(iterlimit({{"test"_n, 7, false}}) == ""); + BOOST_TEST(iterlimit({{"test"_n, 2000, false}}) == "Too many iterators"); // erase iterators and create more - BOOST_TEST(iterlimit({{"eosio.kvram"_n, 6, false}, - {"eosio.kvram"_n, 2, true}, - {"eosio.kvram"_n, 3, false}}) == ""); - BOOST_TEST(iterlimit({{"eosio.kvram"_n, 6, false}, - {"eosio.kvram"_n, 2, true}, - {"eosio.kvram"_n, 4, false}}) == "Too many iterators"); + BOOST_TEST(iterlimit({{"test"_n, 6, false}, + {"test"_n, 2, true}, + {"test"_n, 3, false}}) == ""); + BOOST_TEST(iterlimit({{"test"_n, 6, false}, + {"test"_n, 2, true}, + {"test"_n, 4, false}}) == "Too many iterators"); // fallback limit - testing this is impractical because it uses too much memory // This many iterators would consume at least 400 GiB. - // BOOST_TEST_REQUIRE(set_kv_limits("eosio.kvram"_n, 1024, 1024, 0xFFFFFFFF) == ""); - // BOOST_TEST_REQUIRE(set_kv_limits("eosio.kvram"_n, 1024, 1024, 0xFFFFFFFF) == ""); - // BOOST_TEST(iterlimit({{"eosio.kvram"_n, 0xFFFFFFFF, false}, {"eosio.kvram"_n, 1, false}}) == "Too many iterators"); + // BOOST_TEST_REQUIRE(set_kv_limits("test"_n, 1024, 1024, 0xFFFFFFFF) == ""); + // BOOST_TEST_REQUIRE(set_kv_limits("test"_n, 1024, 1024, 0xFFFFFFFF) == ""); + // BOOST_TEST(iterlimit({{"test"_n, 0xFFFFFFFF, false}, {"test"_n, 1, false}}) == "Too many iterators"); } // Make sure that a failed transaction correctly rolls back changes to the database, @@ -736,29 +736,27 @@ BOOST_FIXTURE_TEST_CASE(kv_key_value_limit, kv_chainbase_tester) try { // } FC_LOG_AND_RETHROW() -constexpr name databases[] = { "eosio.kvram"_n }; - -BOOST_DATA_TEST_CASE_F(kv_chainbase_tester, kv_inc_dec_usage, bdata::make(databases), db) try { // +BOOST_FIXTURE_TEST_CASE(kv_inc_dec_usage, kv_chainbase_tester) try { // test_kv_inc_dec_usage(); } FC_LOG_AND_RETHROW() -BOOST_DATA_TEST_CASE_F(kv_chainbase_tester, kv_inc_usage_and_limit, bdata::make(databases), db) try { // +BOOST_FIXTURE_TEST_CASE(kv_inc_usage_and_limit, kv_chainbase_tester) try { // test_kv_inc_usage_and_limit(); } FC_LOG_AND_RETHROW() -BOOST_DATA_TEST_CASE_F(kv_chainbase_tester, kv_dec_limit_and_usage, bdata::make(databases), db) try { // +BOOST_FIXTURE_TEST_CASE(kv_dec_limit_and_usage, kv_chainbase_tester) try { // test_kv_dec_limit_and_usage(); } FC_LOG_AND_RETHROW() -BOOST_DATA_TEST_CASE_F(kv_chainbase_tester, get_data, bdata::make(databases), db) try { // +BOOST_FIXTURE_TEST_CASE(get_data, kv_chainbase_tester) try { // test_get_data(); } FC_LOG_AND_RETHROW() -BOOST_DATA_TEST_CASE_F(kv_chainbase_tester, other_contract, bdata::make(databases), db) try { // +BOOST_FIXTURE_TEST_CASE(other_contract, kv_chainbase_tester) try { // test_other_contract(); } FC_LOG_AND_RETHROW() @@ -855,16 +853,16 @@ static const char kv_notified_wast[] = R"=====( ) )====="; -BOOST_DATA_TEST_CASE_F(tester, notify, bdata::make(databases), db) { +BOOST_FIXTURE_TEST_CASE(notify, tester) { create_accounts({ "setup"_n, "notified"_n, "notify"_n }); set_code( "setup"_n, kv_setup_wast ); push_action( "eosio"_n, "setpriv"_n, "eosio"_n, mutable_variant_object()("account", "setup"_n)("is_priv", 1)); - BOOST_TEST_REQUIRE(push_action( action({}, "setup"_n, db, construct_names_payload({"notified"_n, "notify"_n})), "setup"_n.to_uint64_t() ) == ""); + BOOST_TEST_REQUIRE(push_action( action({}, "setup"_n, "ignored"_n, construct_names_payload({"notified"_n, "notify"_n})), "setup"_n.to_uint64_t() ) == ""); set_code( "notify"_n, kv_notify_wast ); set_code( "notified"_n, kv_notified_wast ); - BOOST_TEST_REQUIRE(push_action( action({}, "notify"_n, db, {}), "notify"_n.to_uint64_t() ) == ""); + BOOST_TEST_REQUIRE(push_action( action({}, "notify"_n, "ignored"_n, {}), "notify"_n.to_uint64_t() ) == ""); } // Check corner cases of alias checks for the kv_set and kv_get intrinsics @@ -908,19 +906,19 @@ static const char kv_alias_general_wast[] = R"=====( ) )====="; -BOOST_DATA_TEST_CASE_F(tester, alias, bdata::make(databases), db) { +BOOST_FIXTURE_TEST_CASE(alias, tester) { const name alias_pass_account{"alias.pass"_n}; const name alias_general_account{"alias.gen"_n}; create_accounts({ "setup"_n, alias_pass_account, alias_general_account }); set_code( "setup"_n, kv_setup_wast ); push_action( "eosio"_n, "setpriv"_n, "eosio"_n, mutable_variant_object()("account", "setup"_n)("is_priv", 1)); - BOOST_TEST_REQUIRE(push_action( action({}, "setup"_n, db, construct_names_payload({alias_pass_account, alias_general_account})), "setup"_n.to_uint64_t() ) == ""); + BOOST_TEST_REQUIRE(push_action( action({}, "setup"_n, "ignored"_n, construct_names_payload({alias_pass_account, alias_general_account})), "setup"_n.to_uint64_t() ) == ""); set_code( alias_pass_account, kv_alias_pass_wast ); set_code( alias_general_account, kv_alias_general_wast ); - BOOST_TEST_CHECK(push_action( action({}, alias_pass_account, db, {}), alias_pass_account.to_uint64_t() ) == ""); + BOOST_TEST_CHECK(push_action( action({}, alias_pass_account, "ignored"_n, {}), alias_pass_account.to_uint64_t() ) == ""); auto construct_span_payload = [](uint32_t span_start, uint32_t span_size) -> std::vector { std::vector result; @@ -936,28 +934,28 @@ BOOST_DATA_TEST_CASE_F(tester, alias, bdata::make(databases), db) { static const char* const alias_error_msg = "pointers not allowed to alias"; - BOOST_TEST_CHECK(push_action( action({}, alias_general_account, db, construct_span_payload(31, 1)), alias_general_account.to_uint64_t() ) == ""); - BOOST_TEST_CHECK(push_action( action({}, alias_general_account, db, construct_span_payload(36, 28)), alias_general_account.to_uint64_t() ) == ""); - BOOST_TEST_CHECK(push_action( action({}, alias_general_account, db, construct_span_payload(68, 60)), alias_general_account.to_uint64_t() ) == ""); - BOOST_TEST_CHECK(push_action( action({}, alias_general_account, db, construct_span_payload(132, 1)), alias_general_account.to_uint64_t() ) == ""); - - BOOST_TEST_CHECK(push_action( action({}, alias_general_account, db, construct_span_payload(31, 2)), alias_general_account.to_uint64_t() ) == alias_error_msg); - BOOST_TEST_CHECK(push_action( action({}, alias_general_account, db, construct_span_payload(32, 1)), alias_general_account.to_uint64_t() ) == alias_error_msg); - BOOST_TEST_CHECK(push_action( action({}, alias_general_account, db, construct_span_payload(33, 1)), alias_general_account.to_uint64_t() ) == alias_error_msg); - BOOST_TEST_CHECK(push_action( action({}, alias_general_account, db, construct_span_payload(35, 1)), alias_general_account.to_uint64_t() ) == alias_error_msg); - BOOST_TEST_CHECK(push_action( action({}, alias_general_account, db, construct_span_payload(35, 2)), alias_general_account.to_uint64_t() ) == alias_error_msg); - - BOOST_TEST_CHECK(push_action( action({}, alias_general_account, db, construct_span_payload(63, 2)), alias_general_account.to_uint64_t() ) == alias_error_msg); - BOOST_TEST_CHECK(push_action( action({}, alias_general_account, db, construct_span_payload(64, 1)), alias_general_account.to_uint64_t() ) == alias_error_msg); - BOOST_TEST_CHECK(push_action( action({}, alias_general_account, db, construct_span_payload(65, 1)), alias_general_account.to_uint64_t() ) == alias_error_msg); - BOOST_TEST_CHECK(push_action( action({}, alias_general_account, db, construct_span_payload(67, 1)), alias_general_account.to_uint64_t() ) == alias_error_msg); - BOOST_TEST_CHECK(push_action( action({}, alias_general_account, db, construct_span_payload(67, 2)), alias_general_account.to_uint64_t() ) == alias_error_msg); - - BOOST_TEST_CHECK(push_action( action({}, alias_general_account, db, construct_span_payload(127, 2)), alias_general_account.to_uint64_t() ) == alias_error_msg); - BOOST_TEST_CHECK(push_action( action({}, alias_general_account, db, construct_span_payload(128, 1)), alias_general_account.to_uint64_t() ) == alias_error_msg); - BOOST_TEST_CHECK(push_action( action({}, alias_general_account, db, construct_span_payload(129, 1)), alias_general_account.to_uint64_t() ) == alias_error_msg); - BOOST_TEST_CHECK(push_action( action({}, alias_general_account, db, construct_span_payload(131, 1)), alias_general_account.to_uint64_t() ) == alias_error_msg); - BOOST_TEST_CHECK(push_action( action({}, alias_general_account, db, construct_span_payload(131, 2)), alias_general_account.to_uint64_t() ) == alias_error_msg); + BOOST_TEST_CHECK(push_action( action({}, alias_general_account, "ignored"_n, construct_span_payload(31, 1)), alias_general_account.to_uint64_t() ) == ""); + BOOST_TEST_CHECK(push_action( action({}, alias_general_account, "ignored"_n, construct_span_payload(36, 28)), alias_general_account.to_uint64_t() ) == ""); + BOOST_TEST_CHECK(push_action( action({}, alias_general_account, "ignored"_n, construct_span_payload(68, 60)), alias_general_account.to_uint64_t() ) == ""); + BOOST_TEST_CHECK(push_action( action({}, alias_general_account, "ignored"_n, construct_span_payload(132, 1)), alias_general_account.to_uint64_t() ) == ""); + + BOOST_TEST_CHECK(push_action( action({}, alias_general_account, "ignored"_n, construct_span_payload(31, 2)), alias_general_account.to_uint64_t() ) == alias_error_msg); + BOOST_TEST_CHECK(push_action( action({}, alias_general_account, "ignored"_n, construct_span_payload(32, 1)), alias_general_account.to_uint64_t() ) == alias_error_msg); + BOOST_TEST_CHECK(push_action( action({}, alias_general_account, "ignored"_n, construct_span_payload(33, 1)), alias_general_account.to_uint64_t() ) == alias_error_msg); + BOOST_TEST_CHECK(push_action( action({}, alias_general_account, "ignored"_n, construct_span_payload(35, 1)), alias_general_account.to_uint64_t() ) == alias_error_msg); + BOOST_TEST_CHECK(push_action( action({}, alias_general_account, "ignored"_n, construct_span_payload(35, 2)), alias_general_account.to_uint64_t() ) == alias_error_msg); + + BOOST_TEST_CHECK(push_action( action({}, alias_general_account, "ignored"_n, construct_span_payload(63, 2)), alias_general_account.to_uint64_t() ) == alias_error_msg); + BOOST_TEST_CHECK(push_action( action({}, alias_general_account, "ignored"_n, construct_span_payload(64, 1)), alias_general_account.to_uint64_t() ) == alias_error_msg); + BOOST_TEST_CHECK(push_action( action({}, alias_general_account, "ignored"_n, construct_span_payload(65, 1)), alias_general_account.to_uint64_t() ) == alias_error_msg); + BOOST_TEST_CHECK(push_action( action({}, alias_general_account, "ignored"_n, construct_span_payload(67, 1)), alias_general_account.to_uint64_t() ) == alias_error_msg); + BOOST_TEST_CHECK(push_action( action({}, alias_general_account, "ignored"_n, construct_span_payload(67, 2)), alias_general_account.to_uint64_t() ) == alias_error_msg); + + BOOST_TEST_CHECK(push_action( action({}, alias_general_account, "ignored"_n, construct_span_payload(127, 2)), alias_general_account.to_uint64_t() ) == alias_error_msg); + BOOST_TEST_CHECK(push_action( action({}, alias_general_account, "ignored"_n, construct_span_payload(128, 1)), alias_general_account.to_uint64_t() ) == alias_error_msg); + BOOST_TEST_CHECK(push_action( action({}, alias_general_account, "ignored"_n, construct_span_payload(129, 1)), alias_general_account.to_uint64_t() ) == alias_error_msg); + BOOST_TEST_CHECK(push_action( action({}, alias_general_account, "ignored"_n, construct_span_payload(131, 1)), alias_general_account.to_uint64_t() ) == alias_error_msg); + BOOST_TEST_CHECK(push_action( action({}, alias_general_account, "ignored"_n, construct_span_payload(131, 2)), alias_general_account.to_uint64_t() ) == alias_error_msg); } BOOST_AUTO_TEST_SUITE_END() diff --git a/unittests/snapshot_tests.cpp b/unittests/snapshot_tests.cpp index 5849434182..ee916313b4 100644 --- a/unittests/snapshot_tests.cpp +++ b/unittests/snapshot_tests.cpp @@ -703,7 +703,7 @@ BOOST_AUTO_TEST_CASE_TEMPLATE(test_kv_snapshot, SNAPSHOT_SUITE, snapshot_suites) // Calling apply method which will increment the // current value stored signed_transaction trx; - trx.actions.push_back({{{contract, "active"_n}}, contract, "eosio.kvram"_n, {}}); + trx.actions.push_back({{{contract, "active"_n}}, contract, contract, {}}); chain.set_transaction_headers(trx); trx.sign(chain.get_private_key(contract, "active"), chain.control->get_chain_id()); chain.push_transaction(trx); diff --git a/unittests/test-contracts/CMakeLists.txt b/unittests/test-contracts/CMakeLists.txt index 1921ebe782..bf71f598c2 100644 --- a/unittests/test-contracts/CMakeLists.txt +++ b/unittests/test-contracts/CMakeLists.txt @@ -34,5 +34,4 @@ add_subdirectory( action_results ) add_subdirectory( wasm_config_bios ) add_subdirectory( params_test ) add_subdirectory( kv_table_test ) -add_subdirectory( kv_addr_book ) -add_subdirectory( kvload ) +add_subdirectory( kv_addr_book ) \ No newline at end of file diff --git a/unittests/test-contracts/kvload/CMakeLists.txt b/unittests/test-contracts/kvload/CMakeLists.txt deleted file mode 100644 index de21b8a61e..0000000000 --- a/unittests/test-contracts/kvload/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -if( EOSIO_COMPILE_TEST_CONTRACTS ) - add_contract( kvload kvload kvload.cpp ) -else() - configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/kvload.wasm ${CMAKE_CURRENT_BINARY_DIR}/kvload.wasm COPYONLY ) - configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/kvload.abi ${CMAKE_CURRENT_BINARY_DIR}/kvload.abi COPYONLY ) -endif() diff --git a/unittests/test-contracts/kvload/kvload.abi b/unittests/test-contracts/kvload/kvload.abi deleted file mode 100644 index 25fe028e19..0000000000 --- a/unittests/test-contracts/kvload/kvload.abi +++ /dev/null @@ -1,28 +0,0 @@ -{ - "____comment": "This file was generated with eosio-abigen. DO NOT EDIT ", - "version": "eosio::abi/1.2", - "types": [], - "structs": [ - { - "name": "setkvparam", - "base": "", - "fields": [ - { - "name": "db", - "type": "name" - } - ] - } - ], - "actions": [ - { - "name": "setkvparam", - "type": "setkvparam", - "ricardian_contract": "" - } - ], - "tables": [], - "ricardian_clauses": [], - "variants": [], - "action_results": [] -} \ No newline at end of file diff --git a/unittests/test-contracts/kvload/kvload.cpp b/unittests/test-contracts/kvload/kvload.cpp deleted file mode 100644 index d5029da717..0000000000 --- a/unittests/test-contracts/kvload/kvload.cpp +++ /dev/null @@ -1,78 +0,0 @@ -// This contract is based on cdt-develop-box. -// It is used to simulate KV load. It subscribes to txn_test_gen_plugin's -// transfer action. Every transfer in txn_test_gen_plugin calls transfer -// in this contract to perform multiple KV operations. - -#include -#include -#include -#include -#include - -struct kv_config { - uint32_t version = 0; - uint32_t keysizemax = 128; - uint32_t valsizemax = 1048576; - uint32_t itrmax = 1024; -}; - -extern "C" __attribute__((eosio_wasm_import)) void set_kv_parameters_packed(const void* params, uint32_t size); - -using namespace eosio; -using namespace std; - -class [[eosio::contract]] kvload: public eosio::contract { - public: - using contract::contract; - - [[eosio::action]] void setkvparam(name db) { - kv_config c; - set_kv_parameters_packed((const char *)&c, sizeof(c)); - } - - struct row { - uint64_t key; - std::string value; - uint64_t by_key() const { return key; } - }; - - struct table_t : eosio::kv_table { - KV_NAMED_INDEX("by.id", by_key); - - table_t(eosio::name acc, eosio::name tablename) { - init(acc, tablename, eosio::kv_ram, by_key); - } - }; - - // Called from txn_gen plugin per transaction - [[eosio::on_notify("txn.test.t::transfer")]] - uint64_t transfer(const name& from, - const name& to, - const asset& quantity, - const string& memo) - { - kvload::table_t table(_self, "kvload"_n); - uint64_t next_key = 0; - if (table.by_key.begin() != table.by_key.end()) { - auto itr = table.by_key.end(); - --itr; - next_key = itr.value().key + 1; - } - - char data[200]; - for (int i = 0; i < sizeof(data); ++i) { - data[i] = 'a'; - } - - // Amplify number of KV operations by 20 times per transaction - for (int i = 0; i < 20; ++i) { - table.put(kvload::row{.key = next_key, .value = std::string(data, sizeof(data)-1)}); - if (next_key >= 10000000) { - table.erase(kvload::row{.key = next_key - 10000000}); - } - ++next_key; - } - - return next_key; - } -}; diff --git a/unittests/test-contracts/kvload/kvload.wasm b/unittests/test-contracts/kvload/kvload.wasm deleted file mode 100755 index abd41891c7b484374917b234c4c6a2e7027bc7b5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 13896 zcmc(me~esLb;s}fF+XFLEhKT8gak;_v?*F|yyJLh z?A`V3tm8oNI{Z?EkeaHhlmex#)GGfpm1u-SQB}Dlf$#@yTA(V3Qd*%1Rc#ui{sFB@ z`uU#wW_Rphe)Nxa?V0!9{c-NO=X}q(=brnX^B-OoY_zI}9kFz7B1=Z8xtyKX{7J2ixuJl!4ZX=K~om9?dn z`GvK$E(DCB`pNFeqpJ_Pmb&VJk;6lShm$L(yWweae!hEV*j-*+J2pJ_b1qZ!Y<~XO ziIs)n+S2e?pw^t9Ke>Klc!pWdJTpujYpI$hz-VHR{bX*$FABFU=>vm|= z9iiyN$^+fO{Cz9y%Zu(Zl~0E9#qQd0u=1dbRMrY*tAp<8pqd@yTAe?-ve!|Npb?m|k?NEn8Mkv(Effjk6*8z6GOkc{lZWnLVXf<~Ob*PrSJWG= zB#EL%BTZXLJQXDv{Z-Lb>8^Bl94GPBX)R5oG)mGa;yF#@R{YAe7Oh6xw$*4G)vru8 zqVn8%muCbOo8WT}%VRiMygKqD{<&i6+Z1TpN58wLsU-*?f-|^0O-Sx|@7w(-& zU4BFM@yO-V-udVq>FnIObAS71U;6A*Uz_tWk1EZ7vF07+{GI>oV_$XeOlM1b{@RzH z{I~c2yUROSRvW7y(K&bSPyX|p|MG_$Mu;8{pF20!gATv*?5BSF15e(u+5R(M{m@@O zfBV+Bvl)MNXH)xV9Mv~U*NNlVnBl$a7m0VLyt{dlhjL@hM|Vs{`OlQ?nb?(Xd&cdR z)osi)%KTOuM){%bnJ8G%Mz2VH)XNVQ?nY7Zsdqh}oGRj*CL_8$)GLzd*cEBnI8{V9 zPR43do;gLg@=UL&QP-*a#7CZ*e$f~(o{#AUp;-JyRmqv2uNw^?+SKThSe?t?ocVZr z=HZyGG<1AHIFNL`O_Jh-QS3Q;LlkY4S9TJcNK~e`Xl_2ve^zY5x{qw?oh0*#L_y!| zU=(lIw5i`+Ac?|Fx?~oay3$R@kIWXa`o=q)$;o7r!5A9wF5j1JbLwq^g6U9teb$HS zHAuH##Cy~rF55j1NBZ2#FwDosOY-aJ8qaD#8w^TRXt2=5(_+%KV}^`NHxswHPRH$z z>hs4vPkUj`D0O0&k&)c3chyN68y<2@ zg)vIUooq$$PA@1*C(){!RIT(7dS>dth@<{h@9gVRbUa(G_^9t=45*TWFf+z+qg?6G zw|xGkmtJbzzA49M=Yon`k!YD&!)RDcV7w8?;xci|ZuFhWJ_X})^;D;s`DVUPj?AY* zR_q%-&6~TB7E_WKaUG!o`P6%=M!Yzts!>$7ZlYf!zh0tsXO3QQR^V0lKCl;n+pl9rxAF}6s*DjyTxdTa)I;jp|HwCyR za-_xYSqXWqljZjmb@8-tic0f-cl9eBS#-=zSfxxEAevn2r@%dH6 z{-ybYf#QCi_JiK?TdtRvTwzh{|*V^J~+U0XmzE9lKXzp*= z47@m{4qQ^=G_WcBj7rx442LrGoEk0LxB;}=dy3DKqOfd4ypM9xB`>N)3Gj$;d}8qBbx zWt84>AQN*tA2L;AHhaRJre02?#myGA@?4@=x`iEm|0#1Ke<3F_COa7-{+UZ!Bo^h# zdx@=rz=Y_rhv{tGHc7Cmkf_suEmkR3!14zaB-&=xaU{e+3V`YaK){RWv!S4;&5p#$ zK{+6kSU~6D9_pBWk%-YqGK<&>q>9VU3Qa98|9RvH@_V>1{2Plvzat-9V`1tfTlDZN z!2t$377XADF(i#_-YCq6WjpCxAVwG57qnO)dja`+C3d~;fClU?8sJmo$2*hgJHvgk#%4a{Mt(T;bzL%93#Bp@1;Pp( z@wTGh3iMk>f7_+#w;u0o|7qw45fjEMC}_$PWaY9zC9#(~LUh#Zq;gv9qkTppdSVlg zX~9JKkHWR~cv-JdOvRL|D5H|gf+GWp0{eri`E73!H z8CCDGe%2B_m|2TF%0Lb5KP>QciggMLo-@@t3xmNse6S3TSz&5+3B(jb`%M#NhSOTJ zf>jW9vrE(S-lQq>P!KKW$MP}(7})P*6((qA{@ctn&2KMhOsAstIL^gC^0>me7CrI6 zasF%Bm*TLsBCnLlFqB&|3~F5pp_MHfj&KmGAnBA}v&$MVxpq-2AGZBrjcpfVqNq~~ z(5#KNiFLVYJQmDDfyEpu#8=nIZ29`9vmDnIALP`8f?Rwu(wtLdAOK48}AjNI%&61l2#=RsZbfjlvv4v z>@uG`4|A_6?xN1T)Nv}H45uJO=Or=F9AXh zovc_-yQ;_-s?1g9V)bo!G5ea5-}rSiG@J0lVJlQtTcW?3tdR^qoAPOBYxE*{*}4#k z_O$CM$mPt$hsD+CAEEU55QkY9+|&}*AbyvlSD?(2f+`q)1a#I;;9!WqQ+by|?l1)t z2at`*GbQ;1;n3-ai#gI2@DlTkl->muO_jcNWqp1Mz zrkY-DO(|!t&LZWap-bu;eY!o(c79JW3FD??#(f(*kzjWX?5-tN-qvZz?zBy;(lagT zLorzkc7~wi5*o8@PI1Lpn`V&HmgdZlniEg`73RUU#>qdD zcT&t^m-dBfWBpdVi6nyuH?_5`pS8S{+B8jIY#=^z0dV6maYHUs8%J)XX3WQAli!P@ z6g!86`GHZ32bHBuW<2E*eK$G=|;5MZ<2(Ukl8QmC3YS!$vk$f0_T`%KlM0{t0#4 z{y)?$TaLpQ=vLmybUM;8Gp@W+a8phB#^7g0%EhZl5oXB2=7Z9IgmS@y1Pc`U`L*nU zesWzHl0cxmh9POBpoC4_xbbH)ez2h3;DsQVoRAMQB}?J`nv=S9Gxtk_U?9eSFP)yQXS5;{rKB&GoHbaD(WG z6}cuJ112S4H4l42<>P~l501(H1{h##-qfZ>3zib|JP~QIn=OAoq?F6&!gaHPn<;DY zfkvX;rLAS(Vl5*V9@a8;5L(F&2HRsjJRjDuaB?8;P75t%ETJQ?j4ff>H=|&JHg@lF zi3cf24d!vPHj()brC_fO^7}DiLJz7XQ+I8!o7g-U9N}?K4 z!3ZR^ps{YXjLIra!o7a&Ox($4nfP%>!6LHYuRxt#z7;ef>WNKu#4O7JgqIfzcU z+<;DSJE1vcn`Edk3j$midd&L^m|CGHXq9^gK#5$*LScDV~_P7YY zA!Ofb=_5O(p>wd=_KuW+MF)5axuICe3r{f3aH7XhN;7Pd1H}edj7a^B1=>J?4(h7h z6Esy4`Uf12*bxve9$(Kzts-kgl7QYY!5=c>1&uj{&`8Ke7bV$H1MC=sX75|PF&PJ3l;+!;IuGWxN`e z8xH(DMDTc1;zJ4ZNbDW-kv!gho(PH~_OUoTtI0zRHYzTr<2UMSzg(VDA9HztVTK_Cd|?1d?Fj&uv)aTq zI=znkX%MtyCh;Dvc`DUWyDRrFDfxRi)BY>h8fR>3;SlsG46y%Ay;8u|i9}{~@+$T? z;}-}t>yY?~EbF7faH_9kCrv0MegtrPVn?Y&(A7NZXS<2u`sHenEK} zy+{L+bhtq)r7-NAi8=&dRBymspS@CV-L%nh6!MW$vfCS{2PB>B3Lalpt^@Hp2*iUy zTP=`MoZp>&xRP?UiIzc_m=SNZ-V<6S*$BQUu#>{jM0!(ZT4oyUr~z5_+*144c_j-} zf`;>ZkRUZ73mrQHFs57`V(QS^oNNFvGiPR1o?DJ~Ea%O_hg=cHcxN}-!yp7LH^Jn6 zcx8!}Ic&ndrOGWiu+x(+`zG?v|A?1qS~E~C=&^f3~3$UY4fimFw3HHq;|;36B4g5(_54yQmTXODq-Zv z&@0F{`^7~0K=3^}1~$bsWnHFBcK%xM5sx6Rb>jbp>5)~5oge|FP)2FU%AtNSvS@Pb zjV+sv*)YsAvsC;O&`8v&iPfi@vQOtm5>MwwT1e-n(Lzhj=W^~Q?ap&2ainuo+*!sG z)CxZ&!2+u~lSYDIwPtx_8GvbSEAopXEKU6rli^e;$;HYQ$ncXYMoTwiJaZuX6E6}C{Q+kA$K z<5`O;!kTHraTpWafs||SKPHO0z8S?e&hUeoAv{5biCRk5mXJ;~%3La7=NJ(7cx!35 zj9B-iGC6X5f>SGl8yqlOPP8~o;!QjmH+~_+NO}W;J+ji96Qnosy^W!7KCags267OQ zaUm2SSVDbh@|v7nn7z27J!0dq(ICNPl3|R3$!OvY`2^OHi!zkoPou`zd8_z|k&Rv$ z#Rr9~kYR>~5MC!M8pr}D-yE_h(e~i)aD4!_m{6X!-mwh{o`k$xqP>tbDi4YPct;TV zF-YZ08y0R$M$G^rM&At6C(7vUaL*vtH|ez4{K9D3>+6LH?(Iyue#dBx#Peny^qE~n zKZL!aX(z8Z@h$pLn9vf2)9XWZB5&;?ALMy`SI3fZzoV&(U7|3&B43+Cx2PC*L3Y)V zc6dV%k^}!q2d`Tt{8RHk2Qw=*vjDK0>AkvzfRq?cisDLub-ayh2=%Rz8rpF5=s_V zmaiRp+8#Yl>xVz*vN!UBlmS0R=nnkSnqOYgI3~0Qkl$8aHHu&6XDe%T{WM+QIy{ab zyZpZ7_*hH2s}WLN7Pb3e!}F624KO^8SdOkh!7m(BK@UpbgVJmH0RyuZ2NKYU74FfU z^$h|NNg99hDVJS-qZH#G7%UCdr?<8ueXx(pn(NE`<&_8Y_~$&fUG-PfwS0EGsBT4H zp=eLl$ThfW@|ob%;*;?aiAg?Auaji2UhML-jFU_l*{m+Cq0Zx7xV(R9c{wPJ^Qr8r zw}zSPw;`UN#P}tqx-c{>ac-6E4E(EF{D)szpjN8(3b#TiKd-RK@pF@t3xfxJzx$x{ za>N=>?>z!Tte)9E#;5fI3kY;M0M7GkknUg!RJ}*kQPEl$;&t?4rOj#@n0s<_+5Qr} zoLtp(5ce893N*fZW@*iaU+W%SSzZ(a20oD>m6yH#f^O@p{1D{F_G6jI($#f=l(Vd? zF7-*H>~*s@-tC>6=d;X*V?6f}J|E-rB%f#ae4S6!aqdlg&hYspA6G)~@HNi;50uE( AQUCw| From 5a0f19b43c21031f6b847a2f8155e284b3a28867 Mon Sep 17 00:00:00 2001 From: Dmytro Sydorchenko Date: Fri, 4 Feb 2022 12:40:10 -0500 Subject: [PATCH 60/62] restoring kvload contract --- tests/rodeos_utils.py | 2 +- unittests/test-contracts/CMakeLists.txt | 3 +- unittests/test-contracts/kvload/kvload.abi | 28 +++++++ unittests/test-contracts/kvload/kvload.cpp | 78 ++++++++++++++++++++ unittests/test-contracts/kvload/kvload.wasm | Bin 0 -> 13896 bytes 5 files changed, 109 insertions(+), 2 deletions(-) create mode 100644 unittests/test-contracts/kvload/kvload.abi create mode 100644 unittests/test-contracts/kvload/kvload.cpp create mode 100755 unittests/test-contracts/kvload/kvload.wasm diff --git a/tests/rodeos_utils.py b/tests/rodeos_utils.py index b5986b8dd1..935e882b09 100755 --- a/tests/rodeos_utils.py +++ b/tests/rodeos_utils.py @@ -324,7 +324,7 @@ def prepareLoad(self): # due to empty JSON response, which is expected pass - self.prodNode.pushMessage("eosio", "setkvparam", '[]', "--permission eosio") + self.prodNode.pushMessage("eosio", "setkvparam", '[\"ignored\"]', "--permission eosio") cmd="curl --data-binary '[\"eosio\",\"5KQwrPbwdL6PhXujxW37FSSQZ1JiwsST4cqQzDeyXtP79zkvFD3\"]' %s/v1/txn_test_gen/create_test_accounts" % (self.prodNode.endpointHttp) Utils.Print("{}".format(cmd)) try: diff --git a/unittests/test-contracts/CMakeLists.txt b/unittests/test-contracts/CMakeLists.txt index bf71f598c2..1921ebe782 100644 --- a/unittests/test-contracts/CMakeLists.txt +++ b/unittests/test-contracts/CMakeLists.txt @@ -34,4 +34,5 @@ add_subdirectory( action_results ) add_subdirectory( wasm_config_bios ) add_subdirectory( params_test ) add_subdirectory( kv_table_test ) -add_subdirectory( kv_addr_book ) \ No newline at end of file +add_subdirectory( kv_addr_book ) +add_subdirectory( kvload ) diff --git a/unittests/test-contracts/kvload/kvload.abi b/unittests/test-contracts/kvload/kvload.abi new file mode 100644 index 0000000000..25fe028e19 --- /dev/null +++ b/unittests/test-contracts/kvload/kvload.abi @@ -0,0 +1,28 @@ +{ + "____comment": "This file was generated with eosio-abigen. DO NOT EDIT ", + "version": "eosio::abi/1.2", + "types": [], + "structs": [ + { + "name": "setkvparam", + "base": "", + "fields": [ + { + "name": "db", + "type": "name" + } + ] + } + ], + "actions": [ + { + "name": "setkvparam", + "type": "setkvparam", + "ricardian_contract": "" + } + ], + "tables": [], + "ricardian_clauses": [], + "variants": [], + "action_results": [] +} \ No newline at end of file diff --git a/unittests/test-contracts/kvload/kvload.cpp b/unittests/test-contracts/kvload/kvload.cpp new file mode 100644 index 0000000000..d5029da717 --- /dev/null +++ b/unittests/test-contracts/kvload/kvload.cpp @@ -0,0 +1,78 @@ +// This contract is based on cdt-develop-box. +// It is used to simulate KV load. It subscribes to txn_test_gen_plugin's +// transfer action. Every transfer in txn_test_gen_plugin calls transfer +// in this contract to perform multiple KV operations. + +#include +#include +#include +#include +#include + +struct kv_config { + uint32_t version = 0; + uint32_t keysizemax = 128; + uint32_t valsizemax = 1048576; + uint32_t itrmax = 1024; +}; + +extern "C" __attribute__((eosio_wasm_import)) void set_kv_parameters_packed(const void* params, uint32_t size); + +using namespace eosio; +using namespace std; + +class [[eosio::contract]] kvload: public eosio::contract { + public: + using contract::contract; + + [[eosio::action]] void setkvparam(name db) { + kv_config c; + set_kv_parameters_packed((const char *)&c, sizeof(c)); + } + + struct row { + uint64_t key; + std::string value; + uint64_t by_key() const { return key; } + }; + + struct table_t : eosio::kv_table { + KV_NAMED_INDEX("by.id", by_key); + + table_t(eosio::name acc, eosio::name tablename) { + init(acc, tablename, eosio::kv_ram, by_key); + } + }; + + // Called from txn_gen plugin per transaction + [[eosio::on_notify("txn.test.t::transfer")]] + uint64_t transfer(const name& from, + const name& to, + const asset& quantity, + const string& memo) + { + kvload::table_t table(_self, "kvload"_n); + uint64_t next_key = 0; + if (table.by_key.begin() != table.by_key.end()) { + auto itr = table.by_key.end(); + --itr; + next_key = itr.value().key + 1; + } + + char data[200]; + for (int i = 0; i < sizeof(data); ++i) { + data[i] = 'a'; + } + + // Amplify number of KV operations by 20 times per transaction + for (int i = 0; i < 20; ++i) { + table.put(kvload::row{.key = next_key, .value = std::string(data, sizeof(data)-1)}); + if (next_key >= 10000000) { + table.erase(kvload::row{.key = next_key - 10000000}); + } + ++next_key; + } + + return next_key; + } +}; diff --git a/unittests/test-contracts/kvload/kvload.wasm b/unittests/test-contracts/kvload/kvload.wasm new file mode 100755 index 0000000000000000000000000000000000000000..abd41891c7b484374917b234c4c6a2e7027bc7b5 GIT binary patch literal 13896 zcmc(me~esLb;s}fF+XFLEhKT8gak;_v?*F|yyJLh z?A`V3tm8oNI{Z?EkeaHhlmex#)GGfpm1u-SQB}Dlf$#@yTA(V3Qd*%1Rc#ui{sFB@ z`uU#wW_Rphe)Nxa?V0!9{c-NO=X}q(=brnX^B-OoY_zI}9kFz7B1=Z8xtyKX{7J2ixuJl!4ZX=K~om9?dn z`GvK$E(DCB`pNFeqpJ_Pmb&VJk;6lShm$L(yWweae!hEV*j-*+J2pJ_b1qZ!Y<~XO ziIs)n+S2e?pw^t9Ke>Klc!pWdJTpujYpI$hz-VHR{bX*$FABFU=>vm|= z9iiyN$^+fO{Cz9y%Zu(Zl~0E9#qQd0u=1dbRMrY*tAp<8pqd@yTAe?-ve!|Npb?m|k?NEn8Mkv(Effjk6*8z6GOkc{lZWnLVXf<~Ob*PrSJWG= zB#EL%BTZXLJQXDv{Z-Lb>8^Bl94GPBX)R5oG)mGa;yF#@R{YAe7Oh6xw$*4G)vru8 zqVn8%muCbOo8WT}%VRiMygKqD{<&i6+Z1TpN58wLsU-*?f-|^0O-Sx|@7w(-& zU4BFM@yO-V-udVq>FnIObAS71U;6A*Uz_tWk1EZ7vF07+{GI>oV_$XeOlM1b{@RzH z{I~c2yUROSRvW7y(K&bSPyX|p|MG_$Mu;8{pF20!gATv*?5BSF15e(u+5R(M{m@@O zfBV+Bvl)MNXH)xV9Mv~U*NNlVnBl$a7m0VLyt{dlhjL@hM|Vs{`OlQ?nb?(Xd&cdR z)osi)%KTOuM){%bnJ8G%Mz2VH)XNVQ?nY7Zsdqh}oGRj*CL_8$)GLzd*cEBnI8{V9 zPR43do;gLg@=UL&QP-*a#7CZ*e$f~(o{#AUp;-JyRmqv2uNw^?+SKThSe?t?ocVZr z=HZyGG<1AHIFNL`O_Jh-QS3Q;LlkY4S9TJcNK~e`Xl_2ve^zY5x{qw?oh0*#L_y!| zU=(lIw5i`+Ac?|Fx?~oay3$R@kIWXa`o=q)$;o7r!5A9wF5j1JbLwq^g6U9teb$HS zHAuH##Cy~rF55j1NBZ2#FwDosOY-aJ8qaD#8w^TRXt2=5(_+%KV}^`NHxswHPRH$z z>hs4vPkUj`D0O0&k&)c3chyN68y<2@ zg)vIUooq$$PA@1*C(){!RIT(7dS>dth@<{h@9gVRbUa(G_^9t=45*TWFf+z+qg?6G zw|xGkmtJbzzA49M=Yon`k!YD&!)RDcV7w8?;xci|ZuFhWJ_X})^;D;s`DVUPj?AY* zR_q%-&6~TB7E_WKaUG!o`P6%=M!Yzts!>$7ZlYf!zh0tsXO3QQR^V0lKCl;n+pl9rxAF}6s*DjyTxdTa)I;jp|HwCyR za-_xYSqXWqljZjmb@8-tic0f-cl9eBS#-=zSfxxEAevn2r@%dH6 z{-ybYf#QCi_JiK?TdtRvTwzh{|*V^J~+U0XmzE9lKXzp*= z47@m{4qQ^=G_WcBj7rx442LrGoEk0LxB;}=dy3DKqOfd4ypM9xB`>N)3Gj$;d}8qBbx zWt84>AQN*tA2L;AHhaRJre02?#myGA@?4@=x`iEm|0#1Ke<3F_COa7-{+UZ!Bo^h# zdx@=rz=Y_rhv{tGHc7Cmkf_suEmkR3!14zaB-&=xaU{e+3V`YaK){RWv!S4;&5p#$ zK{+6kSU~6D9_pBWk%-YqGK<&>q>9VU3Qa98|9RvH@_V>1{2Plvzat-9V`1tfTlDZN z!2t$377XADF(i#_-YCq6WjpCxAVwG57qnO)dja`+C3d~;fClU?8sJmo$2*hgJHvgk#%4a{Mt(T;bzL%93#Bp@1;Pp( z@wTGh3iMk>f7_+#w;u0o|7qw45fjEMC}_$PWaY9zC9#(~LUh#Zq;gv9qkTppdSVlg zX~9JKkHWR~cv-JdOvRL|D5H|gf+GWp0{eri`E73!H z8CCDGe%2B_m|2TF%0Lb5KP>QciggMLo-@@t3xmNse6S3TSz&5+3B(jb`%M#NhSOTJ zf>jW9vrE(S-lQq>P!KKW$MP}(7})P*6((qA{@ctn&2KMhOsAstIL^gC^0>me7CrI6 zasF%Bm*TLsBCnLlFqB&|3~F5pp_MHfj&KmGAnBA}v&$MVxpq-2AGZBrjcpfVqNq~~ z(5#KNiFLVYJQmDDfyEpu#8=nIZ29`9vmDnIALP`8f?Rwu(wtLdAOK48}AjNI%&61l2#=RsZbfjlvv4v z>@uG`4|A_6?xN1T)Nv}H45uJO=Or=F9AXh zovc_-yQ;_-s?1g9V)bo!G5ea5-}rSiG@J0lVJlQtTcW?3tdR^qoAPOBYxE*{*}4#k z_O$CM$mPt$hsD+CAEEU55QkY9+|&}*AbyvlSD?(2f+`q)1a#I;;9!WqQ+by|?l1)t z2at`*GbQ;1;n3-ai#gI2@DlTkl->muO_jcNWqp1Mz zrkY-DO(|!t&LZWap-bu;eY!o(c79JW3FD??#(f(*kzjWX?5-tN-qvZz?zBy;(lagT zLorzkc7~wi5*o8@PI1Lpn`V&HmgdZlniEg`73RUU#>qdD zcT&t^m-dBfWBpdVi6nyuH?_5`pS8S{+B8jIY#=^z0dV6maYHUs8%J)XX3WQAli!P@ z6g!86`GHZ32bHBuW<2E*eK$G=|;5MZ<2(Ukl8QmC3YS!$vk$f0_T`%KlM0{t0#4 z{y)?$TaLpQ=vLmybUM;8Gp@W+a8phB#^7g0%EhZl5oXB2=7Z9IgmS@y1Pc`U`L*nU zesWzHl0cxmh9POBpoC4_xbbH)ez2h3;DsQVoRAMQB}?J`nv=S9Gxtk_U?9eSFP)yQXS5;{rKB&GoHbaD(WG z6}cuJ112S4H4l42<>P~l501(H1{h##-qfZ>3zib|JP~QIn=OAoq?F6&!gaHPn<;DY zfkvX;rLAS(Vl5*V9@a8;5L(F&2HRsjJRjDuaB?8;P75t%ETJQ?j4ff>H=|&JHg@lF zi3cf24d!vPHj()brC_fO^7}DiLJz7XQ+I8!o7g-U9N}?K4 z!3ZR^ps{YXjLIra!o7a&Ox($4nfP%>!6LHYuRxt#z7;ef>WNKu#4O7JgqIfzcU z+<;DSJE1vcn`Edk3j$midd&L^m|CGHXq9^gK#5$*LScDV~_P7YY zA!Ofb=_5O(p>wd=_KuW+MF)5axuICe3r{f3aH7XhN;7Pd1H}edj7a^B1=>J?4(h7h z6Esy4`Uf12*bxve9$(Kzts-kgl7QYY!5=c>1&uj{&`8Ke7bV$H1MC=sX75|PF&PJ3l;+!;IuGWxN`e z8xH(DMDTc1;zJ4ZNbDW-kv!gho(PH~_OUoTtI0zRHYzTr<2UMSzg(VDA9HztVTK_Cd|?1d?Fj&uv)aTq zI=znkX%MtyCh;Dvc`DUWyDRrFDfxRi)BY>h8fR>3;SlsG46y%Ay;8u|i9}{~@+$T? z;}-}t>yY?~EbF7faH_9kCrv0MegtrPVn?Y&(A7NZXS<2u`sHenEK} zy+{L+bhtq)r7-NAi8=&dRBymspS@CV-L%nh6!MW$vfCS{2PB>B3Lalpt^@Hp2*iUy zTP=`MoZp>&xRP?UiIzc_m=SNZ-V<6S*$BQUu#>{jM0!(ZT4oyUr~z5_+*144c_j-} zf`;>ZkRUZ73mrQHFs57`V(QS^oNNFvGiPR1o?DJ~Ea%O_hg=cHcxN}-!yp7LH^Jn6 zcx8!}Ic&ndrOGWiu+x(+`zG?v|A?1qS~E~C=&^f3~3$UY4fimFw3HHq;|;36B4g5(_54yQmTXODq-Zv z&@0F{`^7~0K=3^}1~$bsWnHFBcK%xM5sx6Rb>jbp>5)~5oge|FP)2FU%AtNSvS@Pb zjV+sv*)YsAvsC;O&`8v&iPfi@vQOtm5>MwwT1e-n(Lzhj=W^~Q?ap&2ainuo+*!sG z)CxZ&!2+u~lSYDIwPtx_8GvbSEAopXEKU6rli^e;$;HYQ$ncXYMoTwiJaZuX6E6}C{Q+kA$K z<5`O;!kTHraTpWafs||SKPHO0z8S?e&hUeoAv{5biCRk5mXJ;~%3La7=NJ(7cx!35 zj9B-iGC6X5f>SGl8yqlOPP8~o;!QjmH+~_+NO}W;J+ji96Qnosy^W!7KCags267OQ zaUm2SSVDbh@|v7nn7z27J!0dq(ICNPl3|R3$!OvY`2^OHi!zkoPou`zd8_z|k&Rv$ z#Rr9~kYR>~5MC!M8pr}D-yE_h(e~i)aD4!_m{6X!-mwh{o`k$xqP>tbDi4YPct;TV zF-YZ08y0R$M$G^rM&At6C(7vUaL*vtH|ez4{K9D3>+6LH?(Iyue#dBx#Peny^qE~n zKZL!aX(z8Z@h$pLn9vf2)9XWZB5&;?ALMy`SI3fZzoV&(U7|3&B43+Cx2PC*L3Y)V zc6dV%k^}!q2d`Tt{8RHk2Qw=*vjDK0>AkvzfRq?cisDLub-ayh2=%Rz8rpF5=s_V zmaiRp+8#Yl>xVz*vN!UBlmS0R=nnkSnqOYgI3~0Qkl$8aHHu&6XDe%T{WM+QIy{ab zyZpZ7_*hH2s}WLN7Pb3e!}F624KO^8SdOkh!7m(BK@UpbgVJmH0RyuZ2NKYU74FfU z^$h|NNg99hDVJS-qZH#G7%UCdr?<8ueXx(pn(NE`<&_8Y_~$&fUG-PfwS0EGsBT4H zp=eLl$ThfW@|ob%;*;?aiAg?Auaji2UhML-jFU_l*{m+Cq0Zx7xV(R9c{wPJ^Qr8r zw}zSPw;`UN#P}tqx-c{>ac-6E4E(EF{D)szpjN8(3b#TiKd-RK@pF@t3xfxJzx$x{ za>N=>?>z!Tte)9E#;5fI3kY;M0M7GkknUg!RJ}*kQPEl$;&t?4rOj#@n0s<_+5Qr} zoLtp(5ce893N*fZW@*iaU+W%SSzZ(a20oD>m6yH#f^O@p{1D{F_G6jI($#f=l(Vd? zF7-*H>~*s@-tC>6=d;X*V?6f}J|E-rB%f#ae4S6!aqdlg&hYspA6G)~@HNi;50uE( AQUCw| literal 0 HcmV?d00001 From a86d80c589f850af50e1d2b9e0d8d2c90db416b9 Mon Sep 17 00:00:00 2001 From: Dmytro Sydorchenko Date: Fri, 4 Feb 2022 13:12:08 -0500 Subject: [PATCH 61/62] kvload cmakefile restored --- unittests/test-contracts/kvload/CMakeLists.txt | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 unittests/test-contracts/kvload/CMakeLists.txt diff --git a/unittests/test-contracts/kvload/CMakeLists.txt b/unittests/test-contracts/kvload/CMakeLists.txt new file mode 100644 index 0000000000..de21b8a61e --- /dev/null +++ b/unittests/test-contracts/kvload/CMakeLists.txt @@ -0,0 +1,6 @@ +if( EOSIO_COMPILE_TEST_CONTRACTS ) + add_contract( kvload kvload kvload.cpp ) +else() + configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/kvload.wasm ${CMAKE_CURRENT_BINARY_DIR}/kvload.wasm COPYONLY ) + configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/kvload.abi ${CMAKE_CURRENT_BINARY_DIR}/kvload.abi COPYONLY ) +endif() From f808948824ed43bda9f9663923aa86f871a24e27 Mon Sep 17 00:00:00 2001 From: Dmytro Sydorchenko Date: Fri, 4 Feb 2022 17:49:15 -0500 Subject: [PATCH 62/62] corrected previous PR --- .../rodeos/include/b1/rodeos/constants.hpp | 3 --- .../rodeos/include/b1/rodeos/rodeos_tables.hpp | 18 +++++++++--------- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/libraries/rodeos/include/b1/rodeos/constants.hpp b/libraries/rodeos/include/b1/rodeos/constants.hpp index 297dd5b0b2..79d0e96ce0 100644 --- a/libraries/rodeos/include/b1/rodeos/constants.hpp +++ b/libraries/rodeos/include/b1/rodeos/constants.hpp @@ -4,9 +4,6 @@ namespace b1::rodeos { -// Used as db parameter to kv_table, not used for anything anymore -inline constexpr eosio::name state_database{ "eosio.state" }; - // account which stores nodeos state inline constexpr eosio::name state_account{ "eosio.state" }; diff --git a/libraries/rodeos/include/b1/rodeos/rodeos_tables.hpp b/libraries/rodeos/include/b1/rodeos/rodeos_tables.hpp index acb08282dd..3bfb14fd52 100644 --- a/libraries/rodeos/include/b1/rodeos/rodeos_tables.hpp +++ b/libraries/rodeos/include/b1/rodeos/rodeos_tables.hpp @@ -79,7 +79,7 @@ struct block_info_kv : eosio::kv_table { } }; block_info_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { - init(state_account, state_database, primary_index, id_index); + init(state_account, eosio::name{ "block.info" }, primary_index, id_index); } }; @@ -88,7 +88,7 @@ struct global_property_kv : eosio::kv_table { [](const auto& var) { return std::vector{}; } }; global_property_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { - init(state_account, state_database, primary_index); + init(state_account, eosio::name{ "global.prop" }, primary_index); } }; @@ -98,7 +98,7 @@ struct account_kv : eosio::kv_table { } }; account_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { - init(state_account, state_database, primary_index); + init(state_account, eosio::name{ "account" }, primary_index); } }; @@ -109,7 +109,7 @@ struct account_metadata_kv : eosio::kv_table { account_metadata_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { - init(state_account, state_database, primary_index); + init(state_account, eosio::name{ "account.meta" }, primary_index); } }; @@ -122,7 +122,7 @@ struct code_kv : eosio::kv_table { }; code_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { - init(state_account, state_database, primary_index); + init(state_account, eosio::name{ "code" }, primary_index); } }; @@ -135,7 +135,7 @@ struct contract_table_kv : eosio::kv_table { }; contract_table_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { - init(state_account, state_database, primary_index); + init(state_account, eosio::name{ "contract.tab" }, primary_index); } }; @@ -150,7 +150,7 @@ struct contract_row_kv : eosio::kv_table { } }; contract_row_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { - init(state_account, state_database, primary_index); + init(state_account, eosio::name{ "contract.row" }, primary_index); } }; @@ -176,7 +176,7 @@ struct contract_index64_kv : eosio::kv_table { contract_index64_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { - init(state_account, state_database, primary_index, secondary_index); + init(state_account, eosio::name{ "contract.i1" }, primary_index, secondary_index); } }; @@ -202,7 +202,7 @@ struct contract_index128_kv : eosio::kv_table { contract_index128_kv(eosio::kv_environment environment) : eosio::kv_table{ std::move(environment) } { - init(state_account, state_database, primary_index, secondary_index); + init(state_account, eosio::name{ "contract.i2" }, primary_index, secondary_index); } };