From 8572b973ccec985c853ff080b8bc4e87829b4740 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Wed, 9 Oct 2024 18:19:47 -0500 Subject: [PATCH 1/5] GH-284 Break out of maybe_switch_forks tight apply blocks loop every ~500ms. --- libraries/chain/controller.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index ecbc97e2d4..4aded4c599 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -4413,6 +4413,7 @@ struct controller_impl { const forked_callback_t& forked_cb, const trx_meta_cache_lookup& trx_lookup ) { auto do_maybe_switch_forks = [&](auto& forkdb) { + auto start = fc::time_point::now(); if( new_head->header.previous == chain_head.id() ) { try { apply_block( br, new_head, s, trx_lookup ); @@ -4472,6 +4473,11 @@ struct controller_impl { shutdown(); break; } + // Break every ~500ms to allow other tasks (e.g. get_info, SHiP) opportunity to run. There is a post + // for every incoming blocks; enough posted tasks to apply all blocks queued to the fork db. + if (!replaying && fc::time_point::now() - start > fc::microseconds(500)) + break; + } catch ( const std::bad_alloc& ) { throw; } catch ( const boost::interprocess::bad_alloc& ) { From 514b4799c9cd7a189753bcb765f183f070f1b786 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Wed, 9 Oct 2024 18:20:53 -0500 Subject: [PATCH 2/5] GH-284 Set replaying flag during "replay" of forkdb. Also add in 1000 block logging during replay and startup of processing of the forkdb. --- libraries/chain/controller.cpp | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index 4aded4c599..d39f4061d2 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -1689,8 +1689,6 @@ struct controller_impl { } void replay(startup_t startup) { - replaying = true; - bool replay_block_log_needed = should_replay_block_log(); auto blog_head = blog.head(); @@ -1838,8 +1836,6 @@ struct controller_impl { }; fork_db.apply(replay_fork_db); - replaying = false; - if( except_ptr ) { std::rethrow_exception( except_ptr ); } @@ -1988,6 +1984,7 @@ struct controller_impl { ilog( "chain database started with hash: ${hash}", ("hash", calculate_integrity_hash()) ); okay_to_print_integrity_hash_on_stop = true; + replaying = true; replay( startup ); // replay any irreversible and reversible blocks ahead of current head if( check_shutdown() ) return; @@ -2034,6 +2031,8 @@ struct controller_impl { fork_db.apply(finish_init); + replaying = false; + // At Leap startup, we want to provide to our local finalizers the correct safety information // to use if they don't already have one. // If we start at a block prior to the IF transition, that information will be provided when @@ -3630,8 +3629,6 @@ struct controller_impl { template void log_applied(controller::block_report& br, const BSP& bsp) const { - if (replaying) // fork_db_root_block_num not available during replay - return; fc::time_point now = fc::time_point::now(); if (now - bsp->timestamp() < fc::minutes(5) || (bsp->block_num() % 1000 == 0)) { ilog("Received block ${id}... #${n} @ ${t} signed by ${p} " // "Received" instead of "Applied" so it matches existing log output From 0434e155f732674886359d9bb22a24084891a4e2 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Wed, 9 Oct 2024 18:42:26 -0500 Subject: [PATCH 3/5] GH-284 milliseconds instead of microseconds --- libraries/chain/controller.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index d39f4061d2..fef4957217 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -4472,7 +4472,7 @@ struct controller_impl { } // Break every ~500ms to allow other tasks (e.g. get_info, SHiP) opportunity to run. There is a post // for every incoming blocks; enough posted tasks to apply all blocks queued to the fork db. - if (!replaying && fc::time_point::now() - start > fc::microseconds(500)) + if (!replaying && fc::time_point::now() - start > fc::milliseconds(500)) break; } catch ( const std::bad_alloc& ) { From 5f6273aae7b330fe71f7e837729b2fcae5b0b89b Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Thu, 10 Oct 2024 09:25:27 -0500 Subject: [PATCH 4/5] GH-284 Use scoped_set_value for replaying. --- libraries/chain/controller.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index fef4957217..ea3b43c9d9 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -1984,7 +1984,7 @@ struct controller_impl { ilog( "chain database started with hash: ${hash}", ("hash", calculate_integrity_hash()) ); okay_to_print_integrity_hash_on_stop = true; - replaying = true; + fc::scoped_set_value r(replaying, true); replay( startup ); // replay any irreversible and reversible blocks ahead of current head if( check_shutdown() ) return; @@ -2031,8 +2031,6 @@ struct controller_impl { fork_db.apply(finish_init); - replaying = false; - // At Leap startup, we want to provide to our local finalizers the correct safety information // to use if they don't already have one. // If we start at a block prior to the IF transition, that information will be provided when From 6c36b95ae8d92eb136da265bac36d88d5cb26f36 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Thu, 10 Oct 2024 09:25:58 -0500 Subject: [PATCH 5/5] GH-284 Always complete a switch_fork --- libraries/chain/controller.cpp | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/libraries/chain/controller.cpp b/libraries/chain/controller.cpp index ea3b43c9d9..bb1fe5fe52 100644 --- a/libraries/chain/controller.cpp +++ b/libraries/chain/controller.cpp @@ -4464,14 +4464,16 @@ struct controller_impl { br = controller::block_report(); bool applied = apply_block( br, bsp, bsp->is_valid() ? controller::block_status::validated : controller::block_status::complete, trx_lookup ); - if (!switch_fork && (!applied || check_shutdown())) { - shutdown(); - break; + if (!switch_fork) { // always complete a switch fork + if (!applied || check_shutdown()) { + shutdown(); + break; + } + // Break every ~500ms to allow other tasks (e.g. get_info, SHiP) opportunity to run. There is a post + // for every incoming blocks; enough posted tasks to apply all blocks queued to the fork db. + if (!replaying && fc::time_point::now() - start > fc::milliseconds(500)) + break; } - // Break every ~500ms to allow other tasks (e.g. get_info, SHiP) opportunity to run. There is a post - // for every incoming blocks; enough posted tasks to apply all blocks queued to the fork db. - if (!replaying && fc::time_point::now() - start > fc::milliseconds(500)) - break; } catch ( const std::bad_alloc& ) { throw;