Extend bootstrap related RPC commands

nano/core_test/bootstrap.cpp

@@ -270,18 +270,12 @@ TEST (bootstrap, trace_base)
 					.sign (key.prv, key.pub)
 					.work (*system.work.generate (key.pub))
 					.build ();
-	//	std::cerr << "Genesis key: " << nano::dev::genesis_key.pub.to_account () << std::endl;
-	//	std::cerr << "Key: " << key.pub.to_account () << std::endl;
-	//	std::cerr << "Genesis: " << nano::dev::genesis->hash ().to_string () << std::endl;
-	//	std::cerr << "send1: " << send1->hash ().to_string () << std::endl;
-	//	std::cerr << "receive1: " << receive1->hash ().to_string () << std::endl;
 	auto & node1 = *system.add_node ();
-	//	std::cerr << "--------------- Start ---------------\n";
+
 	ASSERT_EQ (nano::block_status::progress, node0.process (send1));
 	ASSERT_EQ (nano::block_status::progress, node0.process (receive1));
+
 	ASSERT_EQ (node1.ledger.any.receivable_end (), node1.ledger.any.receivable_upper_bound (node1.ledger.tx_begin_read (), key.pub, 0));
-	//	std::cerr << "node0: " << node0.network.endpoint () << std::endl;
-	//	std::cerr << "node1: " << node1.network.endpoint () << std::endl;
 	ASSERT_TIMELY (10s, node1.block (receive1->hash ()) != nullptr);
 }
 
@@ -571,4 +565,50 @@ TEST (bootstrap, frontier_scan_cannot_prioritize)
 	ASSERT_ALWAYS (1s, std::none_of (opens2.begin (), opens2.end (), [&node1] (auto const & block) {
 		return node1.bootstrap.prioritized (block->account ());
 	}));
+}
+
+/*
+ * Tests that bootstrap.reset() properly recovers when called during an ongoing bootstrap
+ * This mimics node restart behaviour so this also ensures that the bootstrap is able to recover from a node restart
+ */
+TEST (bootstrap, reset)
+{
+	nano::test::system system;
+
+	// Test configuration
+	int const chain_count = 10;
+	int const blocks_per_chain = 5;
+
+	nano::node_config config;
+	// Disable election activation
+	config.backlog_scan.enable = false;
+	config.priority_scheduler.enable = false;
+	config.optimistic_scheduler.enable = false;
+	config.hinted_scheduler.enable = false;
+	// Add request limits to slow down bootstrap
+	config.bootstrap.rate_limit = 30;
+
+	// Start server node
+	auto & node_server = *system.add_node (config);
+
+	// Create multiple chains of blocks
+	auto chains = nano::test::setup_chains (system, node_server, chain_count, blocks_per_chain);
+
+	int const total_blocks = node_server.block_count ();
+	int const halfway_blocks = total_blocks / 2;
+
+	// Start client node and begin bootstrap
+	auto & node_client = *system.add_node (config);
+	ASSERT_LE (node_client.block_count (), halfway_blocks); // Should not be synced yet
+
+	// Wait until bootstrap has started and processed some blocks but not all
+	// Target approximately halfway through
+	ASSERT_TIMELY (15s, node_client.block_count () >= halfway_blocks);
+	ASSERT_LT (node_client.block_count (), total_blocks);
+
+	// Reset bootstrap halfway through the process
+	node_client.bootstrap.reset ();
+
+	// Bootstrap should automatically restart and eventually sync all blocks
+	ASSERT_TIMELY_EQ (30s, node_client.block_count (), total_blocks);
 }

nano/lib/stats_enums.hpp

@@ -166,6 +166,7 @@ enum class detail
 	other,
 	drop,
 	queued,
+	reset,
 
 	// processing queue
 	queue,

nano/node/bootstrap/account_sets.cpp

@@ -19,6 +19,12 @@ nano::bootstrap::account_sets::account_sets (nano::account_sets_config const & c
 {
 }
 
+void nano::bootstrap::account_sets::reset ()
+{
+	priorities.clear ();
+	blocking.clear ();
+}
+
 void nano::bootstrap::account_sets::priority_up (nano::account const & account)
 {
 	if (account.is_zero ())

nano/node/bootstrap/account_sets.hpp

@@ -33,6 +33,8 @@ class account_sets
 public:
 	account_sets (account_sets_config const &, nano::stats &);
 
+	void reset ();
+
 	/**
 	 * If an account is not blocked, increase its priority.
 	 * If the account does not exist in priority set and is not blocked, inserts a new entry.

nano/node/bootstrap/bootstrap_service.cpp

@@ -145,6 +145,20 @@ void nano::bootstrap_service::stop ()
 	workers.stop ();
 }
 
+void nano::bootstrap_service::reset ()
+{
+	nano::lock_guard<nano::mutex> lock{ mutex };
+
+	stats.inc (nano::stat::type::bootstrap, nano::stat::detail::reset);
+	logger.info (nano::log::type::bootstrap, "Resetting bootstrap state");
+
+	accounts.reset ();
+	database_scan.reset ();
+	frontiers.reset ();
+	scoring.reset ();
+	throttle.reset ();
+}
+
 bool nano::bootstrap_service::send (std::shared_ptr<nano::transport::channel> const & channel, async_tag tag)
 {
 	debug_assert (tag.type != query_type::invalid);
@@ -1150,6 +1164,15 @@ auto nano::bootstrap_service::info () const -> nano::bootstrap::account_sets::in
 	return accounts.info ();
 }
 
+auto nano::bootstrap_service::status () const -> status_result
+{
+	nano::lock_guard<nano::mutex> lock{ mutex };
+	return {
+		.priorities = accounts.priority_size (),
+		.blocking = accounts.blocked_size (),
+	};
+}
+
 std::size_t nano::bootstrap_service::compute_throttle_size () const
 {
 	auto ledger_size = ledger.account_count ();

nano/node/bootstrap/bootstrap_service.hpp

@@ -37,21 +37,33 @@ class bootstrap_service
 	void stop ();
 
 	/**
-	 * Process `asc_pull_ack` message coming from network
+	 * Process bootstrap messages coming from the network
 	 */
 	void process (nano::asc_pull_ack const & message, std::shared_ptr<nano::transport::channel> const &);
 
+	/**
+	 * Clears priority and blocking accounts state
+	 */
+	void reset ();
+
 	std::size_t blocked_size () const;
 	std::size_t priority_size () const;
 	std::size_t score_size () const;
 
 	bool prioritized (nano::account const &) const;
 	bool blocked (nano::account const &) const;
 
-	nano::container_info container_info () const;
-
 	nano::bootstrap::account_sets::info_t info () const;
 
+	struct status_result
+	{
+		size_t priorities;
+		size_t blocking;
+	};
+	status_result status () const;
+
+	nano::container_info container_info () const;
+
 private: // Dependencies
 	bootstrap_config const & config;
 	nano::network_constants const & network_constants;

nano/node/bootstrap/database_scan.cpp

@@ -19,6 +19,17 @@ nano::bootstrap::database_scan::database_scan (nano::ledger & ledger_a) :
 {
 }
 
+void nano::bootstrap::database_scan::reset ()
+{
+	queue.clear ();
+
+	account_scanner.next = nano::account{ 0 };
+	account_scanner.completed = 0;
+
+	pending_scanner.next = nano::account{ 0 };
+	pending_scanner.completed = 0;
+}
+
 nano::account nano::bootstrap::database_scan::next (std::function<bool (nano::account const &)> const & filter)
 {
 	if (queue.empty ())

nano/node/bootstrap/database_scan.hpp

@@ -39,6 +39,8 @@ class database_scan
 	// Indicates if a full ledger iteration has taken place e.g. warmed up
 	bool warmed_up () const;
 
+	void reset ();
+
 	nano::container_info container_info () const;
 
 private: // Dependencies

nano/node/bootstrap/frontier_scan.cpp

@@ -23,6 +23,16 @@ nano::bootstrap::frontier_scan::frontier_scan (frontier_scan_config const & conf
 	release_assert (!heads.empty ());
 }
 
+void nano::bootstrap::frontier_scan::reset ()
+{
+	for (auto it = heads.begin (); it != heads.end (); ++it)
+	{
+		heads.modify (it, [] (frontier_head & head) {
+			head.reset ();
+		});
+	}
+}
+
 nano::account nano::bootstrap::frontier_scan::next ()
 {
 	auto const cutoff = std::chrono::steady_clock::now () - config.cooldown;

nano/node/bootstrap/frontier_scan.hpp

@@ -31,6 +31,8 @@ class frontier_scan
 	nano::account next ();
 	bool process (nano::account start, std::deque<std::pair<nano::account, nano::block_hash>> const & response);
 
+	void reset ();
+
 	nano::container_info container_info () const;
 
 private: // Dependencies
@@ -65,6 +67,16 @@ class frontier_scan
 		{
 			return start;
 		}
+
+		void reset ()
+		{
+			next = start;
+			candidates.clear ();
+			requests = 0;
+			completed = 0;
+			timestamp = {};
+			processed = 0;
+		}
 	};
 
 	// clang-format off

nano/node/bootstrap/peer_scoring.cpp

@@ -12,6 +12,11 @@ nano::bootstrap::peer_scoring::peer_scoring (bootstrap_config const & config_a,
 {
 }
 
+void nano::bootstrap::peer_scoring::reset ()
+{
+	scoring.clear ();
+}
+
 bool nano::bootstrap::peer_scoring::limit_exceeded (std::shared_ptr<nano::transport::channel> const & channel) const
 {
 	auto & index = scoring.get<tag_channel> ();

nano/node/bootstrap/peer_scoring.hpp

@@ -24,6 +24,8 @@ class peer_scoring
 public:
 	peer_scoring (bootstrap_config const &, nano::network_constants const &);
 
+	void reset ();
+
 	// Returns true if channel limit has been exceeded
 	bool limit_exceeded (std::shared_ptr<nano::transport::channel> const & channel) const;
 	bool try_send_message (std::shared_ptr<nano::transport::channel> const & channel);
@@ -52,10 +54,12 @@ class peer_scoring
 	{
 	public:
 		explicit peer_score (std::shared_ptr<nano::transport::channel> const &, uint64_t, uint64_t, uint64_t);
+
 		std::weak_ptr<nano::transport::channel> channel;
 		// std::weak_ptr does not provide ordering so the naked pointer is also tracked and used for ordering channels
 		// This pointer may be invalid if the channel has been destroyed
 		nano::transport::channel * channel_ptr;
+
 		// Acquire reference to the shared channel object if it is still valid
 		[[nodiscard]] std::shared_ptr<nano::transport::channel> shared () const
 		{
@@ -66,10 +70,12 @@ class peer_scoring
 			}
 			return result;
 		}
+
 		void decay ()
 		{
 			outstanding = outstanding > 0 ? outstanding - 1 : 0;
 		}
+
 		// Number of outstanding requests to a peer
 		uint64_t outstanding{ 0 };
 		uint64_t request_count_total{ 0 };

nano/node/bootstrap/throttle.cpp

@@ -8,6 +8,12 @@ nano::bootstrap::throttle::throttle (std::size_t size) :
 	debug_assert (size > 0);
 }
 
+void nano::bootstrap::throttle::reset ()
+{
+	successes_m = samples.size ();
+	std::fill (samples.begin (), samples.end (), true);
+}
+
 bool nano::bootstrap::throttle::throttled () const
 {
 	return successes_m == 0;

nano/node/bootstrap/throttle.hpp

@@ -11,17 +11,23 @@ class throttle
 public:
 	// Initialized with all true samples
 	explicit throttle (std::size_t size);
+
 	[[nodiscard]] bool throttled () const;
 	void add (bool success);
+
 	// Resizes the number of samples tracked
 	// Drops the oldest samples if the size decreases
 	// Adds false samples if the size increases
 	void resize (std::size_t size);
+
 	[[nodiscard]] std::size_t size () const;
 	[[nodiscard]] std::size_t successes () const;
 
+	void reset ();
+
 private:
 	void pop ();
+
 	// Bit set that tracks sample results. True when something was retrieved, false otherwise
 	std::deque<bool> samples;
 	std::size_t successes_m;