From a651c4b0ab5c74233c1f1416582dcfdf885a8ffd Mon Sep 17 00:00:00 2001 From: Shamil Gadelshin Date: Wed, 26 Jun 2024 19:06:38 +0700 Subject: [PATCH 01/41] Copy build_network() function. --- Cargo.lock | 3 + crates/subspace-service/Cargo.toml | 3 + crates/subspace-service/src/lib.rs | 1 + crates/subspace-service/src/network.rs | 432 +++++++++++++++++++++++++ 4 files changed, 439 insertions(+) create mode 100644 crates/subspace-service/src/network.rs diff --git a/Cargo.lock b/Cargo.lock index 33b440eebd..c51788e5b5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13117,7 +13117,9 @@ dependencies = [ "sc-executor", "sc-informant", "sc-network", + "sc-network-light", "sc-network-sync", + "sc-network-transactions", "sc-offchain", "sc-proof-of-time", "sc-rpc", @@ -13129,6 +13131,7 @@ dependencies = [ "sc-tracing", "sc-transaction-pool", "sc-transaction-pool-api", + "sc-utils", "schnorrkel", "sp-api", "sp-block-builder", diff --git a/crates/subspace-service/Cargo.toml b/crates/subspace-service/Cargo.toml index 222a3882c6..08f7540cbc 100644 --- a/crates/subspace-service/Cargo.toml +++ b/crates/subspace-service/Cargo.toml @@ -41,7 +41,9 @@ sc-domains = { version = "0.1.0", path = "../sc-domains" } sc-executor = { git = "https://github.com/subspace/polkadot-sdk", rev = "0cbfcb0232bbf71ac5b14cc8c99bf043cec420ef" } sc-informant = { git = "https://github.com/subspace/polkadot-sdk", rev = "0cbfcb0232bbf71ac5b14cc8c99bf043cec420ef" } sc-network = { git = "https://github.com/subspace/polkadot-sdk", rev = "0cbfcb0232bbf71ac5b14cc8c99bf043cec420ef" } +sc-network-light = { git = "https://github.com/subspace/polkadot-sdk", rev = "0cbfcb0232bbf71ac5b14cc8c99bf043cec420ef" } sc-network-sync = { git = "https://github.com/subspace/polkadot-sdk", rev = "0cbfcb0232bbf71ac5b14cc8c99bf043cec420ef" } +sc-network-transactions = { git = "https://github.com/subspace/polkadot-sdk", rev = "0cbfcb0232bbf71ac5b14cc8c99bf043cec420ef" } sc-offchain = { git = "https://github.com/subspace/polkadot-sdk", rev = "0cbfcb0232bbf71ac5b14cc8c99bf043cec420ef" } sc-proof-of-time = { version = "0.1.0", path = "../sc-proof-of-time" } sc-rpc = { git = "https://github.com/subspace/polkadot-sdk", rev = "0cbfcb0232bbf71ac5b14cc8c99bf043cec420ef" } @@ -53,6 +55,7 @@ sc-telemetry = { git = "https://github.com/subspace/polkadot-sdk", rev = "0cbfcb sc-tracing = { git = "https://github.com/subspace/polkadot-sdk", rev = "0cbfcb0232bbf71ac5b14cc8c99bf043cec420ef" } sc-transaction-pool = { git = "https://github.com/subspace/polkadot-sdk", rev = "0cbfcb0232bbf71ac5b14cc8c99bf043cec420ef" } sc-transaction-pool-api = { git = "https://github.com/subspace/polkadot-sdk", rev = "0cbfcb0232bbf71ac5b14cc8c99bf043cec420ef" } +sc-utils = { git = "https://github.com/subspace/polkadot-sdk", rev = "0cbfcb0232bbf71ac5b14cc8c99bf043cec420ef" } schnorrkel = "0.11.4" sp-api = { git = "https://github.com/subspace/polkadot-sdk", rev = "0cbfcb0232bbf71ac5b14cc8c99bf043cec420ef" } sp-blockchain = { git = "https://github.com/subspace/polkadot-sdk", rev = "0cbfcb0232bbf71ac5b14cc8c99bf043cec420ef" } diff --git a/crates/subspace-service/src/lib.rs b/crates/subspace-service/src/lib.rs index 907ef0aa67..80294a9f33 100644 --- a/crates/subspace-service/src/lib.rs +++ b/crates/subspace-service/src/lib.rs @@ -32,6 +32,7 @@ mod metrics; pub mod rpc; pub mod sync_from_dsn; pub mod transaction_pool; +pub mod network; use crate::config::{ChainSyncMode, SubspaceConfiguration, SubspaceNetworking}; use crate::dsn::{create_dsn_instance, DsnConfigurationError}; diff --git a/crates/subspace-service/src/network.rs b/crates/subspace-service/src/network.rs new file mode 100644 index 0000000000..a40b8e7e62 --- /dev/null +++ b/crates/subspace-service/src/network.rs @@ -0,0 +1,432 @@ +use futures::channel::oneshot; +use futures::{pin_mut, FutureExt, StreamExt}; +use sc_client_api::{BlockBackend, BlockchainEvents, ProofProvider}; +use sc_consensus::ImportQueue; +use sc_network::config::{ExHashT, PeerStore}; +use sc_network::service::traits::RequestResponseConfig; +use sc_network::{NetworkBackend, NetworkBlock, Roles}; +use sc_network_light::light_client_requests::handler::LightClientRequestHandler; +use sc_network_sync::block_request_handler::BlockRequestHandler; +use sc_network_sync::engine::SyncingEngine; +use sc_network_sync::service::network::NetworkServiceProvider; +use sc_network_sync::state_request_handler::StateRequestHandler; +use sc_network_sync::warp_request_handler::RequestHandler as WarpSyncRequestHandler; +use sc_network_sync::{SyncingService, WarpSyncParams}; +use sc_service::config::SyncMode; +use sc_service::{ + build_system_rpc_future, BuildNetworkParams, NetworkStarter, TransactionPoolAdapter, +}; +use sc_transaction_pool_api::TransactionPool; +use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedSender}; +use sp_api::ProvideRuntimeApi; +use sp_blockchain::{HeaderBackend, HeaderMetadata}; +use sp_consensus::block_validation::{Chain, DefaultBlockAnnounceValidator}; +use sp_runtime::traits::{Block as BlockT, BlockIdTo, Header, Zero}; +use std::sync::Arc; +use tracing::{debug, warn}; + +/// Build the network service, the network status sinks and an RPC sender. +pub fn build_network( + params: BuildNetworkParams, +) -> Result< + ( + Arc, + TracingUnboundedSender>, + sc_network_transactions::TransactionsHandlerController<::Hash>, + NetworkStarter, + Arc>, + ), + sc_service::Error, +> +where + TBl: BlockT, + TCl: ProvideRuntimeApi + + HeaderMetadata + + Chain + + BlockBackend + + BlockIdTo + + ProofProvider + + HeaderBackend + + BlockchainEvents + + 'static, + TExPool: TransactionPool::Hash> + 'static, + TImpQu: ImportQueue + 'static, + TNet: NetworkBackend::Hash>, +{ + let BuildNetworkParams { + config, + mut net_config, + client, + transaction_pool, + spawn_handle, + import_queue, + block_announce_validator_builder, + warp_sync_params, + block_relay, + metrics, + } = params; + + if warp_sync_params.is_none() && config.network.sync_mode.is_warp() { + return Err("Warp sync enabled, but no warp sync provider configured.".into()); + } + + if client.requires_full_sync() { + match config.network.sync_mode { + SyncMode::LightState { .. } => { + return Err("Fast sync doesn't work for archive nodes".into()) + } + SyncMode::Warp => return Err("Warp sync doesn't work for archive nodes".into()), + SyncMode::Full => {} + } + } + + let protocol_id = config.protocol_id(); + let genesis_hash = client + .block_hash(0u32.into()) + .ok() + .flatten() + .expect("Genesis block exists; qed"); + + let block_announce_validator = if let Some(f) = block_announce_validator_builder { + f(client.clone()) + } else { + Box::new(DefaultBlockAnnounceValidator) + }; + + let (chain_sync_network_provider, chain_sync_network_handle) = NetworkServiceProvider::new(); + let (mut block_server, block_downloader, block_request_protocol_config) = match block_relay { + Some(params) => ( + params.server, + params.downloader, + params.request_response_config, + ), + None => { + // Custom protocol was not specified, use the default block handler. + // Allow both outgoing and incoming requests. + let params = BlockRequestHandler::new::( + chain_sync_network_handle.clone(), + &protocol_id, + config.chain_spec.fork_id(), + client.clone(), + config.network.default_peers_set.in_peers as usize + + config.network.default_peers_set.out_peers as usize, + ); + ( + params.server, + params.downloader, + params.request_response_config, + ) + } + }; + spawn_handle.spawn("block-request-handler", Some("networking"), async move { + block_server.run().await; + }); + + let (state_request_protocol_config, state_request_protocol_name) = { + let num_peer_hint = net_config.network_config.default_peers_set_num_full as usize + + net_config + .network_config + .default_peers_set + .reserved_nodes + .len(); + // Allow both outgoing and incoming requests. + let (handler, protocol_config) = StateRequestHandler::new::( + &protocol_id, + config.chain_spec.fork_id(), + client.clone(), + num_peer_hint, + ); + let config_name = protocol_config.protocol_name().clone(); + + spawn_handle.spawn("state-request-handler", Some("networking"), handler.run()); + (protocol_config, config_name) + }; + + let (warp_sync_protocol_config, warp_request_protocol_name) = match warp_sync_params.as_ref() { + Some(WarpSyncParams::WithProvider(warp_with_provider)) => { + // Allow both outgoing and incoming requests. + let (handler, protocol_config) = WarpSyncRequestHandler::new::<_, TNet>( + protocol_id.clone(), + genesis_hash, + config.chain_spec.fork_id(), + warp_with_provider.clone(), + ); + let config_name = protocol_config.protocol_name().clone(); + + spawn_handle.spawn( + "warp-sync-request-handler", + Some("networking"), + handler.run(), + ); + (Some(protocol_config), Some(config_name)) + } + _ => (None, None), + }; + + let light_client_request_protocol_config = { + // Allow both outgoing and incoming requests. + let (handler, protocol_config) = LightClientRequestHandler::new::( + &protocol_id, + config.chain_spec.fork_id(), + client.clone(), + ); + spawn_handle.spawn( + "light-client-request-handler", + Some("networking"), + handler.run(), + ); + protocol_config + }; + + // install request handlers to `FullNetworkConfiguration` + net_config.add_request_response_protocol(block_request_protocol_config); + net_config.add_request_response_protocol(state_request_protocol_config); + net_config.add_request_response_protocol(light_client_request_protocol_config); + + if let Some(config) = warp_sync_protocol_config { + net_config.add_request_response_protocol(config); + } + + let bitswap_config = config.network.ipfs_server.then(|| { + let (handler, config) = TNet::bitswap_server(client.clone()); + spawn_handle.spawn("bitswap-request-handler", Some("networking"), handler); + + config + }); + + // create transactions protocol and add it to the list of supported protocols of + let peer_store_handle = net_config.peer_store_handle(); + let (transactions_handler_proto, transactions_config) = + sc_network_transactions::TransactionsHandlerPrototype::new::<_, TBl, TNet>( + protocol_id.clone(), + genesis_hash, + config.chain_spec.fork_id(), + metrics.clone(), + Arc::clone(&peer_store_handle), + ); + net_config.add_notification_protocol(transactions_config); + + // Start task for `PeerStore` + let peer_store = net_config.take_peer_store(); + let peer_store_handle = peer_store.handle(); + spawn_handle.spawn("peer-store", Some("networking"), peer_store.run()); + + let (engine, sync_service, block_announce_config) = SyncingEngine::new( + Roles::from(&config.role), + client.clone(), + config + .prometheus_config + .as_ref() + .map(|config| config.registry.clone()) + .as_ref(), + metrics.clone(), + &net_config, + protocol_id.clone(), + &config.chain_spec.fork_id().map(ToOwned::to_owned), + block_announce_validator, + warp_sync_params, + chain_sync_network_handle, + import_queue.service(), + block_downloader, + state_request_protocol_name, + warp_request_protocol_name, + Arc::clone(&peer_store_handle), + config.network.force_synced, + )?; + let sync_service_import_queue = sync_service.clone(); + let sync_service = Arc::new(sync_service); + + let genesis_hash = client + .hash(Zero::zero()) + .ok() + .flatten() + .expect("Genesis block exists; qed"); + let network_params = sc_network::config::Params::::Hash, TNet> { + role: config.role.clone(), + executor: { + let spawn_handle = Clone::clone(&spawn_handle); + Box::new(move |fut| { + spawn_handle.spawn("libp2p-node", Some("networking"), fut); + }) + }, + network_config: net_config, + genesis_hash, + protocol_id: protocol_id.clone(), + fork_id: config.chain_spec.fork_id().map(ToOwned::to_owned), + metrics_registry: config + .prometheus_config + .as_ref() + .map(|config| config.registry.clone()), + block_announce_config, + bitswap_config, + notification_metrics: metrics, + }; + + let has_bootnodes = !network_params + .network_config + .network_config + .boot_nodes + .is_empty(); + let network_mut = TNet::new(network_params)?; + let network = network_mut.network_service().clone(); + + let (tx_handler, tx_handler_controller) = transactions_handler_proto.build( + network.clone(), + sync_service.clone(), + Arc::new(TransactionPoolAdapter::new( + transaction_pool, + client.clone(), + )), + config + .prometheus_config + .as_ref() + .map(|config| &config.registry), + )?; + spawn_handle.spawn_blocking( + "network-transactions-handler", + Some("networking"), + tx_handler.run(), + ); + + spawn_handle.spawn_blocking( + "chain-sync-network-service-provider", + Some("networking"), + chain_sync_network_provider.run(Arc::new(network.clone())), + ); + spawn_handle.spawn( + "import-queue", + None, + import_queue.run(Box::new(sync_service_import_queue)), + ); + spawn_handle.spawn_blocking("syncing", None, engine.run()); + + let (system_rpc_tx, system_rpc_rx) = tracing_unbounded("mpsc_system_rpc", 10_000); + spawn_handle.spawn( + "system-rpc-handler", + Some("networking"), + build_system_rpc_future::<_, _, ::Hash>( + config.role.clone(), + network_mut.network_service(), + sync_service.clone(), + client.clone(), + system_rpc_rx, + has_bootnodes, + ), + ); + + let future = build_network_future::<_, _, ::Hash, _>( + network_mut, + client, + sync_service.clone(), + config.announce_block, + ); + + // TODO: Normally, one is supposed to pass a list of notifications protocols supported by the + // node through the `NetworkConfiguration` struct. But because this function doesn't know in + // advance which components, such as GrandPa or Polkadot, will be plugged on top of the + // service, it is unfortunately not possible to do so without some deep refactoring. To + // bypass this problem, the `NetworkService` provides a `register_notifications_protocol` + // method that can be called even after the network has been initialized. However, we want to + // avoid the situation where `register_notifications_protocol` is called *after* the network + // actually connects to other peers. For this reason, we delay the process of the network + // future until the user calls `NetworkStarter::start_network`. + // + // This entire hack should eventually be removed in favour of passing the list of protocols + // through the configuration. + // + // See also https://github.com/paritytech/substrate/issues/6827 + let (network_start_tx, network_start_rx) = oneshot::channel(); + + // The network worker is responsible for gathering all network messages and processing + // them. This is quite a heavy task, and at the time of the writing of this comment it + // frequently happens that this future takes several seconds or in some situations + // even more than a minute until it has processed its entire queue. This is clearly an + // issue, and ideally we would like to fix the network future to take as little time as + // possible, but we also take the extra harm-prevention measure to execute the networking + // future using `spawn_blocking`. + spawn_handle.spawn_blocking("network-worker", Some("networking"), async move { + if network_start_rx.await.is_err() { + warn!("The NetworkStart returned as part of `build_network` has been silently dropped"); + // This `return` might seem unnecessary, but we don't want to make it look like + // everything is working as normal even though the user is clearly misusing the API. + return; + } + + future.await + }); + + Ok(( + network, + system_rpc_tx, + tx_handler_controller, + NetworkStarter::new(network_start_tx), + sync_service.clone(), + )) +} + +/// Builds a future that continuously polls the network. +async fn build_network_future< + B: BlockT, + C: BlockchainEvents + + HeaderBackend + + BlockBackend + + HeaderMetadata + + ProofProvider + + Send + + Sync + + 'static, + H: ExHashT, + N: NetworkBackend::Hash>, +>( + network: N, + client: Arc, + sync_service: Arc>, + announce_imported_blocks: bool, +) { + let mut imported_blocks_stream = client.import_notification_stream().fuse(); + + // Stream of finalized blocks reported by the client. + let mut finality_notification_stream = client.finality_notification_stream().fuse(); + + let network_run = network.run().fuse(); + pin_mut!(network_run); + + loop { + futures::select! { + // List of blocks that the client has imported. + notification = imported_blocks_stream.next() => { + let notification = match notification { + Some(n) => n, + // If this stream is shut down, that means the client has shut down, and the + // most appropriate thing to do for the network future is to shut down too. + None => { + debug!("Block import stream has terminated, shutting down the network future."); + return + }, + }; + + if announce_imported_blocks { + sync_service.announce_block(notification.hash, None); + } + + if notification.is_new_best { + sync_service.new_best_block_imported( + notification.hash, + *notification.header.number(), + ); + } + } + + // List of blocks that the client has finalized. + notification = finality_notification_stream.select_next_some() => { + sync_service.on_block_finalized(notification.hash, notification.header); + } + + // Drive the network. Shut down the network future if `NetworkWorker` has terminated. + _ = network_run => { + debug!("`NetworkWorker` has terminated, shutting down the network future."); + return + } + } + } +} From feeff3fa7710fa1660d8d4cd9576eb6a0b8083c6 Mon Sep 17 00:00:00 2001 From: Shamil Gadelshin Date: Thu, 27 Jun 2024 14:50:06 +0700 Subject: [PATCH 02/41] Add mmr request handler --- Cargo.lock | 3 + crates/subspace-service/Cargo.toml | 3 + .../src/mmr/request_handler.rs | 296 ++++++++++++++++++ 3 files changed, 302 insertions(+) create mode 100644 crates/subspace-service/src/mmr/request_handler.rs diff --git a/Cargo.lock b/Cargo.lock index c51788e5b5..c81e388cc5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13090,6 +13090,8 @@ dependencies = [ name = "subspace-service" version = "0.1.0" dependencies = [ + "array-bytes", + "async-channel 1.9.0", "async-trait", "cross-domain-message-gossip", "domain-runtime-primitives", @@ -13132,6 +13134,7 @@ dependencies = [ "sc-transaction-pool", "sc-transaction-pool-api", "sc-utils", + "schnellru", "schnorrkel", "sp-api", "sp-block-builder", diff --git a/crates/subspace-service/Cargo.toml b/crates/subspace-service/Cargo.toml index 08f7540cbc..877a23ab0b 100644 --- a/crates/subspace-service/Cargo.toml +++ b/crates/subspace-service/Cargo.toml @@ -16,6 +16,8 @@ include = [ targets = ["x86_64-unknown-linux-gnu"] [dependencies] +array-bytes = "6.2.2" +async-channel = "1.8.0" async-trait = "0.1.80" cross-domain-message-gossip = { version = "0.1.0", path = "../../domains/client/cross-domain-message-gossip" } domain-runtime-primitives = { version = "0.1.0", path = "../../domains/primitives/runtime" } @@ -56,6 +58,7 @@ sc-tracing = { git = "https://github.com/subspace/polkadot-sdk", rev = "0cbfcb02 sc-transaction-pool = { git = "https://github.com/subspace/polkadot-sdk", rev = "0cbfcb0232bbf71ac5b14cc8c99bf043cec420ef" } sc-transaction-pool-api = { git = "https://github.com/subspace/polkadot-sdk", rev = "0cbfcb0232bbf71ac5b14cc8c99bf043cec420ef" } sc-utils = { git = "https://github.com/subspace/polkadot-sdk", rev = "0cbfcb0232bbf71ac5b14cc8c99bf043cec420ef" } +schnellru = "0.2.1" schnorrkel = "0.11.4" sp-api = { git = "https://github.com/subspace/polkadot-sdk", rev = "0cbfcb0232bbf71ac5b14cc8c99bf043cec420ef" } sp-blockchain = { git = "https://github.com/subspace/polkadot-sdk", rev = "0cbfcb0232bbf71ac5b14cc8c99bf043cec420ef" } diff --git a/crates/subspace-service/src/mmr/request_handler.rs b/crates/subspace-service/src/mmr/request_handler.rs new file mode 100644 index 0000000000..fc7ddb6a51 --- /dev/null +++ b/crates/subspace-service/src/mmr/request_handler.rs @@ -0,0 +1,296 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Substrate. + +// Substrate is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Substrate is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Substrate. If not, see . + +use crate::mmr::get_offchain_key; +use futures::channel::oneshot; +use futures::stream::StreamExt; +use parity_scale_codec::{Decode, Encode}; +use sc_client_api::{BlockBackend, ProofProvider}; +use sc_network::config::ProtocolId; +use sc_network::request_responses::{IncomingRequest, OutgoingResponse}; +use sc_network::{NetworkBackend, PeerId}; +use schnellru::{ByLength, LruMap}; +use sp_core::offchain::storage::OffchainDb; +use sp_core::offchain::{DbExternalities, OffchainStorage, StorageKind}; +use sp_runtime::codec; +use sp_runtime::traits::Block as BlockT; +use std::collections::BTreeMap; +use std::hash::{Hash, Hasher}; +use std::marker::PhantomData; +use std::sync::Arc; +use std::time::Duration; +use subspace_core_primitives::BlockNumber; +use tracing::{debug, error, trace}; + +const MAX_NUMBER_OF_SAME_REQUESTS_PER_PEER: usize = 2; + +/// Defines max items per request +pub const MAX_MMR_ITEMS: u32 = 20000; + +mod rep { + use sc_network::ReputationChange as Rep; + + /// Reputation change when a peer sent us the same request multiple times. + pub const SAME_REQUEST: Rep = Rep::new(i32::MIN, "Same state request multiple times"); +} + +/// Generates a `RequestResponseProtocolConfig` for the state request protocol, refusing incoming +/// requests. +pub fn generate_protocol_config< + Hash: AsRef<[u8]>, + B: BlockT, + N: NetworkBackend::Hash>, +>( + _: &ProtocolId, + genesis_hash: Hash, + fork_id: Option<&str>, + inbound_queue: async_channel::Sender, +) -> N::RequestResponseProtocolConfig { + N::request_response_config( + generate_protocol_name(genesis_hash, fork_id).into(), + Vec::new(), + 1024 * 1024, + 16 * 1024 * 1024, + Duration::from_secs(40), + Some(inbound_queue), + ) +} + +/// Generate the state protocol name from the genesis hash and fork id. +pub fn generate_protocol_name>( + genesis_hash: Hash, + fork_id: Option<&str>, +) -> String { + let genesis_hash = genesis_hash.as_ref(); + if let Some(fork_id) = fork_id { + format!( + "/{}/{}/mmr/1", + array_bytes::bytes2hex("", genesis_hash), + fork_id + ) + } else { + format!("/{}/mmr/1", array_bytes::bytes2hex("", genesis_hash)) + } +} + +/// The key of [`BlockRequestHandler::seen_requests`]. +#[derive(Eq, PartialEq, Clone)] +struct SeenRequestsKey { + peer: PeerId, + block_number: BlockNumber, +} + +#[allow(clippy::derived_hash_with_manual_eq)] +impl Hash for SeenRequestsKey { + fn hash(&self, state: &mut H) { + self.peer.hash(state); + self.block_number.hash(state); + } +} + +/// Request MMR data from a peer. +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, Encode, Decode, Debug)] +pub struct MmrRequest { + /// Starting block for MMR + pub starting_block: BlockNumber, + /// Max returned data items + pub limit: BlockNumber, +} + +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, Encode, Decode, Debug)] +pub struct MmrResponse { + /// MMR-leafs related to block number + pub mmr_data: BTreeMap>, +} + +/// The value of [`StateRequestHandler::seen_requests`]. +enum SeenRequestsValue { + /// First time we have seen the request. + First, + /// We have fulfilled the request `n` times. + Fulfilled(usize), +} + +/// Handler for incoming block requests from a remote peer. +pub struct MmrRequestHandler { + request_receiver: async_channel::Receiver, + /// Maps from request to number of times we have seen this request. + /// + /// This is used to check if a peer is spamming us with the same request. + seen_requests: LruMap, + + offchain_db: OffchainDb, + + _phantom: PhantomData, +} + +impl MmrRequestHandler +where + Block: BlockT, + + OS: OffchainStorage, +{ + /// Create a new [`MmrRequestHandler`]. + pub fn new( + protocol_id: &ProtocolId, + fork_id: Option<&str>, + client: Arc, + num_peer_hint: usize, + offchain_storage: OS, + ) -> (Self, NB::RequestResponseProtocolConfig) + where + NB: NetworkBackend::Hash>, + Client: BlockBackend + ProofProvider + Send + Sync + 'static, + { + // Reserve enough request slots for one request per peer when we are at the maximum + // number of peers. + let capacity = std::cmp::max(num_peer_hint, 1); + let (tx, request_receiver) = async_channel::bounded(capacity); + + let protocol_config = generate_protocol_config::<_, Block, NB>( + protocol_id, + client + .block_hash(0u32.into()) + .ok() + .flatten() + .expect("Genesis block exists; qed"), + fork_id, + tx, + ); + + let capacity = ByLength::new(num_peer_hint.max(1) as u32 * 2); + let seen_requests = LruMap::new(capacity); + + ( + Self { + request_receiver, + seen_requests, + offchain_db: OffchainDb::new(offchain_storage), + _phantom: PhantomData, + }, + protocol_config, + ) + } + + /// Run [`StateRequestHandler`]. + pub async fn run(mut self) { + while let Some(request) = self.request_receiver.next().await { + let IncomingRequest { + peer, + payload, + pending_response, + } = request; + + match self.handle_request(payload, pending_response, &peer) { + Ok(()) => debug!("Handled MMR request from {}.", peer), + Err(e) => error!("Failed to handle MMR request from {}: {}", peer, e,), + } + } + } + + fn handle_request( + &mut self, + payload: Vec, + pending_response: oneshot::Sender, + peer: &PeerId, + ) -> Result<(), HandleRequestError> { + let request = MmrRequest::decode(&mut payload.as_slice())?; + + let key = SeenRequestsKey { + peer: *peer, + block_number: request.starting_block, + }; + + let mut reputation_changes = Vec::new(); + + match self.seen_requests.get(&key) { + Some(SeenRequestsValue::First) => {} + Some(SeenRequestsValue::Fulfilled(ref mut requests)) => { + *requests = requests.saturating_add(1); + + if *requests > MAX_NUMBER_OF_SAME_REQUESTS_PER_PEER { + reputation_changes.push(rep::SAME_REQUEST); + } + } + None => { + self.seen_requests + .insert(key.clone(), SeenRequestsValue::First); + } + } + + trace!("Handle MMR request: {peer}, request: {request:?}",); + + let result = if request.limit > MAX_MMR_ITEMS { + error!( + "Invalid MMR request from peer={peer}: {:?}", + HandleRequestError::MaxItemsLimitExceeded + ); + + Err(()) + } else { + let mut mmr_data = BTreeMap::new(); + for block_number in request.starting_block..(request.starting_block + request.limit) { + let canon_key = get_offchain_key(block_number.into()); + let storage_value = self + .offchain_db + .local_storage_get(StorageKind::PERSISTENT, &canon_key); + + if let Some(storage_value) = storage_value { + mmr_data.insert(block_number, storage_value); + } else { + break; // No more storage values + } + } + + if let Some(value) = self.seen_requests.get(&key) { + // If this is the first time we have processed this request, we need to change + // it to `Fulfilled`. + if let SeenRequestsValue::First = value { + *value = SeenRequestsValue::Fulfilled(1); + } + } + + let response = MmrResponse { mmr_data }; + + Ok(response.encode()) + }; + + pending_response + .send(OutgoingResponse { + result, + reputation_changes, + sent_feedback: None, + }) + .map_err(|_| HandleRequestError::SendResponse) + } +} + +#[derive(Debug, thiserror::Error)] +enum HandleRequestError { + #[error("Invalid request: max MMR items limit exceeded.")] + MaxItemsLimitExceeded, + + #[error(transparent)] + Client(#[from] sp_blockchain::Error), + + #[error("Failed to send response.")] + SendResponse, + + #[error("Failed to decode request: {0}.")] + Decode(#[from] codec::Error), +} From f66f8dced38eefc83e70f1d4f57c6317fe77941c Mon Sep 17 00:00:00 2001 From: Shamil Gadelshin Date: Tue, 9 Jul 2024 18:34:16 +0400 Subject: [PATCH 03/41] Integrate mmr-handler into the network configuration --- crates/subspace-service/src/network.rs | 33 ++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/crates/subspace-service/src/network.rs b/crates/subspace-service/src/network.rs index a40b8e7e62..815b58466e 100644 --- a/crates/subspace-service/src/network.rs +++ b/crates/subspace-service/src/network.rs @@ -1,8 +1,9 @@ +use crate::mmr::request_handler::MmrRequestHandler; use futures::channel::oneshot; use futures::{pin_mut, FutureExt, StreamExt}; use sc_client_api::{BlockBackend, BlockchainEvents, ProofProvider}; use sc_consensus::ImportQueue; -use sc_network::config::{ExHashT, PeerStore}; +use sc_network::config::PeerStore; use sc_network::service::traits::RequestResponseConfig; use sc_network::{NetworkBackend, NetworkBlock, Roles}; use sc_network_light::light_client_requests::handler::LightClientRequestHandler; @@ -21,13 +22,16 @@ use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedSender}; use sp_api::ProvideRuntimeApi; use sp_blockchain::{HeaderBackend, HeaderMetadata}; use sp_consensus::block_validation::{Chain, DefaultBlockAnnounceValidator}; +use sp_core::offchain::OffchainStorage; use sp_runtime::traits::{Block as BlockT, BlockIdTo, Header, Zero}; use std::sync::Arc; use tracing::{debug, warn}; /// Build the network service, the network status sinks and an RPC sender. -pub fn build_network( +#[allow(clippy::type_complexity)] +pub fn build_network( params: BuildNetworkParams, + offchain_storage: Option, ) -> Result< ( Arc, @@ -52,6 +56,7 @@ where TExPool: TransactionPool::Hash> + 'static, TImpQu: ImportQueue + 'static, TNet: NetworkBackend::Hash>, + TOs: OffchainStorage + 'static, { let BuildNetworkParams { config, @@ -178,6 +183,27 @@ where protocol_config }; + if let Some(offchain_storage) = offchain_storage { + let num_peer_hint = net_config.network_config.default_peers_set_num_full as usize + + net_config + .network_config + .default_peers_set + .reserved_nodes + .len(); + + // Allow both outgoing and incoming requests. + let (handler, protocol_config) = MmrRequestHandler::new::( + &protocol_id, + config.chain_spec.fork_id(), + client.clone(), + num_peer_hint, + offchain_storage, + ); + spawn_handle.spawn("mmr-request-handler", Some("networking"), handler.run()); + + net_config.add_request_response_protocol(protocol_config); + } + // install request handlers to `FullNetworkConfiguration` net_config.add_request_response_protocol(block_request_protocol_config); net_config.add_request_response_protocol(state_request_protocol_config); @@ -314,7 +340,7 @@ where ), ); - let future = build_network_future::<_, _, ::Hash, _>( + let future = build_network_future( network_mut, client, sync_service.clone(), @@ -375,7 +401,6 @@ async fn build_network_future< + Send + Sync + 'static, - H: ExHashT, N: NetworkBackend::Hash>, >( network: N, From 4e1c93e652227bcc505d7b78a37ec352c6213f81 Mon Sep 17 00:00:00 2001 From: Shamil Gadelshin Date: Tue, 9 Jul 2024 18:36:10 +0400 Subject: [PATCH 04/41] Introduce mmr-sync method. --- crates/subspace-service/src/mmr.rs | 9 + crates/subspace-service/src/mmr/sync.rs | 227 ++++++++++++++++++++++++ 2 files changed, 236 insertions(+) create mode 100644 crates/subspace-service/src/mmr.rs create mode 100644 crates/subspace-service/src/mmr/sync.rs diff --git a/crates/subspace-service/src/mmr.rs b/crates/subspace-service/src/mmr.rs new file mode 100644 index 0000000000..590f5e9d1f --- /dev/null +++ b/crates/subspace-service/src/mmr.rs @@ -0,0 +1,9 @@ +use sp_mmr_primitives::utils::NodesUtils; +use sp_mmr_primitives::{NodeIndex, INDEXING_PREFIX}; + +pub(crate) mod request_handler; +pub(crate) mod sync; + +pub(crate) fn get_offchain_key(index: NodeIndex) -> Vec { + NodesUtils::node_canon_offchain_key(INDEXING_PREFIX, index) +} diff --git a/crates/subspace-service/src/mmr/sync.rs b/crates/subspace-service/src/mmr/sync.rs new file mode 100644 index 0000000000..7f15527bca --- /dev/null +++ b/crates/subspace-service/src/mmr/sync.rs @@ -0,0 +1,227 @@ +#![allow(dead_code)] // TODO: enable after the domain-sync implementation + +use crate::mmr::get_offchain_key; +use crate::mmr::request_handler::{generate_protocol_name, MmrRequest, MmrResponse, MAX_MMR_ITEMS}; +use futures::channel::oneshot; +use parity_scale_codec::{Decode, Encode}; +use sc_network::{IfDisconnected, NetworkRequest, PeerId, RequestFailure}; +use sc_network_sync::SyncingService; +use sp_blockchain::HeaderBackend; +use sp_core::offchain::storage::OffchainDb; +use sp_core::offchain::{DbExternalities, OffchainStorage, StorageKind}; +use sp_runtime::traits::Block as BlockT; +use std::sync::Arc; +use std::time::Duration; +use subspace_core_primitives::BlockNumber; +use tokio::time::sleep; +use tracing::{debug, error, trace}; + +const SYNC_PAUSE: Duration = Duration::from_secs(5); + +/// Synchronize MMR-leafs from remote offchain storage of the synced peer. +pub async fn mmr_sync( + fork_id: Option, + client: Arc, + network_service: NR, + sync_service: Arc>, + offchain_storage: OS, +) where + Block: BlockT, + NR: NetworkRequest, + Client: HeaderBackend, + OS: OffchainStorage, +{ + debug!("MMR sync started."); + let info = client.info(); + let protocol_name = generate_protocol_name(info.genesis_hash, fork_id.as_deref()); + + let mut offchain_db = OffchainDb::new(offchain_storage); + + // Look for existing local MMR-entries + let mut starting_block = { + let mut starting_block: Option = None; + for block_number in 0..=BlockNumber::MAX { + let canon_key = get_offchain_key(block_number.into()); + if offchain_db + .local_storage_get(StorageKind::PERSISTENT, &canon_key) + .is_none() + { + starting_block = Some(block_number); + break; + } + } + + match starting_block { + None => { + error!("Can't get starting MMR block - MMR storage is corrupted."); + return; + } + Some(last_processed_block) => { + debug!("MMR-sync last processed block: {last_processed_block}"); + + last_processed_block + } + } + }; + + 'outer: loop { + let peers_info = match sync_service.peers_info().await { + Ok(peers_info) => peers_info, + Err(error) => { + error!("Peers info request returned an error: {error}",); + sleep(SYNC_PAUSE).await; + + continue; + } + }; + + // Enumerate peers until we find a suitable source for MMR + 'peers: for (peer_id, peer_info) in peers_info.iter() { + trace!("MMR sync. peer = {peer_id}, info = {:?}", peer_info); + + if !peer_info.is_synced { + trace!("MMR sync skipped (not synced). peer = {peer_id}"); + + continue; + } + + // Request MMR until target block reached. + loop { + let target_block_number = { + let best_block = sync_service.best_seen_block().await; + + match best_block { + Ok(Some(block)) => { + debug!("MMR-sync. Best seen block={block}"); + + block + } + Ok(None) => { + debug!("Can't obtain best sync block for MMR-sync."); + break 'peers; + } + Err(err) => { + error!("Can't obtain best sync block for MMR-sync. Error={err}"); + break 'peers; + } + } + }; + + let request = MmrRequest { + starting_block, + limit: MAX_MMR_ITEMS, + }; + let response = + send_mmr_request(protocol_name.clone(), *peer_id, request, &network_service) + .await; + + match response { + Ok(response) => { + trace!("Response: {:?}", response.mmr_data.len()); + + if response.mmr_data.is_empty() { + debug!("Empty response from peer={}", peer_id); + break; + } + + // Save the MMR-items from response to the local storage + 'data: for (block_number, data) in response.mmr_data.iter() { + // Ensure continuous sync + if *block_number == starting_block { + let canon_key = get_offchain_key((*block_number).into()); + offchain_db.local_storage_set( + StorageKind::PERSISTENT, + &canon_key, + data, + ); + + starting_block += 1; + } else { + debug!( + "MMR-sync gap detected={peer_id}, block_number={block_number}", + ); + break 'data; // We don't support gaps in MMR data + } + } + } + Err(error) => { + debug!("MMR sync request failed. peer = {peer_id}: {error}"); + + continue 'peers; + } + } + + // Actual MMR-items may exceed this number, however, we will catch up with the rest + // when we sync the remaining data (consensus and domain chains). + if target_block_number <= starting_block.into() { + debug!("Target block number reached: {target_block_number}"); + break 'outer; + } + } + } + debug!("No synced peers to handle the MMR-sync. Pausing...",); + sleep(SYNC_PAUSE).await; + } + + debug!("MMR sync finished."); +} + +/// MMR-sync error +#[derive(Debug, thiserror::Error)] +pub enum MmrResponseError { + #[error("MMR request failed: {0}")] + RequestFailed(#[from] RequestFailure), + + #[error("MMR request canceled")] + RequestCanceled, + + #[error("MMR request failed: invalid protocol")] + InvalidProtocol, + + #[error("Failed to decode response: {0}")] + DecodeFailed(String), +} + +async fn send_mmr_request( + protocol_name: String, + peer_id: PeerId, + request: MmrRequest, + network_service: &NR, +) -> Result { + let (tx, rx) = oneshot::channel(); + + debug!("Sending request: {request:?} (peer={peer_id})"); + + let encoded_request = request.encode(); + + network_service.start_request( + peer_id, + protocol_name.clone().into(), + encoded_request, + None, + tx, + IfDisconnected::ImmediateError, + ); + + let result = rx.await.map_err(|_| MmrResponseError::RequestCanceled)?; + + match result { + Ok((data, response_protocol_name)) => { + if response_protocol_name != protocol_name.into() { + return Err(MmrResponseError::InvalidProtocol); + } + + let response = decode_mmr_response(&data).map_err(MmrResponseError::DecodeFailed)?; + + Ok(response) + } + Err(error) => Err(error.into()), + } +} + +fn decode_mmr_response(mut response: &[u8]) -> Result { + let response = MmrResponse::decode(&mut response) + .map_err(|error| format!("Failed to decode state response: {error}"))?; + + Ok(response) +} From b6fe2b63c4fd6038aaafcf4a25365a157e712f7a Mon Sep 17 00:00:00 2001 From: Shamil Gadelshin Date: Tue, 9 Jul 2024 18:36:26 +0400 Subject: [PATCH 05/41] Integrate custom Substrate network. --- crates/subspace-service/src/lib.rs | 59 ++++++++++++++++++++---------- 1 file changed, 39 insertions(+), 20 deletions(-) diff --git a/crates/subspace-service/src/lib.rs b/crates/subspace-service/src/lib.rs index 80294a9f33..00f2c07ea8 100644 --- a/crates/subspace-service/src/lib.rs +++ b/crates/subspace-service/src/lib.rs @@ -29,10 +29,11 @@ pub mod config; pub mod dsn; mod metrics; +pub(crate) mod mmr; +pub(crate) mod network; pub mod rpc; pub mod sync_from_dsn; pub mod transaction_pool; -pub mod network; use crate::config::{ChainSyncMode, SubspaceConfiguration, SubspaceNetworking}; use crate::dsn::{create_dsn_instance, DsnConfigurationError}; @@ -837,24 +838,27 @@ where net_config.add_notification_protocol(pot_gossip_notification_config); let pause_sync = Arc::clone(&net_config.network_config.pause_sync); let (network_service, system_rpc_tx, tx_handler_controller, network_starter, sync_service) = - sc_service::build_network(sc_service::BuildNetworkParams { - config: &config.base, - net_config, - client: client.clone(), - transaction_pool: transaction_pool.clone(), - spawn_handle: task_manager.spawn_handle(), - import_queue, - block_announce_validator_builder: None, - warp_sync_params: None, - block_relay, - metrics: NotificationMetrics::new( - config - .base - .prometheus_config - .as_ref() - .map(|cfg| &cfg.registry), - ), - })?; + network::build_network( + sc_service::BuildNetworkParams { + config: &config.base, + net_config, + client: client.clone(), + transaction_pool: transaction_pool.clone(), + spawn_handle: task_manager.spawn_handle(), + import_queue, + block_announce_validator_builder: None, + warp_sync_params: None, + block_relay, + metrics: NotificationMetrics::new( + config + .base + .prometheus_config + .as_ref() + .map(|cfg| &cfg.registry), + ), + }, + backend.offchain_storage(), + )?; task_manager.spawn_handle().spawn( "sync-target-follower", @@ -930,7 +934,7 @@ where let snap_sync_task = snap_sync( segment_headers_store.clone(), node.clone(), - fork_id, + fork_id.clone(), Arc::clone(&client), import_queue_service1, pause_sync.clone(), @@ -939,6 +943,21 @@ where sync_service.clone(), ); + // TODO: enable after the domain-sync implementation + // if let Some(offchain_storage) = backend.offchain_storage() { + // let mmr_sync_task = mmr_sync( + // fork_id, + // Arc::clone(&client), + // Arc::clone(&network_service), + // sync_service.clone(), + // offchain_storage, + // ); + // + // task_manager + // .spawn_handle() + // .spawn("mmr-sync", Some("mmr-sync"), mmr_sync_task); + // } + let (observer, worker) = sync_from_dsn::create_observer_and_worker( segment_headers_store.clone(), Arc::clone(&network_service), From f73e5465afc9890cdb0209fd56e05a5ac730014b Mon Sep 17 00:00:00 2001 From: Shamil Gadelshin Date: Wed, 17 Jul 2024 12:06:54 +0400 Subject: [PATCH 06/41] Fix MMR position naming. --- .../src/mmr/request_handler.rs | 25 +++---- crates/subspace-service/src/mmr/sync.rs | 65 +++++++++++-------- 2 files changed, 50 insertions(+), 40 deletions(-) diff --git a/crates/subspace-service/src/mmr/request_handler.rs b/crates/subspace-service/src/mmr/request_handler.rs index fc7ddb6a51..470382edb4 100644 --- a/crates/subspace-service/src/mmr/request_handler.rs +++ b/crates/subspace-service/src/mmr/request_handler.rs @@ -32,7 +32,6 @@ use std::hash::{Hash, Hasher}; use std::marker::PhantomData; use std::sync::Arc; use std::time::Duration; -use subspace_core_primitives::BlockNumber; use tracing::{debug, error, trace}; const MAX_NUMBER_OF_SAME_REQUESTS_PER_PEER: usize = 2; @@ -90,14 +89,14 @@ pub fn generate_protocol_name>( #[derive(Eq, PartialEq, Clone)] struct SeenRequestsKey { peer: PeerId, - block_number: BlockNumber, + starting_position: u32, } #[allow(clippy::derived_hash_with_manual_eq)] impl Hash for SeenRequestsKey { fn hash(&self, state: &mut H) { self.peer.hash(state); - self.block_number.hash(state); + self.starting_position.hash(state); } } @@ -105,17 +104,17 @@ impl Hash for SeenRequestsKey { #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, Encode, Decode, Debug)] pub struct MmrRequest { - /// Starting block for MMR - pub starting_block: BlockNumber, - /// Max returned data items - pub limit: BlockNumber, + /// Starting position for MMR node. + pub starting_position: u32, + /// Max returned nodes. + pub limit: u32, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, Encode, Decode, Debug)] pub struct MmrResponse { - /// MMR-leafs related to block number - pub mmr_data: BTreeMap>, + /// MMR-nodes related to node position + pub mmr_data: BTreeMap>, } /// The value of [`StateRequestHandler::seen_requests`]. @@ -213,7 +212,7 @@ where let key = SeenRequestsKey { peer: *peer, - block_number: request.starting_block, + starting_position: request.starting_position, }; let mut reputation_changes = Vec::new(); @@ -244,7 +243,9 @@ where Err(()) } else { let mut mmr_data = BTreeMap::new(); - for block_number in request.starting_block..(request.starting_block + request.limit) { + for block_number in + request.starting_position..(request.starting_position + request.limit) + { let canon_key = get_offchain_key(block_number.into()); let storage_value = self .offchain_db @@ -282,7 +283,7 @@ where #[derive(Debug, thiserror::Error)] enum HandleRequestError { - #[error("Invalid request: max MMR items limit exceeded.")] + #[error("Invalid request: max MMR nodes limit exceeded.")] MaxItemsLimitExceeded, #[error(transparent)] diff --git a/crates/subspace-service/src/mmr/sync.rs b/crates/subspace-service/src/mmr/sync.rs index 7f15527bca..ac40d661cd 100644 --- a/crates/subspace-service/src/mmr/sync.rs +++ b/crates/subspace-service/src/mmr/sync.rs @@ -9,10 +9,10 @@ use sc_network_sync::SyncingService; use sp_blockchain::HeaderBackend; use sp_core::offchain::storage::OffchainDb; use sp_core::offchain::{DbExternalities, OffchainStorage, StorageKind}; +use sp_mmr_primitives::utils::NodesUtils; use sp_runtime::traits::Block as BlockT; use std::sync::Arc; use std::time::Duration; -use subspace_core_primitives::BlockNumber; use tokio::time::sleep; use tracing::{debug, error, trace}; @@ -37,29 +37,29 @@ pub async fn mmr_sync( let mut offchain_db = OffchainDb::new(offchain_storage); - // Look for existing local MMR-entries - let mut starting_block = { - let mut starting_block: Option = None; - for block_number in 0..=BlockNumber::MAX { - let canon_key = get_offchain_key(block_number.into()); + // Look for existing local MMR-nodes + let mut starting_position = { + let mut starting_position: Option = None; + for position in 0..=u32::MAX { + let canon_key = get_offchain_key(position.into()); if offchain_db .local_storage_get(StorageKind::PERSISTENT, &canon_key) .is_none() { - starting_block = Some(block_number); + starting_position = Some(position); break; } } - match starting_block { + match starting_position { None => { - error!("Can't get starting MMR block - MMR storage is corrupted."); + error!("Can't get starting MMR position - MMR storage is corrupted."); return; } - Some(last_processed_block) => { - debug!("MMR-sync last processed block: {last_processed_block}"); + Some(last_processed_position) => { + debug!("MMR-sync last processed position: {last_processed_position}"); - last_processed_block + last_processed_position } } }; @@ -87,14 +87,25 @@ pub async fn mmr_sync( // Request MMR until target block reached. loop { - let target_block_number = { + let target_position = { let best_block = sync_service.best_seen_block().await; match best_block { Ok(Some(block)) => { - debug!("MMR-sync. Best seen block={block}"); + let block_number: u32 = block + .try_into() + .map_err(|_| "Can't convert block number to u32") + .expect("We convert BlockNumber which is defined as u32."); + let nodes = NodesUtils::new(block_number.into()); - block + let target_position = nodes.size().saturating_sub(1); + + debug!( + "MMR-sync. Best seen block={}, Node target position={}", + block_number, target_position + ); + + target_position } Ok(None) => { debug!("Can't obtain best sync block for MMR-sync."); @@ -108,7 +119,7 @@ pub async fn mmr_sync( }; let request = MmrRequest { - starting_block, + starting_position, limit: MAX_MMR_ITEMS, }; let response = @@ -124,22 +135,20 @@ pub async fn mmr_sync( break; } - // Save the MMR-items from response to the local storage - 'data: for (block_number, data) in response.mmr_data.iter() { + // Save the MMR-nodes from response to the local storage + 'data: for (position, data) in response.mmr_data.iter() { // Ensure continuous sync - if *block_number == starting_block { - let canon_key = get_offchain_key((*block_number).into()); + if *position == starting_position { + let canon_key = get_offchain_key((*position).into()); offchain_db.local_storage_set( StorageKind::PERSISTENT, &canon_key, data, ); - starting_block += 1; + starting_position += 1; } else { - debug!( - "MMR-sync gap detected={peer_id}, block_number={block_number}", - ); + debug!("MMR-sync gap detected={peer_id}, position={position}",); break 'data; // We don't support gaps in MMR data } } @@ -151,15 +160,15 @@ pub async fn mmr_sync( } } - // Actual MMR-items may exceed this number, however, we will catch up with the rest + // Actual MMR-nodes may exceed this number, however, we will catch up with the rest // when we sync the remaining data (consensus and domain chains). - if target_block_number <= starting_block.into() { - debug!("Target block number reached: {target_block_number}"); + if target_position <= starting_position.into() { + debug!("Target position reached: {target_position}"); break 'outer; } } } - debug!("No synced peers to handle the MMR-sync. Pausing...",); + debug!(%starting_position, "No synced peers to handle the MMR-sync. Pausing...",); sleep(SYNC_PAUSE).await; } From 175ccd9f019f4e41a956e35bad293530b4bece77 Mon Sep 17 00:00:00 2001 From: Shamil Gadelshin Date: Wed, 17 Jul 2024 14:41:05 +0400 Subject: [PATCH 07/41] Remove commented code. --- crates/subspace-service/src/lib.rs | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/crates/subspace-service/src/lib.rs b/crates/subspace-service/src/lib.rs index 00f2c07ea8..df5cc9553c 100644 --- a/crates/subspace-service/src/lib.rs +++ b/crates/subspace-service/src/lib.rs @@ -943,21 +943,6 @@ where sync_service.clone(), ); - // TODO: enable after the domain-sync implementation - // if let Some(offchain_storage) = backend.offchain_storage() { - // let mmr_sync_task = mmr_sync( - // fork_id, - // Arc::clone(&client), - // Arc::clone(&network_service), - // sync_service.clone(), - // offchain_storage, - // ); - // - // task_manager - // .spawn_handle() - // .spawn("mmr-sync", Some("mmr-sync"), mmr_sync_task); - // } - let (observer, worker) = sync_from_dsn::create_observer_and_worker( segment_headers_store.clone(), Arc::clone(&network_service), From 8c35cbe6ca884cc28b65be481908f7299857c8fd Mon Sep 17 00:00:00 2001 From: Shamil Gadelshin Date: Mon, 22 Jul 2024 17:26:41 +0400 Subject: [PATCH 08/41] Remove build_network() copy. --- crates/subspace-service/src/lib.rs | 71 ++-- crates/subspace-service/src/network.rs | 457 ------------------------- 2 files changed, 46 insertions(+), 482 deletions(-) delete mode 100644 crates/subspace-service/src/network.rs diff --git a/crates/subspace-service/src/lib.rs b/crates/subspace-service/src/lib.rs index 64c5420a11..14ec425d9d 100644 --- a/crates/subspace-service/src/lib.rs +++ b/crates/subspace-service/src/lib.rs @@ -30,7 +30,6 @@ pub mod config; pub mod dsn; mod metrics; pub(crate) mod mmr; -pub(crate) mod network; pub mod rpc; pub mod sync_from_dsn; pub mod transaction_pool; @@ -38,6 +37,7 @@ pub mod transaction_pool; use crate::config::{ChainSyncMode, SubspaceConfiguration, SubspaceNetworking}; use crate::dsn::{create_dsn_instance, DsnConfigurationError}; use crate::metrics::NodeMetrics; +use crate::mmr::request_handler::MmrRequestHandler; use crate::sync_from_dsn::piece_validator::SegmentCommitmentPieceValidator; use crate::sync_from_dsn::snap_sync::snap_sync; use crate::transaction_pool::FullPool; @@ -74,7 +74,7 @@ use sc_consensus_subspace::verifier::{SubspaceVerifier, SubspaceVerifierOptions} use sc_consensus_subspace::SubspaceLink; use sc_domains::ExtensionsFactory as DomainsExtensionFactory; use sc_network::service::traits::NetworkService; -use sc_network::{NotificationMetrics, NotificationService}; +use sc_network::{NetworkWorker, NotificationMetrics, NotificationService}; use sc_proof_of_time::source::gossip::pot_gossip_peers_set_config; use sc_proof_of_time::source::{PotSlotInfo, PotSourceWorker}; use sc_proof_of_time::verifier::PotVerifier; @@ -732,6 +732,7 @@ where } = other; let offchain_indexing_enabled = config.offchain_worker.indexing_enabled; + let fork_id = config.base.chain_spec.fork_id().map(String::from); let (node, bootstrap_nodes) = match config.subspace_networking { SubspaceNetworking::Reuse { node, @@ -846,28 +847,50 @@ where pot_gossip_peers_set_config(); net_config.add_notification_protocol(pot_gossip_notification_config); let pause_sync = Arc::clone(&net_config.network_config.pause_sync); + + if let Some(offchain_storage) = backend.offchain_storage() { + let num_peer_hint = net_config.network_config.default_peers_set_num_full as usize + + net_config + .network_config + .default_peers_set + .reserved_nodes + .len(); + + // Allow both outgoing and incoming requests. + let (handler, protocol_config) = + MmrRequestHandler::new::::Hash>, _>( + &config.base.protocol_id(), + fork_id.as_deref(), + client.clone(), + num_peer_hint, + offchain_storage, + ); + task_manager + .spawn_handle() + .spawn("mmr-request-handler", Some("networking"), handler.run()); + + net_config.add_request_response_protocol(protocol_config); + } + let (network_service, system_rpc_tx, tx_handler_controller, network_starter, sync_service) = - network::build_network( - sc_service::BuildNetworkParams { - config: &config.base, - net_config, - client: client.clone(), - transaction_pool: transaction_pool.clone(), - spawn_handle: task_manager.spawn_handle(), - import_queue, - block_announce_validator_builder: None, - warp_sync_params: None, - block_relay, - metrics: NotificationMetrics::new( - config - .base - .prometheus_config - .as_ref() - .map(|cfg| &cfg.registry), - ), - }, - backend.offchain_storage(), - )?; + sc_service::build_network(sc_service::BuildNetworkParams { + config: &config.base, + net_config, + client: client.clone(), + transaction_pool: transaction_pool.clone(), + spawn_handle: task_manager.spawn_handle(), + import_queue, + block_announce_validator_builder: None, + warp_sync_params: None, + block_relay, + metrics: NotificationMetrics::new( + config + .base + .prometheus_config + .as_ref() + .map(|cfg| &cfg.registry), + ), + })?; task_manager.spawn_handle().spawn( "sync-target-follower", @@ -938,8 +961,6 @@ where pause_sync.store(true, Ordering::Release); } - let fork_id = config.base.chain_spec.fork_id().map(String::from); - let snap_sync_task = snap_sync( segment_headers_store.clone(), node.clone(), diff --git a/crates/subspace-service/src/network.rs b/crates/subspace-service/src/network.rs deleted file mode 100644 index 815b58466e..0000000000 --- a/crates/subspace-service/src/network.rs +++ /dev/null @@ -1,457 +0,0 @@ -use crate::mmr::request_handler::MmrRequestHandler; -use futures::channel::oneshot; -use futures::{pin_mut, FutureExt, StreamExt}; -use sc_client_api::{BlockBackend, BlockchainEvents, ProofProvider}; -use sc_consensus::ImportQueue; -use sc_network::config::PeerStore; -use sc_network::service::traits::RequestResponseConfig; -use sc_network::{NetworkBackend, NetworkBlock, Roles}; -use sc_network_light::light_client_requests::handler::LightClientRequestHandler; -use sc_network_sync::block_request_handler::BlockRequestHandler; -use sc_network_sync::engine::SyncingEngine; -use sc_network_sync::service::network::NetworkServiceProvider; -use sc_network_sync::state_request_handler::StateRequestHandler; -use sc_network_sync::warp_request_handler::RequestHandler as WarpSyncRequestHandler; -use sc_network_sync::{SyncingService, WarpSyncParams}; -use sc_service::config::SyncMode; -use sc_service::{ - build_system_rpc_future, BuildNetworkParams, NetworkStarter, TransactionPoolAdapter, -}; -use sc_transaction_pool_api::TransactionPool; -use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedSender}; -use sp_api::ProvideRuntimeApi; -use sp_blockchain::{HeaderBackend, HeaderMetadata}; -use sp_consensus::block_validation::{Chain, DefaultBlockAnnounceValidator}; -use sp_core::offchain::OffchainStorage; -use sp_runtime::traits::{Block as BlockT, BlockIdTo, Header, Zero}; -use std::sync::Arc; -use tracing::{debug, warn}; - -/// Build the network service, the network status sinks and an RPC sender. -#[allow(clippy::type_complexity)] -pub fn build_network( - params: BuildNetworkParams, - offchain_storage: Option, -) -> Result< - ( - Arc, - TracingUnboundedSender>, - sc_network_transactions::TransactionsHandlerController<::Hash>, - NetworkStarter, - Arc>, - ), - sc_service::Error, -> -where - TBl: BlockT, - TCl: ProvideRuntimeApi - + HeaderMetadata - + Chain - + BlockBackend - + BlockIdTo - + ProofProvider - + HeaderBackend - + BlockchainEvents - + 'static, - TExPool: TransactionPool::Hash> + 'static, - TImpQu: ImportQueue + 'static, - TNet: NetworkBackend::Hash>, - TOs: OffchainStorage + 'static, -{ - let BuildNetworkParams { - config, - mut net_config, - client, - transaction_pool, - spawn_handle, - import_queue, - block_announce_validator_builder, - warp_sync_params, - block_relay, - metrics, - } = params; - - if warp_sync_params.is_none() && config.network.sync_mode.is_warp() { - return Err("Warp sync enabled, but no warp sync provider configured.".into()); - } - - if client.requires_full_sync() { - match config.network.sync_mode { - SyncMode::LightState { .. } => { - return Err("Fast sync doesn't work for archive nodes".into()) - } - SyncMode::Warp => return Err("Warp sync doesn't work for archive nodes".into()), - SyncMode::Full => {} - } - } - - let protocol_id = config.protocol_id(); - let genesis_hash = client - .block_hash(0u32.into()) - .ok() - .flatten() - .expect("Genesis block exists; qed"); - - let block_announce_validator = if let Some(f) = block_announce_validator_builder { - f(client.clone()) - } else { - Box::new(DefaultBlockAnnounceValidator) - }; - - let (chain_sync_network_provider, chain_sync_network_handle) = NetworkServiceProvider::new(); - let (mut block_server, block_downloader, block_request_protocol_config) = match block_relay { - Some(params) => ( - params.server, - params.downloader, - params.request_response_config, - ), - None => { - // Custom protocol was not specified, use the default block handler. - // Allow both outgoing and incoming requests. - let params = BlockRequestHandler::new::( - chain_sync_network_handle.clone(), - &protocol_id, - config.chain_spec.fork_id(), - client.clone(), - config.network.default_peers_set.in_peers as usize - + config.network.default_peers_set.out_peers as usize, - ); - ( - params.server, - params.downloader, - params.request_response_config, - ) - } - }; - spawn_handle.spawn("block-request-handler", Some("networking"), async move { - block_server.run().await; - }); - - let (state_request_protocol_config, state_request_protocol_name) = { - let num_peer_hint = net_config.network_config.default_peers_set_num_full as usize - + net_config - .network_config - .default_peers_set - .reserved_nodes - .len(); - // Allow both outgoing and incoming requests. - let (handler, protocol_config) = StateRequestHandler::new::( - &protocol_id, - config.chain_spec.fork_id(), - client.clone(), - num_peer_hint, - ); - let config_name = protocol_config.protocol_name().clone(); - - spawn_handle.spawn("state-request-handler", Some("networking"), handler.run()); - (protocol_config, config_name) - }; - - let (warp_sync_protocol_config, warp_request_protocol_name) = match warp_sync_params.as_ref() { - Some(WarpSyncParams::WithProvider(warp_with_provider)) => { - // Allow both outgoing and incoming requests. - let (handler, protocol_config) = WarpSyncRequestHandler::new::<_, TNet>( - protocol_id.clone(), - genesis_hash, - config.chain_spec.fork_id(), - warp_with_provider.clone(), - ); - let config_name = protocol_config.protocol_name().clone(); - - spawn_handle.spawn( - "warp-sync-request-handler", - Some("networking"), - handler.run(), - ); - (Some(protocol_config), Some(config_name)) - } - _ => (None, None), - }; - - let light_client_request_protocol_config = { - // Allow both outgoing and incoming requests. - let (handler, protocol_config) = LightClientRequestHandler::new::( - &protocol_id, - config.chain_spec.fork_id(), - client.clone(), - ); - spawn_handle.spawn( - "light-client-request-handler", - Some("networking"), - handler.run(), - ); - protocol_config - }; - - if let Some(offchain_storage) = offchain_storage { - let num_peer_hint = net_config.network_config.default_peers_set_num_full as usize - + net_config - .network_config - .default_peers_set - .reserved_nodes - .len(); - - // Allow both outgoing and incoming requests. - let (handler, protocol_config) = MmrRequestHandler::new::( - &protocol_id, - config.chain_spec.fork_id(), - client.clone(), - num_peer_hint, - offchain_storage, - ); - spawn_handle.spawn("mmr-request-handler", Some("networking"), handler.run()); - - net_config.add_request_response_protocol(protocol_config); - } - - // install request handlers to `FullNetworkConfiguration` - net_config.add_request_response_protocol(block_request_protocol_config); - net_config.add_request_response_protocol(state_request_protocol_config); - net_config.add_request_response_protocol(light_client_request_protocol_config); - - if let Some(config) = warp_sync_protocol_config { - net_config.add_request_response_protocol(config); - } - - let bitswap_config = config.network.ipfs_server.then(|| { - let (handler, config) = TNet::bitswap_server(client.clone()); - spawn_handle.spawn("bitswap-request-handler", Some("networking"), handler); - - config - }); - - // create transactions protocol and add it to the list of supported protocols of - let peer_store_handle = net_config.peer_store_handle(); - let (transactions_handler_proto, transactions_config) = - sc_network_transactions::TransactionsHandlerPrototype::new::<_, TBl, TNet>( - protocol_id.clone(), - genesis_hash, - config.chain_spec.fork_id(), - metrics.clone(), - Arc::clone(&peer_store_handle), - ); - net_config.add_notification_protocol(transactions_config); - - // Start task for `PeerStore` - let peer_store = net_config.take_peer_store(); - let peer_store_handle = peer_store.handle(); - spawn_handle.spawn("peer-store", Some("networking"), peer_store.run()); - - let (engine, sync_service, block_announce_config) = SyncingEngine::new( - Roles::from(&config.role), - client.clone(), - config - .prometheus_config - .as_ref() - .map(|config| config.registry.clone()) - .as_ref(), - metrics.clone(), - &net_config, - protocol_id.clone(), - &config.chain_spec.fork_id().map(ToOwned::to_owned), - block_announce_validator, - warp_sync_params, - chain_sync_network_handle, - import_queue.service(), - block_downloader, - state_request_protocol_name, - warp_request_protocol_name, - Arc::clone(&peer_store_handle), - config.network.force_synced, - )?; - let sync_service_import_queue = sync_service.clone(); - let sync_service = Arc::new(sync_service); - - let genesis_hash = client - .hash(Zero::zero()) - .ok() - .flatten() - .expect("Genesis block exists; qed"); - let network_params = sc_network::config::Params::::Hash, TNet> { - role: config.role.clone(), - executor: { - let spawn_handle = Clone::clone(&spawn_handle); - Box::new(move |fut| { - spawn_handle.spawn("libp2p-node", Some("networking"), fut); - }) - }, - network_config: net_config, - genesis_hash, - protocol_id: protocol_id.clone(), - fork_id: config.chain_spec.fork_id().map(ToOwned::to_owned), - metrics_registry: config - .prometheus_config - .as_ref() - .map(|config| config.registry.clone()), - block_announce_config, - bitswap_config, - notification_metrics: metrics, - }; - - let has_bootnodes = !network_params - .network_config - .network_config - .boot_nodes - .is_empty(); - let network_mut = TNet::new(network_params)?; - let network = network_mut.network_service().clone(); - - let (tx_handler, tx_handler_controller) = transactions_handler_proto.build( - network.clone(), - sync_service.clone(), - Arc::new(TransactionPoolAdapter::new( - transaction_pool, - client.clone(), - )), - config - .prometheus_config - .as_ref() - .map(|config| &config.registry), - )?; - spawn_handle.spawn_blocking( - "network-transactions-handler", - Some("networking"), - tx_handler.run(), - ); - - spawn_handle.spawn_blocking( - "chain-sync-network-service-provider", - Some("networking"), - chain_sync_network_provider.run(Arc::new(network.clone())), - ); - spawn_handle.spawn( - "import-queue", - None, - import_queue.run(Box::new(sync_service_import_queue)), - ); - spawn_handle.spawn_blocking("syncing", None, engine.run()); - - let (system_rpc_tx, system_rpc_rx) = tracing_unbounded("mpsc_system_rpc", 10_000); - spawn_handle.spawn( - "system-rpc-handler", - Some("networking"), - build_system_rpc_future::<_, _, ::Hash>( - config.role.clone(), - network_mut.network_service(), - sync_service.clone(), - client.clone(), - system_rpc_rx, - has_bootnodes, - ), - ); - - let future = build_network_future( - network_mut, - client, - sync_service.clone(), - config.announce_block, - ); - - // TODO: Normally, one is supposed to pass a list of notifications protocols supported by the - // node through the `NetworkConfiguration` struct. But because this function doesn't know in - // advance which components, such as GrandPa or Polkadot, will be plugged on top of the - // service, it is unfortunately not possible to do so without some deep refactoring. To - // bypass this problem, the `NetworkService` provides a `register_notifications_protocol` - // method that can be called even after the network has been initialized. However, we want to - // avoid the situation where `register_notifications_protocol` is called *after* the network - // actually connects to other peers. For this reason, we delay the process of the network - // future until the user calls `NetworkStarter::start_network`. - // - // This entire hack should eventually be removed in favour of passing the list of protocols - // through the configuration. - // - // See also https://github.com/paritytech/substrate/issues/6827 - let (network_start_tx, network_start_rx) = oneshot::channel(); - - // The network worker is responsible for gathering all network messages and processing - // them. This is quite a heavy task, and at the time of the writing of this comment it - // frequently happens that this future takes several seconds or in some situations - // even more than a minute until it has processed its entire queue. This is clearly an - // issue, and ideally we would like to fix the network future to take as little time as - // possible, but we also take the extra harm-prevention measure to execute the networking - // future using `spawn_blocking`. - spawn_handle.spawn_blocking("network-worker", Some("networking"), async move { - if network_start_rx.await.is_err() { - warn!("The NetworkStart returned as part of `build_network` has been silently dropped"); - // This `return` might seem unnecessary, but we don't want to make it look like - // everything is working as normal even though the user is clearly misusing the API. - return; - } - - future.await - }); - - Ok(( - network, - system_rpc_tx, - tx_handler_controller, - NetworkStarter::new(network_start_tx), - sync_service.clone(), - )) -} - -/// Builds a future that continuously polls the network. -async fn build_network_future< - B: BlockT, - C: BlockchainEvents - + HeaderBackend - + BlockBackend - + HeaderMetadata - + ProofProvider - + Send - + Sync - + 'static, - N: NetworkBackend::Hash>, ->( - network: N, - client: Arc, - sync_service: Arc>, - announce_imported_blocks: bool, -) { - let mut imported_blocks_stream = client.import_notification_stream().fuse(); - - // Stream of finalized blocks reported by the client. - let mut finality_notification_stream = client.finality_notification_stream().fuse(); - - let network_run = network.run().fuse(); - pin_mut!(network_run); - - loop { - futures::select! { - // List of blocks that the client has imported. - notification = imported_blocks_stream.next() => { - let notification = match notification { - Some(n) => n, - // If this stream is shut down, that means the client has shut down, and the - // most appropriate thing to do for the network future is to shut down too. - None => { - debug!("Block import stream has terminated, shutting down the network future."); - return - }, - }; - - if announce_imported_blocks { - sync_service.announce_block(notification.hash, None); - } - - if notification.is_new_best { - sync_service.new_best_block_imported( - notification.hash, - *notification.header.number(), - ); - } - } - - // List of blocks that the client has finalized. - notification = finality_notification_stream.select_next_some() => { - sync_service.on_block_finalized(notification.hash, notification.header); - } - - // Drive the network. Shut down the network future if `NetworkWorker` has terminated. - _ = network_run => { - debug!("`NetworkWorker` has terminated, shutting down the network future."); - return - } - } - } -} From c1ddc135183f59ac40eddc063fcca23b414ca0e0 Mon Sep 17 00:00:00 2001 From: Nazar Mokrynskyi Date: Mon, 22 Jul 2024 13:24:10 +0300 Subject: [PATCH 09/41] Some renaming to make future changes smaller --- crates/subspace-farmer/src/farmer_cache.rs | 76 ++++++++++++---------- 1 file changed, 43 insertions(+), 33 deletions(-) diff --git a/crates/subspace-farmer/src/farmer_cache.rs b/crates/subspace-farmer/src/farmer_cache.rs index b0f7af23df..f5a242d280 100644 --- a/crates/subspace-farmer/src/farmer_cache.rs +++ b/crates/subspace-farmer/src/farmer_cache.rs @@ -178,21 +178,21 @@ where let mut caches = self.piece_caches.write().await; for (cache_index, cache) in caches.iter_mut().enumerate() { - let Some(offset) = cache.stored_pieces.remove(&key) else { + let Some(piece_offset) = cache.stored_pieces.remove(&key) else { // Not this disk farm continue; }; // Making offset as unoccupied and remove corresponding key from heap - cache.free_offsets.push_front(offset); - match cache.backend.read_piece_index(offset).await { + cache.free_offsets.push_front(piece_offset); + match cache.backend.read_piece_index(piece_offset).await { Ok(Some(piece_index)) => { worker_state.heap.remove(KeyWrapper(piece_index)); } Ok(None) => { warn!( %cache_index, - %offset, + %piece_offset, "Piece index out of range, this is likely an implementation bug, \ not freeing heap element" ); @@ -202,7 +202,7 @@ where %error, %cache_index, ?key, - %offset, + %piece_offset, "Error while reading piece from cache, might be a disk corruption" ); } @@ -248,7 +248,7 @@ where .zip(new_piece_caches) .enumerate() .map( - |(index, ((mut stored_pieces, mut free_offsets), new_cache))| { + |(cache_index, ((mut stored_pieces, mut free_offsets), new_cache))| { if let Some(metrics) = &self.metrics { metrics .piece_cache_capacity_total @@ -262,7 +262,7 @@ where let mut maybe_contents = match new_cache.contents().await { Ok(contents) => Some(contents), Err(error) => { - warn!(%index, %error, "Failed to get cache contents"); + warn!(%cache_index, %error, "Failed to get cache contents"); None } @@ -280,11 +280,12 @@ where stored_pieces.reserve(new_cache.max_num_elements() as usize); while let Some(maybe_element_details) = contents.next().await { - let (offset, maybe_piece_index) = match maybe_element_details { + let (piece_offset, maybe_piece_index) = match maybe_element_details + { Ok(element_details) => element_details, Err(error) => { warn!( - %index, + %cache_index, %error, "Failed to get cache contents element details" ); @@ -295,11 +296,11 @@ where Some(piece_index) => { stored_pieces.insert( RecordKey::from(piece_index.to_multihash()), - offset, + piece_offset, ); } None => { - free_offsets.push_back(offset); + free_offsets.push_back(piece_offset); } } @@ -316,7 +317,7 @@ where backend: new_cache, } }, - format!("piece-cache.{index}"), + format!("piece-cache.{cache_index}"), ) }, ) @@ -419,8 +420,8 @@ where state .stored_pieces .extract_if(|key, _offset| piece_indices_to_store.remove(key).is_none()) - .for_each(|(_piece_index, offset)| { - state.free_offsets.push_front(offset); + .for_each(|(_piece_index, piece_offset)| { + state.free_offsets.push_front(piece_offset); }); }); @@ -494,17 +495,20 @@ where sorted_caches.sort_by_key(|(_, cache)| cache.stored_pieces.len()); if !stream::iter(sorted_caches) .any(|(cache_index, cache)| async move { - let Some(offset) = cache.free_offsets.pop_front() else { + let Some(piece_offset) = cache.free_offsets.pop_front() else { return false; }; - if let Err(error) = cache.backend.write_piece(offset, *piece_index, piece).await + if let Err(error) = cache + .backend + .write_piece(piece_offset, *piece_index, piece) + .await { error!( %error, %cache_index, %piece_index, - %offset, + %piece_offset, "Failed to write piece into cache" ); return false; @@ -514,7 +518,7 @@ where } cache .stored_pieces - .insert(RecordKey::from(piece_index.to_multihash()), offset); + .insert(RecordKey::from(piece_index.to_multihash()), piece_offset); true }) .await @@ -760,18 +764,21 @@ where Some(KeyWrapper(old_piece_index)) => { for (cache_index, cache) in caches.iter_mut().enumerate() { let old_record_key = RecordKey::from(old_piece_index.to_multihash()); - let Some(offset) = cache.stored_pieces.remove(&old_record_key) else { + let Some(piece_offset) = cache.stored_pieces.remove(&old_record_key) else { // Not this disk farm continue; }; - if let Err(error) = cache.backend.write_piece(offset, piece_index, &piece).await + if let Err(error) = cache + .backend + .write_piece(piece_offset, piece_index, &piece) + .await { error!( %error, %cache_index, %piece_index, - %offset, + %piece_offset, "Failed to write piece into cache" ); } else { @@ -779,10 +786,10 @@ where %cache_index, %old_piece_index, %piece_index, - %offset, + %piece_offset, "Successfully replaced old cached piece" ); - cache.stored_pieces.insert(record_key, offset); + cache.stored_pieces.insert(record_key, piece_offset); } return; } @@ -801,31 +808,34 @@ where // populated first sorted_caches.sort_by_key(|(_, cache)| cache.stored_pieces.len()); for (cache_index, cache) in sorted_caches { - let Some(offset) = cache.free_offsets.pop_front() else { + let Some(piece_offset) = cache.free_offsets.pop_front() else { // Not this disk farm continue; }; - if let Err(error) = cache.backend.write_piece(offset, piece_index, &piece).await + if let Err(error) = cache + .backend + .write_piece(piece_offset, piece_index, &piece) + .await { error!( %error, %cache_index, %piece_index, - %offset, + %piece_offset, "Failed to write piece into cache" ); } else { trace!( %cache_index, %piece_index, - %offset, + %piece_offset, "Successfully stored piece in cache" ); if let Some(metrics) = &self.metrics { metrics.piece_cache_capacity_used.inc(); } - cache.stored_pieces.insert(record_key, offset); + cache.stored_pieces.insert(record_key, piece_offset); } return; } @@ -982,10 +992,10 @@ impl FarmerCache { /// Get piece from cache pub async fn get_piece(&self, key: RecordKey) -> Option { for (cache_index, cache) in self.piece_caches.read().await.iter().enumerate() { - let Some(&offset) = cache.stored_pieces.get(&key) else { + let Some(&piece_offset) = cache.stored_pieces.get(&key) else { continue; }; - match cache.backend.read_piece(offset).await { + match cache.backend.read_piece(piece_offset).await { Ok(maybe_piece) => { return match maybe_piece { Some((_piece_index, piece)) => { @@ -1007,7 +1017,7 @@ impl FarmerCache { %error, %cache_index, ?key, - %offset, + %piece_offset, "Error while reading piece from cache, might be a disk corruption" ); @@ -1050,13 +1060,13 @@ impl FarmerCache { let key = RecordKey::from(piece_index.to_multihash()); for cache in self.piece_caches.read().await.iter() { - let Some(&offset) = cache.stored_pieces.get(&key) else { + let Some(&piece_offset) = cache.stored_pieces.get(&key) else { continue; }; if let Some(metrics) = &self.metrics { metrics.cache_find_hit.inc(); } - return Some((*cache.backend.id(), offset)); + return Some((*cache.backend.id(), piece_offset)); } if let Some(metrics) = &self.metrics { From 5a6c4a59ac63fb797faf3bef52095bd28cbe294d Mon Sep 17 00:00:00 2001 From: linning Date: Fri, 26 Jul 2024 04:01:52 +0800 Subject: [PATCH 10/41] Fix an edge case when checking receipt's domain block number in block-preprocessor Signed-off-by: linning --- domains/client/block-preprocessor/src/lib.rs | 44 +++++++++++++++----- 1 file changed, 34 insertions(+), 10 deletions(-) diff --git a/domains/client/block-preprocessor/src/lib.rs b/domains/client/block-preprocessor/src/lib.rs index 356ce3672e..70bb95f6e0 100644 --- a/domains/client/block-preprocessor/src/lib.rs +++ b/domains/client/block-preprocessor/src/lib.rs @@ -29,7 +29,8 @@ use sp_domains::{ }; use sp_messenger::MessengerApi; use sp_mmr_primitives::MmrApi; -use sp_runtime::traits::{Block as BlockT, Hash as HashT, NumberFor}; +use sp_runtime::traits::{Block as BlockT, Hash as HashT, Header, NumberFor}; +use sp_runtime::DigestItem; use sp_state_machine::LayoutV1; use sp_subspace_mmr::ConsensusChainMmrLeafProof; use sp_weights::Weight; @@ -235,15 +236,38 @@ where if consensus_spec_version >= 6 && bundle.receipt().domain_block_number != parent_domain_number { - return Err(sp_blockchain::Error::RuntimeApiError( - ApiError::Application( - format!( - "Unexpected bundle in consensus block: {:?}, something must be wrong", - at_consensus_hash - ) - .into(), - ), - )); + // If there consensus runtime just upgraded to spec version 6, which bring the receipt + // gap check, the bundle that included in the same block is doesn't perform the check + // because the new runtime take effect in the next block so skip the check here too. + let is_consensus_runtime_upgraded_to_6 = { + let consensus_header = self + .consensus_client + .header(at_consensus_hash)? + .ok_or_else(|| { + sp_blockchain::Error::Backend(format!( + "Consensus block header of {at_consensus_hash:?} unavailable" + )) + })?; + + let runtime_upgraded = consensus_header + .digest() + .logs() + .iter() + .any(|di| di == &DigestItem::RuntimeEnvironmentUpdated); + + runtime_upgraded && consensus_spec_version == 6 + }; + if !is_consensus_runtime_upgraded_to_6 { + return Err(sp_blockchain::Error::RuntimeApiError( + ApiError::Application( + format!( + "Unexpected bundle in consensus block: {:?}, something must be wrong", + at_consensus_hash + ) + .into(), + ), + )); + } } let extrinsic_root = bundle.extrinsics_root(); From 0344bf50022fb6174e0a1a69b48ff6a36eb1e7a1 Mon Sep 17 00:00:00 2001 From: linning Date: Fri, 26 Jul 2024 04:03:43 +0800 Subject: [PATCH 11/41] Downgrade ExpectingReceiptGap and UnexpectedReceiptGap error to debug level log They are also expected to happen in case of network delay Signed-off-by: linning --- crates/pallet-domains/src/lib.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/pallet-domains/src/lib.rs b/crates/pallet-domains/src/lib.rs index d219192077..38a73ca55f 100644 --- a/crates/pallet-domains/src/lib.rs +++ b/crates/pallet-domains/src/lib.rs @@ -1994,7 +1994,9 @@ impl Pallet { | BundleError::SlotInThePast | BundleError::SlotInTheFuture | BundleError::InvalidProofOfTime - | BundleError::SlotSmallerThanPreviousBlockBundle => { + | BundleError::SlotSmallerThanPreviousBlockBundle + | BundleError::ExpectingReceiptGap + | BundleError::UnexpectedReceiptGap => { log::debug!( target: "runtime::domains", "Bad bundle/receipt, domain {domain_id:?}, operator {operator_id:?}, error: {err:?}", From c90df1bdb51a649d506e8457425fb2a725e495cb Mon Sep 17 00:00:00 2001 From: Nazar Mokrynskyi Date: Fri, 26 Jul 2024 19:11:36 +0300 Subject: [PATCH 12/41] Fix PoT verification when number of iterations change --- crates/sc-consensus-subspace/src/verifier.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/sc-consensus-subspace/src/verifier.rs b/crates/sc-consensus-subspace/src/verifier.rs index 3ba04a60ec..9f0c642e55 100644 --- a/crates/sc-consensus-subspace/src/verifier.rs +++ b/crates/sc-consensus-subspace/src/verifier.rs @@ -294,7 +294,7 @@ where } let future_slot = slot + self.chain_constants.block_authoring_delay(); - let slot_to_check = Slot::from( + let first_slot_to_check = Slot::from( future_slot .checked_sub(checkpoints.len() as u64 - 1) .ok_or(VerificationError::InvalidProofOfTime)?, @@ -303,13 +303,13 @@ where .pot_parameters_change .as_ref() .and_then(|parameters_change| { - (parameters_change.slot == slot_to_check) + (parameters_change.slot <= first_slot_to_check) .then_some(parameters_change.slot_iterations) }) .unwrap_or(subspace_digest_items.pot_slot_iterations); let mut pot_input = PotNextSlotInput { - slot: slot_to_check, + slot: first_slot_to_check, slot_iterations, seed, }; From 699bce67635742d4f1c5086292c6e0b9e415620c Mon Sep 17 00:00:00 2001 From: Nazar Mokrynskyi Date: Fri, 26 Jul 2024 19:38:36 +0300 Subject: [PATCH 13/41] Correct PoT slot iterations digest to conform to documented invariant of being number of iterations for the slot that directly follows parent block's slot --- crates/pallet-subspace/src/lib.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/crates/pallet-subspace/src/lib.rs b/crates/pallet-subspace/src/lib.rs index 7608cafa53..a7be3b7c51 100644 --- a/crates/pallet-subspace/src/lib.rs +++ b/crates/pallet-subspace/src/lib.rs @@ -958,6 +958,13 @@ impl Pallet { T::EraChangeTrigger::trigger::(block_number); { + let pot_slot_iterations = + PotSlotIterations::::get().expect("Always initialized during genesis; qed"); + // This is what we had after previous block + frame_system::Pallet::::deposit_log(DigestItem::pot_slot_iterations( + pot_slot_iterations, + )); + let mut maybe_pot_slot_iterations_update = PotSlotIterationsUpdate::::get(); // Check PoT slot iterations update and apply it if it is time to do so, while also // removing corresponding storage item @@ -976,15 +983,11 @@ impl Pallet { PotSlotIterations::::put(update.slot_iterations); update.slot_iterations } else { - PotSlotIterations::::get().expect("Always initialized during genesis; qed") + pot_slot_iterations }; let pot_entropy_injection_interval = T::PotEntropyInjectionInterval::get(); let pot_entropy_injection_delay = T::PotEntropyInjectionDelay::get(); - frame_system::Pallet::::deposit_log(DigestItem::pot_slot_iterations( - pot_slot_iterations, - )); - let mut entropy = PotEntropy::::get(); let lookback_in_blocks = pot_entropy_injection_interval * BlockNumberFor::::from(T::PotEntropyInjectionLookbackDepth::get()); From 6b31926457ebfac0c4c04d707d5b211dbb0ce3f3 Mon Sep 17 00:00:00 2001 From: linning Date: Mon, 29 Jul 2024 18:05:06 +0800 Subject: [PATCH 14/41] Remove codec(skip) for the maybe_domain_sudo_call_proof field in fraud proof Signed-off-by: linning --- crates/sp-domains-fraud-proof/src/storage_proof.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/crates/sp-domains-fraud-proof/src/storage_proof.rs b/crates/sp-domains-fraud-proof/src/storage_proof.rs index 1781d478e1..2ced96ff0b 100644 --- a/crates/sp-domains-fraud-proof/src/storage_proof.rs +++ b/crates/sp-domains-fraud-proof/src/storage_proof.rs @@ -420,8 +420,6 @@ pub struct DomainInherentExtrinsicDataProof { pub dynamic_cost_of_storage_proof: DynamicCostOfStorageProof, pub consensus_chain_byte_fee_proof: ConsensusTransactionByteFeeProof, pub domain_chain_allowlist_proof: DomainChainsAllowlistUpdateStorageProof, - // TODO: remove this before next consensus runtime upgrade. Skipping to maintain compatibility with Gemini - #[codec(skip)] pub maybe_domain_sudo_call_proof: Option, } From d398c75eac7cb67af4a18d05048ecb818de605c2 Mon Sep 17 00:00:00 2001 From: linning Date: Mon, 29 Jul 2024 18:06:05 +0800 Subject: [PATCH 15/41] Bump consensus runtime spec version Signed-off-by: linning --- crates/subspace-runtime/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/subspace-runtime/src/lib.rs b/crates/subspace-runtime/src/lib.rs index ecf2be9111..6c372188e8 100644 --- a/crates/subspace-runtime/src/lib.rs +++ b/crates/subspace-runtime/src/lib.rs @@ -118,7 +118,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { spec_name: create_runtime_str!("subspace"), impl_name: create_runtime_str!("subspace"), authoring_version: 0, - spec_version: 5, + spec_version: 6, impl_version: 0, apis: RUNTIME_API_VERSIONS, transaction_version: 0, From 7d52dbb55570fa7c047af3ed1733ca1b3ca30f0e Mon Sep 17 00:00:00 2001 From: linning Date: Mon, 29 Jul 2024 21:07:03 +0800 Subject: [PATCH 16/41] Add pallet-utility to the evm and auto-id domain runtime Signed-off-by: linning --- Cargo.lock | 2 ++ domains/runtime/auto-id/Cargo.toml | 5 ++++- domains/runtime/auto-id/src/lib.rs | 8 ++++++++ domains/runtime/evm/Cargo.toml | 3 +++ domains/runtime/evm/src/lib.rs | 8 ++++++++ 5 files changed, 25 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index be42f65e86..5ef59303f4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1133,6 +1133,7 @@ dependencies = [ "pallet-transaction-payment", "pallet-transaction-payment-rpc-runtime-api", "pallet-transporter", + "pallet-utility", "parity-scale-codec", "scale-info", "sp-api", @@ -3452,6 +3453,7 @@ dependencies = [ "pallet-transaction-payment", "pallet-transaction-payment-rpc-runtime-api", "pallet-transporter", + "pallet-utility", "parity-scale-codec", "scale-info", "sp-api", diff --git a/domains/runtime/auto-id/Cargo.toml b/domains/runtime/auto-id/Cargo.toml index 8bc5bbb14c..d698ece1c5 100644 --- a/domains/runtime/auto-id/Cargo.toml +++ b/domains/runtime/auto-id/Cargo.toml @@ -35,6 +35,7 @@ pallet-timestamp = { default-features = false, git = "https://github.com/subspac pallet-transaction-payment = { default-features = false, git = "https://github.com/subspace/polkadot-sdk", rev = "0cbfcb0232bbf71ac5b14cc8c99bf043cec420ef" } pallet-transaction-payment-rpc-runtime-api = { default-features = false, git = "https://github.com/subspace/polkadot-sdk", rev = "0cbfcb0232bbf71ac5b14cc8c99bf043cec420ef" } pallet-transporter = { version = "0.1.0", path = "../../pallets/transporter", default-features = false } +pallet-utility = { default-features = false, git = "https://github.com/subspace/polkadot-sdk", rev = "0cbfcb0232bbf71ac5b14cc8c99bf043cec420ef" } scale-info = { version = "2.11.2", default-features = false, features = ["derive"] } sp-api = { default-features = false, git = "https://github.com/subspace/polkadot-sdk", rev = "0cbfcb0232bbf71ac5b14cc8c99bf043cec420ef" } sp-block-builder = { default-features = false, git = "https://github.com/subspace/polkadot-sdk", rev = "0cbfcb0232bbf71ac5b14cc8c99bf043cec420ef" } @@ -85,6 +86,7 @@ std = [ "pallet-transaction-payment/std", "pallet-transaction-payment-rpc-runtime-api/std", "pallet-transporter/std", + "pallet-utility/std", "scale-info/std", "sp-api/std", "sp-block-builder/std", @@ -120,5 +122,6 @@ runtime-benchmarks = [ "pallet-auto-id/runtime-benchmarks", "pallet-balances/runtime-benchmarks", "pallet-messenger/runtime-benchmarks", - "pallet-domain-id/runtime-benchmarks" + "pallet-domain-id/runtime-benchmarks", + "pallet-utility/runtime-benchmarks" ] diff --git a/domains/runtime/auto-id/src/lib.rs b/domains/runtime/auto-id/src/lib.rs index 2469d48f77..6814ff7155 100644 --- a/domains/runtime/auto-id/src/lib.rs +++ b/domains/runtime/auto-id/src/lib.rs @@ -469,6 +469,13 @@ impl pallet_domain_sudo::Config for Runtime { type IntoRuntimeCall = IntoRuntimeCall; } +impl pallet_utility::Config for Runtime { + type RuntimeEvent = RuntimeEvent; + type RuntimeCall = RuntimeCall; + type PalletsOrigin = OriginCaller; + type WeightInfo = pallet_utility::weights::SubstrateWeight; +} + // Create the runtime by composing the FRAME pallets that were previously configured. // // NOTE: Currently domain runtime does not naturally support the pallets with inherent extrinsics. @@ -481,6 +488,7 @@ construct_runtime!( // extrinsic. Timestamp: pallet_timestamp = 1, ExecutivePallet: domain_pallet_executive = 2, + Utility: pallet_utility = 8, // monetary stuff Balances: pallet_balances = 20, diff --git a/domains/runtime/evm/Cargo.toml b/domains/runtime/evm/Cargo.toml index e58dfdec4c..6e53479fc5 100644 --- a/domains/runtime/evm/Cargo.toml +++ b/domains/runtime/evm/Cargo.toml @@ -45,6 +45,7 @@ pallet-timestamp = { default-features = false, git = "https://github.com/subspac pallet-transaction-payment = { default-features = false, git = "https://github.com/subspace/polkadot-sdk", rev = "0cbfcb0232bbf71ac5b14cc8c99bf043cec420ef" } pallet-transaction-payment-rpc-runtime-api = { default-features = false, git = "https://github.com/subspace/polkadot-sdk", rev = "0cbfcb0232bbf71ac5b14cc8c99bf043cec420ef" } pallet-transporter = { version = "0.1.0", path = "../../pallets/transporter", default-features = false } +pallet-utility = { default-features = false, git = "https://github.com/subspace/polkadot-sdk", rev = "0cbfcb0232bbf71ac5b14cc8c99bf043cec420ef" } scale-info = { version = "2.11.2", default-features = false, features = ["derive"] } sp-api = { default-features = false, git = "https://github.com/subspace/polkadot-sdk", rev = "0cbfcb0232bbf71ac5b14cc8c99bf043cec420ef" } sp-block-builder = { default-features = false, git = "https://github.com/subspace/polkadot-sdk", rev = "0cbfcb0232bbf71ac5b14cc8c99bf043cec420ef" } @@ -105,6 +106,7 @@ std = [ "pallet-transaction-payment-rpc-runtime-api/std", "pallet-transaction-payment/std", "pallet-transporter/std", + "pallet-utility/std", "scale-info/std", "sp-api/std", "sp-block-builder/std", @@ -139,6 +141,7 @@ runtime-benchmarks = [ "pallet-ethereum/runtime-benchmarks", "pallet-evm/runtime-benchmarks", "pallet-messenger/runtime-benchmarks", + "pallet-utility/runtime-benchmarks", "sp-runtime/runtime-benchmarks", "sp-storage", ] diff --git a/domains/runtime/evm/src/lib.rs b/domains/runtime/evm/src/lib.rs index 59459bbd06..4b04aa538a 100644 --- a/domains/runtime/evm/src/lib.rs +++ b/domains/runtime/evm/src/lib.rs @@ -743,6 +743,13 @@ impl pallet_domain_sudo::Config for Runtime { type IntoRuntimeCall = IntoRuntimeCall; } +impl pallet_utility::Config for Runtime { + type RuntimeEvent = RuntimeEvent; + type RuntimeCall = RuntimeCall; + type PalletsOrigin = OriginCaller; + type WeightInfo = pallet_utility::weights::SubstrateWeight; +} + // Create the runtime by composing the FRAME pallets that were previously configured. // // NOTE: Currently domain runtime does not naturally support the pallets with inherent extrinsics. @@ -755,6 +762,7 @@ construct_runtime!( // extrinsic. Timestamp: pallet_timestamp = 1, ExecutivePallet: domain_pallet_executive = 2, + Utility: pallet_utility = 8, // monetary stuff Balances: pallet_balances = 20, From def9968c24047be449785ab0d3557eb0e93492ab Mon Sep 17 00:00:00 2001 From: linning Date: Mon, 29 Jul 2024 21:07:51 +0800 Subject: [PATCH 17/41] Bump the evm and auto-id domain runtime spec version Signed-off-by: linning --- domains/runtime/auto-id/src/lib.rs | 2 +- domains/runtime/evm/src/lib.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/domains/runtime/auto-id/src/lib.rs b/domains/runtime/auto-id/src/lib.rs index 6814ff7155..8e2d4ed0ff 100644 --- a/domains/runtime/auto-id/src/lib.rs +++ b/domains/runtime/auto-id/src/lib.rs @@ -120,7 +120,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { spec_name: create_runtime_str!("subspace-auto-id-domain"), impl_name: create_runtime_str!("subspace-auto-id-domain"), authoring_version: 0, - spec_version: 0, + spec_version: 1, impl_version: 0, apis: RUNTIME_API_VERSIONS, transaction_version: 0, diff --git a/domains/runtime/evm/src/lib.rs b/domains/runtime/evm/src/lib.rs index 4b04aa538a..35097cc9ac 100644 --- a/domains/runtime/evm/src/lib.rs +++ b/domains/runtime/evm/src/lib.rs @@ -247,7 +247,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { spec_name: create_runtime_str!("subspace-evm-domain"), impl_name: create_runtime_str!("subspace-evm-domain"), authoring_version: 0, - spec_version: 1, + spec_version: 2, impl_version: 0, apis: RUNTIME_API_VERSIONS, transaction_version: 0, From 16ee41f9903b0f42f9b806c1137f5fe539948cfa Mon Sep 17 00:00:00 2001 From: Shamil Gadelshin Date: Mon, 29 Jul 2024 14:54:04 +0400 Subject: [PATCH 18/41] Override sync_mode by dev flag. --- crates/subspace-node/src/commands/run/consensus.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/crates/subspace-node/src/commands/run/consensus.rs b/crates/subspace-node/src/commands/run/consensus.rs index b074569279..88332356e1 100644 --- a/crates/subspace-node/src/commands/run/consensus.rs +++ b/crates/subspace-node/src/commands/run/consensus.rs @@ -407,8 +407,8 @@ pub(super) struct ConsensusChainOptions { timekeeper_options: TimekeeperOptions, /// Sync mode - #[arg(long, default_value_t = ChainSyncMode::Snap)] - sync: ChainSyncMode, + #[arg(long, default_value = None)] + sync: Option, } pub(super) struct PrometheusConfiguration { @@ -451,11 +451,12 @@ pub(super) fn create_consensus_chain_configuration( dsn_options, storage_monitor, mut timekeeper_options, - sync, + mut sync, } = consensus_node_options; let transaction_pool; let rpc_cors; + // Development mode handling is limited to this section { if dev { @@ -468,6 +469,10 @@ pub(super) fn create_consensus_chain_configuration( force_authoring = true; network_options.allow_private_ips = true; timekeeper_options.timekeeper = true; + + if sync.is_none() { + sync.replace(ChainSyncMode::Full); + } } transaction_pool = pool_config.transaction_pool(dev); @@ -487,6 +492,9 @@ pub(super) fn create_consensus_chain_configuration( }); } + // Snap sync is the default mode. + let sync = sync.unwrap_or(ChainSyncMode::Snap); + let chain_spec = match chain.as_deref() { Some("gemini-3h-compiled") => chain_spec::gemini_3h_compiled()?, Some("gemini-3h") => chain_spec::gemini_3h_config()?, From ef9c3471bebd0fc9fc000b3e7f56532edadfbedf Mon Sep 17 00:00:00 2001 From: teor Date: Mon, 29 Jul 2024 13:26:39 +1000 Subject: [PATCH 19/41] Add sync mode names to help and error messages --- crates/subspace-node/src/commands/run.rs | 2 +- crates/subspace-node/src/commands/run/consensus.rs | 2 ++ crates/subspace-service/src/config.rs | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/subspace-node/src/commands/run.rs b/crates/subspace-node/src/commands/run.rs index 06702d064e..e0d84a9f3d 100644 --- a/crates/subspace-node/src/commands/run.rs +++ b/crates/subspace-node/src/commands/run.rs @@ -118,7 +118,7 @@ pub async fn run(run_options: RunOptions) -> Result<(), Error> { if maybe_domain_configuration.is_some() && subspace_configuration.sync == ChainSyncMode::Snap { return Err(Error::Other( - "Snap sync mode is not supported for domains".to_string(), + "Snap sync mode is not supported for domains, use full sync".to_string(), )); } diff --git a/crates/subspace-node/src/commands/run/consensus.rs b/crates/subspace-node/src/commands/run/consensus.rs index 88332356e1..513e239699 100644 --- a/crates/subspace-node/src/commands/run/consensus.rs +++ b/crates/subspace-node/src/commands/run/consensus.rs @@ -407,6 +407,8 @@ pub(super) struct ConsensusChainOptions { timekeeper_options: TimekeeperOptions, /// Sync mode + /// + /// Examples: `snap`, `full` #[arg(long, default_value = None)] sync: Option, } diff --git a/crates/subspace-service/src/config.rs b/crates/subspace-service/src/config.rs index eea1e60dd0..b98aaf5e25 100644 --- a/crates/subspace-service/src/config.rs +++ b/crates/subspace-service/src/config.rs @@ -306,7 +306,7 @@ impl FromStr for ChainSyncMode { match input { "full" => Ok(Self::Full), "snap" => Ok(Self::Snap), - _ => Err("Unsupported sync type".to_string()), + _ => Err("Unsupported sync type: use full or snap".to_string()), } } } From c9b5c5d10ae87d1a02887e05c4d9ccc03b9ac8af Mon Sep 17 00:00:00 2001 From: tedious Date: Wed, 17 Jul 2024 21:04:34 +0800 Subject: [PATCH 20/41] opt: farmer cache flatten piece_caches --- .../commands/cluster/controller/caches.rs | 6 +- .../src/bin/subspace-farmer/commands/farm.rs | 4 +- .../commands/shared/network.rs | 9 +- crates/subspace-farmer/src/cluster/cache.rs | 3 + .../subspace-farmer/src/cluster/controller.rs | 6 +- crates/subspace-farmer/src/farmer_cache.rs | 645 ++++++++++-------- .../subspace-farmer/src/farmer_cache/tests.rs | 6 +- .../src/farmer_piece_getter.rs | 48 +- 8 files changed, 415 insertions(+), 312 deletions(-) diff --git a/crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/controller/caches.rs b/crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/controller/caches.rs index 17a1109f2c..a57a71e6e6 100644 --- a/crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/controller/caches.rs +++ b/crates/subspace-farmer/src/bin/subspace-farmer/commands/cluster/controller/caches.rs @@ -15,7 +15,9 @@ use std::future::{ready, Future}; use std::pin::{pin, Pin}; use std::sync::Arc; use std::time::{Duration, Instant}; -use subspace_farmer::cluster::cache::{ClusterCacheIdentifyBroadcast, ClusterPieceCache}; +use subspace_farmer::cluster::cache::{ + ClusterCacheIdentifyBroadcast, ClusterCacheIndex, ClusterPieceCache, +}; use subspace_farmer::cluster::controller::ClusterControllerCacheIdentifyBroadcast; use subspace_farmer::cluster::nats_client::NatsClient; use subspace_farmer::farm::{PieceCache, PieceCacheId}; @@ -86,7 +88,7 @@ impl KnownCaches { pub(super) async fn maintain_caches( cache_group: &str, nats_client: &NatsClient, - farmer_cache: FarmerCache, + farmer_cache: FarmerCache, ) -> anyhow::Result<()> { let mut known_caches = KnownCaches::default(); diff --git a/crates/subspace-farmer/src/bin/subspace-farmer/commands/farm.rs b/crates/subspace-farmer/src/bin/subspace-farmer/commands/farm.rs index c2c70d7f55..c3d53a6687 100644 --- a/crates/subspace-farmer/src/bin/subspace-farmer/commands/farm.rs +++ b/crates/subspace-farmer/src/bin/subspace-farmer/commands/farm.rs @@ -58,6 +58,8 @@ const GET_PIECE_MAX_INTERVAL: Duration = Duration::from_secs(40); const MAX_SPACE_PLEDGED_FOR_PLOT_CACHE_ON_WINDOWS: u64 = 7 * 1024 * 1024 * 1024 * 1024; const FARM_ERROR_PRINT_INTERVAL: Duration = Duration::from_secs(30); +type CacheIndex = u8; + /// Arguments for farmer #[derive(Debug, Parser)] pub(crate) struct FarmingArgs { @@ -342,7 +344,7 @@ where let should_start_prometheus_server = !prometheus_listen_on.is_empty(); let (farmer_cache, farmer_cache_worker) = - FarmerCache::new(node_client.clone(), peer_id, Some(&mut registry)); + FarmerCache::::new(node_client.clone(), peer_id, Some(&mut registry)); let node_client = CachingProxyNodeClient::new(node_client) .await diff --git a/crates/subspace-farmer/src/bin/subspace-farmer/commands/shared/network.rs b/crates/subspace-farmer/src/bin/subspace-farmer/commands/shared/network.rs index 29b5129c4e..3328956b78 100644 --- a/crates/subspace-farmer/src/bin/subspace-farmer/commands/shared/network.rs +++ b/crates/subspace-farmer/src/bin/subspace-farmer/commands/shared/network.rs @@ -70,7 +70,7 @@ pub(in super::super) struct NetworkArgs { } #[allow(clippy::too_many_arguments)] -pub(in super::super) fn configure_network( +pub(in super::super) fn configure_network( protocol_prefix: String, base_path: &Path, keypair: Keypair, @@ -87,12 +87,15 @@ pub(in super::super) fn configure_network( }: NetworkArgs, weak_plotted_pieces: Weak>>, node_client: NC, - farmer_cache: FarmerCache, + farmer_cache: FarmerCache, prometheus_metrics_registry: Option<&mut Registry>, -) -> Result<(Node, NodeRunner), anyhow::Error> +) -> Result<(Node, NodeRunner>), anyhow::Error> where FarmIndex: Hash + Eq + Copy + fmt::Debug + Send + Sync + 'static, usize: From, + CacheIndex: Hash + Eq + Copy + fmt::Debug + fmt::Display + Send + Sync + 'static, + usize: From, + CacheIndex: TryFrom, NC: NodeClientExt + Clone, { let known_peers_registry = KnownPeersManager::new(KnownPeersManagerConfig { diff --git a/crates/subspace-farmer/src/cluster/cache.rs b/crates/subspace-farmer/src/cluster/cache.rs index 5d8e423ad7..edd624de5a 100644 --- a/crates/subspace-farmer/src/cluster/cache.rs +++ b/crates/subspace-farmer/src/cluster/cache.rs @@ -26,6 +26,9 @@ use tracing::{debug, error, info, trace, warn}; const MIN_CACHE_IDENTIFICATION_INTERVAL: Duration = Duration::from_secs(1); +/// Type alias for cache index used by cluster. +pub type ClusterCacheIndex = u16; + /// Broadcast with identification details by caches #[derive(Debug, Clone, Encode, Decode)] pub struct ClusterCacheIdentifyBroadcast { diff --git a/crates/subspace-farmer/src/cluster/controller.rs b/crates/subspace-farmer/src/cluster/controller.rs index 55d0fc51c9..6aea3a0cac 100644 --- a/crates/subspace-farmer/src/cluster/controller.rs +++ b/crates/subspace-farmer/src/cluster/controller.rs @@ -6,7 +6,7 @@ //! client implementations designed to work with cluster controller and a service function to drive //! the backend part of the controller. -use crate::cluster::cache::ClusterCacheReadPieceRequest; +use crate::cluster::cache::{ClusterCacheIndex, ClusterCacheReadPieceRequest}; use crate::cluster::nats_client::{ GenericBroadcast, GenericNotification, GenericRequest, NatsClient, }; @@ -435,7 +435,7 @@ pub async fn controller_service( nats_client: &NatsClient, node_client: &NC, piece_getter: &PG, - farmer_cache: &FarmerCache, + farmer_cache: &FarmerCache, instance: &str, primary_instance: bool, ) -> anyhow::Result<()> @@ -721,7 +721,7 @@ where async fn find_piece_responder( nats_client: &NatsClient, - farmer_cache: &FarmerCache, + farmer_cache: &FarmerCache, ) -> anyhow::Result<()> { nats_client .request_responder( diff --git a/crates/subspace-farmer/src/farmer_cache.rs b/crates/subspace-farmer/src/farmer_cache.rs index f5a242d280..fa5e15af70 100644 --- a/crates/subspace-farmer/src/farmer_cache.rs +++ b/crates/subspace-farmer/src/farmer_cache.rs @@ -14,10 +14,11 @@ use crate::utils::run_future_in_dedicated_thread; use async_lock::RwLock as AsyncRwLock; use event_listener_primitives::{Bag, HandlerId}; use futures::stream::{FuturesOrdered, FuturesUnordered}; -use futures::{select, stream, FutureExt, StreamExt}; +use futures::{select, FutureExt, StreamExt}; use prometheus_client::registry::Registry; use rayon::prelude::*; use std::collections::{HashMap, VecDeque}; +use std::hash::Hash; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use std::time::Duration; @@ -51,10 +52,46 @@ struct Handlers { progress: Handler, } +#[derive(Debug, Clone, Copy)] +struct FarmerCacheOffset { + cache_index: CacheIndex, + piece_offset: PieceCacheOffset, +} + +impl FarmerCacheOffset +where + CacheIndex: Hash + Eq + Copy + fmt::Debug + fmt::Display + Send + Sync + 'static, + CacheIndex: TryFrom, +{ + fn new(cache_index: CacheIndex, piece_offset: PieceCacheOffset) -> Self { + Self { + cache_index, + piece_offset, + } + } +} + #[derive(Debug, Clone)] -struct PieceCacheState { - stored_pieces: HashMap, - free_offsets: VecDeque, +struct PieceCachesState { + stored_pieces: HashMap>, + free_offsets: VecDeque>, + backends: Vec>, +} + +impl Default for PieceCachesState { + fn default() -> Self { + Self { + stored_pieces: HashMap::default(), + free_offsets: VecDeque::default(), + backends: Vec::default(), + } + } +} + +#[derive(Debug)] +struct CacheState { + cache_stored_pieces: HashMap>, + cache_free_offsets: VecDeque>, backend: Arc, } @@ -77,22 +114,25 @@ struct CacheWorkerState { /// Farmer cache worker used to drive the farmer cache backend #[derive(Debug)] #[must_use = "Farmer cache will not work unless its worker is running"] -pub struct FarmerCacheWorker +pub struct FarmerCacheWorker where NC: fmt::Debug, { peer_id: PeerId, node_client: NC, - piece_caches: Arc>>, + piece_caches: Arc>>, plot_caches: Arc, handlers: Arc, worker_receiver: Option>, metrics: Option>, } -impl FarmerCacheWorker +impl FarmerCacheWorker where NC: NodeClient, + CacheIndex: Hash + Eq + Copy + fmt::Debug + fmt::Display + Send + Sync + 'static, + usize: From, + CacheIndex: TryFrom, { /// Run the cache worker with provided piece getter. /// @@ -176,38 +216,39 @@ where // TODO: Consider implementing optional re-sync of the piece instead of just forgetting WorkerCommand::ForgetKey { key } => { let mut caches = self.piece_caches.write().await; + let Some(offset) = caches.stored_pieces.remove(&key) else { + // Key not exist + return; + }; + let cache_index = usize::from(offset.cache_index); + let piece_offset = offset.piece_offset; + let Some(backend) = caches.backends.get(cache_index).cloned() else { + // Cache backend not exist + return; + }; - for (cache_index, cache) in caches.iter_mut().enumerate() { - let Some(piece_offset) = cache.stored_pieces.remove(&key) else { - // Not this disk farm - continue; - }; - - // Making offset as unoccupied and remove corresponding key from heap - cache.free_offsets.push_front(piece_offset); - match cache.backend.read_piece_index(piece_offset).await { - Ok(Some(piece_index)) => { - worker_state.heap.remove(KeyWrapper(piece_index)); - } - Ok(None) => { - warn!( - %cache_index, - %piece_offset, - "Piece index out of range, this is likely an implementation bug, \ - not freeing heap element" - ); - } - Err(error) => { - error!( - %error, - %cache_index, - ?key, - %piece_offset, - "Error while reading piece from cache, might be a disk corruption" - ); - } + caches.free_offsets.push_front(offset); + match backend.read_piece_index(piece_offset).await { + Ok(Some(piece_index)) => { + worker_state.heap.remove(KeyWrapper(piece_index)); + } + Ok(None) => { + warn!( + %cache_index, + %piece_offset, + "Piece index out of range, this is likely an implementation bug, \ + not freeing heap element" + ); + } + Err(error) => { + error!( + %error, + %cache_index, + ?key, + %piece_offset, + "Error while reading piece from cache, might be a disk corruption" + ); } - return; } } } @@ -222,18 +263,15 @@ where PG: PieceGetter, { info!("Initializing piece cache"); + // Pull old cache state since it will be replaced with a new one and reuse its allocations - let cache_state = mem::take(&mut *self.piece_caches.write().await); - let mut stored_pieces = Vec::with_capacity(new_piece_caches.len()); - let mut free_offsets = Vec::with_capacity(new_piece_caches.len()); - for mut state in cache_state { - state.stored_pieces.clear(); - stored_pieces.push(state.stored_pieces); - state.free_offsets.clear(); - free_offsets.push(state.free_offsets); - } - stored_pieces.resize(new_piece_caches.len(), HashMap::default()); - free_offsets.resize(new_piece_caches.len(), VecDeque::default()); + let PieceCachesState { + mut stored_pieces, + mut free_offsets, + backends: _, + } = mem::take(&mut *self.piece_caches.write().await); + stored_pieces.clear(); + free_offsets.clear(); debug!("Collecting pieces that were in the cache before"); @@ -241,107 +279,116 @@ where metrics.piece_cache_capacity_total.set(0); metrics.piece_cache_capacity_used.set(0); } + // Build cache state of all backends - let maybe_caches_futures = stored_pieces + let piece_caches_number = new_piece_caches.len(); + let maybe_caches_futures = new_piece_caches .into_iter() - .zip(free_offsets) - .zip(new_piece_caches) .enumerate() - .map( - |(cache_index, ((mut stored_pieces, mut free_offsets), new_cache))| { - if let Some(metrics) = &self.metrics { - metrics - .piece_cache_capacity_total - .inc_by(new_cache.max_num_elements() as i64); - } - run_future_in_dedicated_thread( - move || async move { - // Hack with first collecting into `Option` with `Option::take()` call - // later is to satisfy compiler that gets confused about ownership - // otherwise - let mut maybe_contents = match new_cache.contents().await { - Ok(contents) => Some(contents), - Err(error) => { - warn!(%cache_index, %error, "Failed to get cache contents"); - - None - } - }; - let Some(mut contents) = maybe_contents.take() else { - drop(maybe_contents); - - return PieceCacheState { - stored_pieces, - free_offsets, - backend: new_cache, - }; - }; - - stored_pieces.reserve(new_cache.max_num_elements() as usize); - - while let Some(maybe_element_details) = contents.next().await { - let (piece_offset, maybe_piece_index) = match maybe_element_details - { - Ok(element_details) => element_details, - Err(error) => { - warn!( - %cache_index, - %error, - "Failed to get cache contents element details" - ); - break; - } - }; - match maybe_piece_index { - Some(piece_index) => { - stored_pieces.insert( - RecordKey::from(piece_index.to_multihash()), - piece_offset, - ); - } - None => { - free_offsets.push_back(piece_offset); - } - } - - // Allow for task to be aborted - yield_now().await; - } + .filter_map(|(cache_index, new_cache)| { + let Ok(cache_index) = CacheIndex::try_from(cache_index) else { + warn!( + ?piece_caches_number, + "Too many piece caches provided, {cache_index} cache will be ignored", + ); + return None; + }; + + if let Some(metrics) = &self.metrics { + metrics + .piece_cache_capacity_total + .inc_by(new_cache.max_num_elements() as i64); + } + + let init_fut = async move { + // Hack with first collecting into `Option` with `Option::take()` call + // later is to satisfy compiler that gets confused about ownership + // otherwise + let mut maybe_contents = match new_cache.contents().await { + Ok(contents) => Some(contents), + Err(error) => { + warn!(%cache_index, %error, "Failed to get cache contents"); + + None + } + }; + + #[allow(clippy::mutable_key_type)] + let mut cache_stored_pieces = HashMap::new(); + let mut cache_free_offsets = VecDeque::new(); - drop(maybe_contents); - drop(contents); + let Some(mut contents) = maybe_contents.take() else { + drop(maybe_contents); + + return CacheState { + cache_stored_pieces, + cache_free_offsets, + backend: new_cache, + }; + }; - PieceCacheState { - stored_pieces, - free_offsets, - backend: new_cache, + while let Some(maybe_element_details) = contents.next().await { + let (piece_offset, maybe_piece_index) = match maybe_element_details { + Ok(element_details) => element_details, + Err(error) => { + warn!( + %cache_index, + %error, + "Failed to get cache contents element details" + ); + break; } - }, - format!("piece-cache.{cache_index}"), - ) - }, - ) + }; + let offset = FarmerCacheOffset::new(cache_index, piece_offset); + match maybe_piece_index { + Some(piece_index) => { + cache_stored_pieces + .insert(RecordKey::from(piece_index.to_multihash()), offset); + } + None => { + cache_free_offsets.push_back(offset); + } + } + + // Allow for task to be aborted + yield_now().await; + } + + drop(maybe_contents); + drop(contents); + + CacheState { + cache_stored_pieces, + cache_free_offsets, + backend: new_cache, + } + }; + + Some(run_future_in_dedicated_thread( + move || init_fut, + format!("piece-cache.{cache_index}"), + )) + }) .collect::, _>>(); let caches_futures = match maybe_caches_futures { Ok(caches_futures) => caches_futures, Err(error) => { - error!( - %error, - "Failed to spawn piece cache reading thread" - ); + error!(%error, "Failed to spawn piece cache reading thread"); return; } }; - let mut caches = Vec::with_capacity(caches_futures.len()); + let mut backends = Vec::with_capacity(caches_futures.len()); let mut caches_futures = caches_futures.into_iter().collect::>(); while let Some(maybe_cache) = caches_futures.next().await { match maybe_cache { - Ok(cache) => { - caches.push(cache); + Ok(mut cache) => { + stored_pieces.extend(cache.cache_stored_pieces.into_iter()); + free_offsets.append(&mut cache.cache_free_offsets); + backends.push(cache.backend); } Err(_cancelled) => { error!("Piece cache reading thread panicked"); @@ -351,6 +398,12 @@ where }; } + let mut caches = PieceCachesState { + stored_pieces, + free_offsets, + backends, + }; + info!("Synchronizing piece cache"); let last_segment_index = loop { @@ -387,14 +440,10 @@ where debug!(%last_segment_index, "Identified last segment index"); + let limit = caches.stored_pieces.len() + caches.free_offsets.len(); worker_state.heap.clear(); // Change limit to number of pieces - worker_state.heap.set_limit( - caches - .iter() - .map(|state| state.stored_pieces.len() + state.free_offsets.len()) - .sum::(), - ); + worker_state.heap.set_limit(limit); for segment_index in SegmentIndex::ZERO..=last_segment_index { for piece_index in segment_index.segment_piece_indexes() { @@ -413,23 +462,26 @@ where }) .collect::>(); - caches.iter_mut().for_each(|state| { - // Filter-out piece indices that are stored, but should not be as well as clean - // `inserted_piece_indices` from already stored piece indices, leaving just those that are - // still missing in cache - state - .stored_pieces - .extract_if(|key, _offset| piece_indices_to_store.remove(key).is_none()) - .for_each(|(_piece_index, piece_offset)| { - state.free_offsets.push_front(piece_offset); - }); - }); + let mut piece_caches_capacity_used = vec![0u32; caches.backends.len()]; + // Filter-out piece indices that are stored, but should not be as well as clean + // `inserted_piece_indices` from already stored piece indices, leaving just those that are + // still missing in cache + caches + .stored_pieces + .extract_if(|key, _offset| piece_indices_to_store.remove(key).is_none()) + .for_each(|(_piece_index, offset)| { + caches.free_offsets.push_front(offset); + }); if let Some(metrics) = &self.metrics { - for cache in &mut caches { + for offset in caches.stored_pieces.values() { + piece_caches_capacity_used[usize::from(offset.cache_index)] += 1; + } + + for cache_used in piece_caches_capacity_used { metrics .piece_cache_capacity_used - .inc_by(cache.stored_pieces.len() as i64); + .inc_by(i64::from(cache_used)); } } @@ -488,46 +540,36 @@ where continue; }; + // TODO: Make the cache load as balanced as possible + // It will likely result in higher read load on one disk and lower on another now + // Find plot in which there is a place for new piece to be stored - let mut sorted_caches = caches.iter_mut().enumerate().collect::>(); - // Sort piece caches by number of stored pieces to fill those that are less - // populated first - sorted_caches.sort_by_key(|(_, cache)| cache.stored_pieces.len()); - if !stream::iter(sorted_caches) - .any(|(cache_index, cache)| async move { - let Some(piece_offset) = cache.free_offsets.pop_front() else { - return false; - }; + let Some(offset) = caches.free_offsets.pop_front() else { + error!( + %piece_index, + "Failed to store piece in cache, there was no space" + ); + break; + }; - if let Err(error) = cache - .backend - .write_piece(piece_offset, *piece_index, piece) - .await - { - error!( - %error, - %cache_index, - %piece_index, - %piece_offset, - "Failed to write piece into cache" - ); - return false; - } - if let Some(metrics) = &self.metrics { - metrics.piece_cache_capacity_used.inc(); - } - cache - .stored_pieces - .insert(RecordKey::from(piece_index.to_multihash()), piece_offset); - true - }) - .await + let cache_index = usize::from(offset.cache_index); + let piece_offset = offset.piece_offset; + if let Some(backend) = caches.backends.get(cache_index) + && let Err(error) = backend.write_piece(piece_offset, *piece_index, piece).await { + // TODO: Will likely need to cache problematic backend indices to avoid hitting it over and over again repeatedly error!( + %error, + %cache_index, %piece_index, - "Failed to store piece in cache, there was no space" + %piece_offset, + "Failed to write piece into cache" ); + continue; } + caches + .stored_pieces + .insert(RecordKey::from(piece_index.to_multihash()), offset); downloaded_pieces_count += 1; // Do not print anything or send progress notification after last piece until piece @@ -762,89 +804,91 @@ where match worker_state.heap.insert(heap_key) { // Entry is already occupied, we need to find and replace old piece with new one Some(KeyWrapper(old_piece_index)) => { - for (cache_index, cache) in caches.iter_mut().enumerate() { - let old_record_key = RecordKey::from(old_piece_index.to_multihash()); - let Some(piece_offset) = cache.stored_pieces.remove(&old_record_key) else { - // Not this disk farm - continue; - }; - - if let Err(error) = cache - .backend - .write_piece(piece_offset, piece_index, &piece) - .await - { - error!( - %error, - %cache_index, - %piece_index, - %piece_offset, - "Failed to write piece into cache" - ); - } else { - trace!( - %cache_index, - %old_piece_index, - %piece_index, - %piece_offset, - "Successfully replaced old cached piece" - ); - cache.stored_pieces.insert(record_key, piece_offset); - } + let old_record_key = RecordKey::from(old_piece_index.to_multihash()); + let Some(offset) = caches.stored_pieces.remove(&old_record_key) else { + // Not this disk farm + warn!( + %old_piece_index, + %piece_index, + "Should have replaced cached piece, but it didn't happen, this is an \ + implementation bug" + ); + return; + }; + let cache_index = usize::from(offset.cache_index); + let piece_offset = offset.piece_offset; + let Some(backend) = caches.backends.get(cache_index) else { + // Cache backend not exist + warn!( + %cache_index, + %piece_index, + "Should have a cached backend, but it didn't exist, this is an \ + implementation bug" + ); return; + }; + if let Err(error) = backend.write_piece(piece_offset, piece_index, &piece).await { + error!( + %error, + %cache_index, + %piece_index, + %piece_offset, + "Failed to write piece into cache" + ); + } else { + trace!( + %cache_index, + %old_piece_index, + %piece_index, + %piece_offset, + "Successfully replaced old cached piece" + ); + caches.stored_pieces.insert(record_key, offset); } - - warn!( - %old_piece_index, - %piece_index, - "Should have replaced cached piece, but it didn't happen, this is an \ - implementation bug" - ); } // There is free space in cache, need to find a free spot and place piece there None => { - let mut sorted_caches = caches.iter_mut().enumerate().collect::>(); - // Sort piece caches by number of stored pieces to fill those that are less - // populated first - sorted_caches.sort_by_key(|(_, cache)| cache.stored_pieces.len()); - for (cache_index, cache) in sorted_caches { - let Some(piece_offset) = cache.free_offsets.pop_front() else { - // Not this disk farm - continue; - }; + let Some(offset) = caches.free_offsets.pop_front() else { + warn!( + %piece_index, + "Should have inserted piece into cache, but it didn't happen, this is an \ + implementation bug" + ); + return; + }; + let cache_index = usize::from(offset.cache_index); + let piece_offset = offset.piece_offset; + let Some(backend) = caches.backends.get(cache_index) else { + // Cache backend not exist + warn!( + %cache_index, + %piece_index, + "Should have a cached backend, but it didn't exist, this is an \ + implementation bug" + ); + return; + }; - if let Err(error) = cache - .backend - .write_piece(piece_offset, piece_index, &piece) - .await - { - error!( - %error, - %cache_index, - %piece_index, - %piece_offset, - "Failed to write piece into cache" - ); - } else { - trace!( - %cache_index, - %piece_index, - %piece_offset, - "Successfully stored piece in cache" - ); - if let Some(metrics) = &self.metrics { - metrics.piece_cache_capacity_used.inc(); - } - cache.stored_pieces.insert(record_key, piece_offset); + if let Err(error) = backend.write_piece(piece_offset, piece_index, &piece).await { + error!( + %error, + %cache_index, + %piece_index, + %piece_offset, + "Failed to write piece into cache" + ); + } else { + trace!( + %cache_index, + %piece_index, + %piece_offset, + "Successfully stored piece in cache" + ); + if let Some(metrics) = &self.metrics { + metrics.piece_cache_capacity_used.inc(); } - return; + caches.stored_pieces.insert(record_key, offset); } - - warn!( - %piece_index, - "Should have inserted piece into cache, but it didn't happen, this is an \ - implementation bug" - ); } }; } @@ -933,10 +977,10 @@ impl PlotCaches { /// where piece cache is not enough to store all the pieces on the network, while there is a lot of /// space in the plot that is not used by sectors yet and can be leverage as extra caching space. #[derive(Debug, Clone)] -pub struct FarmerCache { +pub struct FarmerCache { peer_id: PeerId, /// Individual dedicated piece caches - piece_caches: Arc>>, + piece_caches: Arc>>, /// Additional piece caches plot_caches: Arc, handlers: Arc, @@ -945,7 +989,12 @@ pub struct FarmerCache { metrics: Option>, } -impl FarmerCache { +impl FarmerCache +where + CacheIndex: Hash + Eq + Copy + fmt::Debug + fmt::Display + Send + Sync + 'static, + usize: From, + CacheIndex: TryFrom, +{ /// Create new piece cache instance and corresponding worker. /// /// NOTE: Returned future is async, but does blocking operations and should be running in @@ -954,7 +1003,7 @@ impl FarmerCache { node_client: NC, peer_id: PeerId, registry: Option<&mut Registry>, - ) -> (Self, FarmerCacheWorker) + ) -> (Self, FarmerCacheWorker) where NC: NodeClient, { @@ -991,11 +1040,23 @@ impl FarmerCache { /// Get piece from cache pub async fn get_piece(&self, key: RecordKey) -> Option { - for (cache_index, cache) in self.piece_caches.read().await.iter().enumerate() { - let Some(&piece_offset) = cache.stored_pieces.get(&key) else { - continue; - }; - match cache.backend.read_piece(piece_offset).await { + let maybe_piece_found = { + let caches = self.piece_caches.read().await; + + caches.stored_pieces.get(&key).and_then(|offset| { + let cache_index = usize::from(offset.cache_index); + let piece_offset = offset.piece_offset; + + Some(( + piece_offset, + cache_index, + caches.backends.get(cache_index)?.clone(), + )) + }) + }; + + if let Some((piece_offset, cache_index, backend)) = maybe_piece_found { + match backend.read_piece(piece_offset).await { Ok(maybe_piece) => { return match maybe_piece { Some((_piece_index, piece)) => { @@ -1059,14 +1120,22 @@ impl FarmerCache { ) -> Option<(PieceCacheId, PieceCacheOffset)> { let key = RecordKey::from(piece_index.to_multihash()); - for cache in self.piece_caches.read().await.iter() { - let Some(&piece_offset) = cache.stored_pieces.get(&key) else { - continue; - }; + let caches = self.piece_caches.read().await; + let Some(offset) = caches.stored_pieces.get(&key) else { + if let Some(metrics) = &self.metrics { + metrics.cache_find_miss.inc(); + } + + return None; + }; + let cache_index = usize::from(offset.cache_index); + let piece_offset = offset.piece_offset; + + if let Some(backend) = caches.backends.get(cache_index) { if let Some(metrics) = &self.metrics { metrics.cache_find_hit.inc(); } - return Some((*cache.backend.id(), piece_offset)); + return Some((*backend.id(), piece_offset)); } if let Some(metrics) = &self.metrics { @@ -1113,21 +1182,29 @@ impl FarmerCache { } } -impl LocalRecordProvider for FarmerCache { +impl LocalRecordProvider for FarmerCache +where + CacheIndex: Hash + Eq + Copy + fmt::Debug + fmt::Display + Send + Sync + 'static, + usize: From, + CacheIndex: TryFrom, +{ fn record(&self, key: &RecordKey) -> Option { - for piece_cache in self.piece_caches.try_read()?.iter() { - if piece_cache.stored_pieces.contains_key(key) { - // Note: We store our own provider records locally without local addresses - // to avoid redundant storage and outdated addresses. Instead, these are - // acquired on demand when returning a `ProviderRecord` for the local node. - return Some(ProviderRecord { - key: key.clone(), - provider: self.peer_id, - expires: None, - addresses: Vec::new(), - }); - }; - } + if self + .piece_caches + .try_read()? + .stored_pieces + .contains_key(key) + { + // Note: We store our own provider records locally without local addresses + // to avoid redundant storage and outdated addresses. Instead, these are + // acquired on demand when returning a `ProviderRecord` for the local node. + return Some(ProviderRecord { + key: key.clone(), + provider: self.peer_id, + expires: None, + addresses: Vec::new(), + }); + }; let found_fut = self .plot_caches diff --git a/crates/subspace-farmer/src/farmer_cache/tests.rs b/crates/subspace-farmer/src/farmer_cache/tests.rs index 2ffd343d27..289191d960 100644 --- a/crates/subspace-farmer/src/farmer_cache/tests.rs +++ b/crates/subspace-farmer/src/farmer_cache/tests.rs @@ -24,6 +24,8 @@ use subspace_rpc_primitives::{ }; use tempfile::tempdir; +type TestCacheIndex = u8; + #[derive(Debug, Clone)] struct MockNodeClient { current_segment_index: Arc, @@ -185,7 +187,7 @@ async fn basic() { { let (farmer_cache, farmer_cache_worker) = - FarmerCache::new(node_client.clone(), public_key.to_peer_id(), None); + FarmerCache::::new(node_client.clone(), public_key.to_peer_id(), None); let farmer_cache_worker_exited = tokio::spawn(farmer_cache_worker.run(piece_getter.clone())); @@ -385,7 +387,7 @@ async fn basic() { pieces.lock().clear(); let (farmer_cache, farmer_cache_worker) = - FarmerCache::new(node_client.clone(), public_key.to_peer_id(), None); + FarmerCache::::new(node_client.clone(), public_key.to_peer_id(), None); let farmer_cache_worker_exited = tokio::spawn(farmer_cache_worker.run(piece_getter)); diff --git a/crates/subspace-farmer/src/farmer_piece_getter.rs b/crates/subspace-farmer/src/farmer_piece_getter.rs index 8df41d865a..67d4878389 100644 --- a/crates/subspace-farmer/src/farmer_piece_getter.rs +++ b/crates/subspace-farmer/src/farmer_piece_getter.rs @@ -103,9 +103,9 @@ pub struct DsnCacheRetryPolicy { pub backoff: ExponentialBackoff, } -struct Inner { +struct Inner { piece_provider: PieceProvider, - farmer_cache: FarmerCache, + farmer_cache: FarmerCache, node_client: NC, plotted_pieces: Arc>>, dsn_cache_retry_policy: DsnCacheRetryPolicy, @@ -116,18 +116,20 @@ struct Inner { /// Farmer-specific piece getter. /// /// Implements [`PieceGetter`] for plotting purposes, but useful outside of that as well. -pub struct FarmerPieceGetter { - inner: Arc>, +pub struct FarmerPieceGetter { + inner: Arc>, } -impl fmt::Debug for FarmerPieceGetter { +impl fmt::Debug + for FarmerPieceGetter +{ #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("FarmerPieceGetter").finish_non_exhaustive() } } -impl Clone for FarmerPieceGetter { +impl Clone for FarmerPieceGetter { #[inline] fn clone(&self) -> Self { Self { @@ -136,17 +138,20 @@ impl Clone for FarmerPieceGetter { } } -impl FarmerPieceGetter +impl FarmerPieceGetter where FarmIndex: Hash + Eq + Copy + fmt::Debug + Send + Sync + 'static, usize: From, + CacheIndex: Hash + Eq + Copy + fmt::Debug + fmt::Display + Send + Sync + 'static, + usize: From, + CacheIndex: TryFrom, PV: PieceValidator + Send + 'static, NC: NodeClient, { /// Create new instance pub fn new( piece_provider: PieceProvider, - farmer_cache: FarmerCache, + farmer_cache: FarmerCache, node_client: NC, plotted_pieces: Arc>>, dsn_cache_retry_policy: DsnCacheRetryPolicy, @@ -359,7 +364,7 @@ where /// Downgrade to [`WeakFarmerPieceGetter`] in order to break reference cycles with internally /// used [`Arc`] - pub fn downgrade(&self) -> WeakFarmerPieceGetter { + pub fn downgrade(&self) -> WeakFarmerPieceGetter { WeakFarmerPieceGetter { inner: Arc::downgrade(&self.inner), } @@ -367,10 +372,13 @@ where } #[async_trait] -impl PieceGetter for FarmerPieceGetter +impl PieceGetter for FarmerPieceGetter where FarmIndex: Hash + Eq + Copy + fmt::Debug + Send + Sync + 'static, usize: From, + CacheIndex: Hash + Eq + Copy + fmt::Debug + fmt::Display + Send + Sync + 'static, + usize: From, + CacheIndex: TryFrom, PV: PieceValidator + Send + 'static, NC: NodeClient, { @@ -409,11 +417,13 @@ where } /// Weak farmer piece getter, can be upgraded to [`FarmerPieceGetter`] -pub struct WeakFarmerPieceGetter { - inner: Weak>, +pub struct WeakFarmerPieceGetter { + inner: Weak>, } -impl fmt::Debug for WeakFarmerPieceGetter { +impl fmt::Debug + for WeakFarmerPieceGetter +{ #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("WeakFarmerPieceGetter") @@ -421,7 +431,7 @@ impl fmt::Debug for WeakFarmerPieceGetter } } -impl Clone for WeakFarmerPieceGetter { +impl Clone for WeakFarmerPieceGetter { #[inline] fn clone(&self) -> Self { Self { @@ -431,10 +441,14 @@ impl Clone for WeakFarmerPieceGetter { } #[async_trait] -impl PieceGetter for WeakFarmerPieceGetter +impl PieceGetter + for WeakFarmerPieceGetter where FarmIndex: Hash + Eq + Copy + fmt::Debug + Send + Sync + 'static, usize: From, + CacheIndex: Hash + Eq + Copy + fmt::Debug + fmt::Display + Send + Sync + 'static, + usize: From, + CacheIndex: TryFrom, PV: PieceValidator + Send + 'static, NC: NodeClient, { @@ -451,9 +465,9 @@ where } } -impl WeakFarmerPieceGetter { +impl WeakFarmerPieceGetter { /// Try to upgrade to [`FarmerPieceGetter`] if there is at least one other instance of it alive - pub fn upgrade(&self) -> Option> { + pub fn upgrade(&self) -> Option> { Some(FarmerPieceGetter { inner: self.inner.upgrade()?, }) From 040bbfadbaa1ce568c7ae7667a44cb45282f4d0f Mon Sep 17 00:00:00 2001 From: tedious Date: Fri, 26 Jul 2024 21:49:04 +0800 Subject: [PATCH 21/41] chore: tiny refactor and rename --- crates/subspace-farmer/src/farmer_cache.rs | 74 +++++++++++++++------- 1 file changed, 52 insertions(+), 22 deletions(-) diff --git a/crates/subspace-farmer/src/farmer_cache.rs b/crates/subspace-farmer/src/farmer_cache.rs index fa5e15af70..a9afbbb3a6 100644 --- a/crates/subspace-farmer/src/farmer_cache.rs +++ b/crates/subspace-farmer/src/farmer_cache.rs @@ -71,18 +71,41 @@ where } } +#[derive(Debug, Clone)] +struct CacheBackend { + backend: Arc, + total_capacity: u32, +} + +impl std::ops::Deref for CacheBackend { + type Target = Arc; + + fn deref(&self) -> &Self::Target { + &self.backend + } +} + +impl CacheBackend { + fn new(backend: Arc, total_capacity: u32) -> Self { + Self { + backend, + total_capacity, + } + } +} + #[derive(Debug, Clone)] struct PieceCachesState { stored_pieces: HashMap>, - free_offsets: VecDeque>, - backends: Vec>, + dangling_free_offsets: VecDeque>, + backends: Vec, } impl Default for PieceCachesState { fn default() -> Self { Self { stored_pieces: HashMap::default(), - free_offsets: VecDeque::default(), + dangling_free_offsets: VecDeque::default(), backends: Vec::default(), } } @@ -91,8 +114,8 @@ impl Default for PieceCachesState { #[derive(Debug)] struct CacheState { cache_stored_pieces: HashMap>, - cache_free_offsets: VecDeque>, - backend: Arc, + cache_free_offsets: Vec>, + backend: CacheBackend, } #[derive(Debug)] @@ -227,7 +250,7 @@ where return; }; - caches.free_offsets.push_front(offset); + caches.dangling_free_offsets.push_front(offset); match backend.read_piece_index(piece_offset).await { Ok(Some(piece_index)) => { worker_state.heap.remove(KeyWrapper(piece_index)); @@ -267,11 +290,11 @@ where // Pull old cache state since it will be replaced with a new one and reuse its allocations let PieceCachesState { mut stored_pieces, - mut free_offsets, + mut dangling_free_offsets, backends: _, } = mem::take(&mut *self.piece_caches.write().await); stored_pieces.clear(); - free_offsets.clear(); + dangling_free_offsets.clear(); debug!("Collecting pieces that were in the cache before"); @@ -286,6 +309,8 @@ where .into_iter() .enumerate() .filter_map(|(cache_index, new_cache)| { + let total_capacity = new_cache.max_num_elements(); + let backend = CacheBackend::new(new_cache, total_capacity); let Ok(cache_index) = CacheIndex::try_from(cache_index) else { warn!( ?piece_caches_number, @@ -297,14 +322,14 @@ where if let Some(metrics) = &self.metrics { metrics .piece_cache_capacity_total - .inc_by(new_cache.max_num_elements() as i64); + .inc_by(total_capacity as i64); } let init_fut = async move { // Hack with first collecting into `Option` with `Option::take()` call // later is to satisfy compiler that gets confused about ownership // otherwise - let mut maybe_contents = match new_cache.contents().await { + let mut maybe_contents = match backend.backend.contents().await { Ok(contents) => Some(contents), Err(error) => { warn!(%cache_index, %error, "Failed to get cache contents"); @@ -315,7 +340,7 @@ where #[allow(clippy::mutable_key_type)] let mut cache_stored_pieces = HashMap::new(); - let mut cache_free_offsets = VecDeque::new(); + let mut cache_free_offsets = Vec::new(); let Some(mut contents) = maybe_contents.take() else { drop(maybe_contents); @@ -323,7 +348,7 @@ where return CacheState { cache_stored_pieces, cache_free_offsets, - backend: new_cache, + backend, }; }; @@ -346,7 +371,7 @@ where .insert(RecordKey::from(piece_index.to_multihash()), offset); } None => { - cache_free_offsets.push_back(offset); + cache_free_offsets.push(offset); } } @@ -360,7 +385,7 @@ where CacheState { cache_stored_pieces, cache_free_offsets, - backend: new_cache, + backend, } }; @@ -385,10 +410,12 @@ where while let Some(maybe_cache) = caches_futures.next().await { match maybe_cache { - Ok(mut cache) => { + Ok(cache) => { + let backend = cache.backend; + let free_offsets = cache.cache_free_offsets; stored_pieces.extend(cache.cache_stored_pieces.into_iter()); - free_offsets.append(&mut cache.cache_free_offsets); - backends.push(cache.backend); + dangling_free_offsets.extend(free_offsets.into_iter()); + backends.push(backend); } Err(_cancelled) => { error!("Piece cache reading thread panicked"); @@ -400,7 +427,7 @@ where let mut caches = PieceCachesState { stored_pieces, - free_offsets, + dangling_free_offsets, backends, }; @@ -440,7 +467,10 @@ where debug!(%last_segment_index, "Identified last segment index"); - let limit = caches.stored_pieces.len() + caches.free_offsets.len(); + let limit = caches + .backends + .iter() + .fold(0usize, |acc, backend| acc + backend.total_capacity as usize); worker_state.heap.clear(); // Change limit to number of pieces worker_state.heap.set_limit(limit); @@ -470,7 +500,7 @@ where .stored_pieces .extract_if(|key, _offset| piece_indices_to_store.remove(key).is_none()) .for_each(|(_piece_index, offset)| { - caches.free_offsets.push_front(offset); + caches.dangling_free_offsets.push_front(offset); }); if let Some(metrics) = &self.metrics { @@ -544,7 +574,7 @@ where // It will likely result in higher read load on one disk and lower on another now // Find plot in which there is a place for new piece to be stored - let Some(offset) = caches.free_offsets.pop_front() else { + let Some(offset) = caches.dangling_free_offsets.pop_front() else { error!( %piece_index, "Failed to store piece in cache, there was no space" @@ -848,7 +878,7 @@ where } // There is free space in cache, need to find a free spot and place piece there None => { - let Some(offset) = caches.free_offsets.pop_front() else { + let Some(offset) = caches.dangling_free_offsets.pop_front() else { warn!( %piece_index, "Should have inserted piece into cache, but it didn't happen, this is an \ From 6c98cf6e8a956b7621c4533daf5c200ca0992242 Mon Sep 17 00:00:00 2001 From: tedious Date: Sat, 27 Jul 2024 23:53:28 +0800 Subject: [PATCH 22/41] opt: refactor farmer cache free offsets --- crates/subspace-farmer/src/farmer_cache.rs | 73 +++++++++++++++++++--- 1 file changed, 65 insertions(+), 8 deletions(-) diff --git a/crates/subspace-farmer/src/farmer_cache.rs b/crates/subspace-farmer/src/farmer_cache.rs index a9afbbb3a6..9f54e3959b 100644 --- a/crates/subspace-farmer/src/farmer_cache.rs +++ b/crates/subspace-farmer/src/farmer_cache.rs @@ -74,6 +74,7 @@ where #[derive(Debug, Clone)] struct CacheBackend { backend: Arc, + used_capacity: u32, total_capacity: u32, } @@ -89,9 +90,25 @@ impl CacheBackend { fn new(backend: Arc, total_capacity: u32) -> Self { Self { backend, + used_capacity: 0, total_capacity, } } + + fn next_free(&mut self) -> Option { + let offset = self.used_capacity; + if offset < self.total_capacity { + self.used_capacity += 1; + Some(PieceCacheOffset(offset)) + } else { + debug!(?offset, total_capacity = ?self.total_capacity, "No free space in cache backend"); + None + } + } + + fn free_size(&self) -> u32 { + self.total_capacity - self.used_capacity + } } #[derive(Debug, Clone)] @@ -101,6 +118,41 @@ struct PieceCachesState { backends: Vec, } +impl PieceCachesState +where + CacheIndex: Hash + Eq + Copy + fmt::Debug + fmt::Display + Send + Sync + 'static, + usize: From, + CacheIndex: TryFrom, +{ + fn pop_free_offset(&mut self) -> Option> { + match self.dangling_free_offsets.pop_front() { + Some(free_offset) => { + debug!(?free_offset, "Popped dangling free offset"); + Some(free_offset) + } + None => { + let mut sorted_backends = self + .backends + .iter_mut() + .enumerate() + .filter_map(|(cache_index, backend)| { + Some((CacheIndex::try_from(cache_index).ok()?, backend)) + }) + .collect::>(); + sorted_backends.sort_unstable_by_key(|(_, backend)| backend.free_size()); + sorted_backends + .into_iter() + .rev() + .find_map(|(cache_index, backend)| { + backend + .next_free() + .map(|free_offset| FarmerCacheOffset::new(cache_index, free_offset)) + }) + } + } + } +} + impl Default for PieceCachesState { fn default() -> Self { Self { @@ -310,7 +362,7 @@ where .enumerate() .filter_map(|(cache_index, new_cache)| { let total_capacity = new_cache.max_num_elements(); - let backend = CacheBackend::new(new_cache, total_capacity); + let mut backend = CacheBackend::new(new_cache, total_capacity); let Ok(cache_index) = CacheIndex::try_from(cache_index) else { warn!( ?piece_caches_number, @@ -326,6 +378,8 @@ where } let init_fut = async move { + let used_capacity = &mut backend.used_capacity; + // Hack with first collecting into `Option` with `Option::take()` call // later is to satisfy compiler that gets confused about ownership // otherwise @@ -367,6 +421,7 @@ where let offset = FarmerCacheOffset::new(cache_index, piece_offset); match maybe_piece_index { Some(piece_index) => { + *used_capacity = piece_offset.0 + 1; cache_stored_pieces .insert(RecordKey::from(piece_index.to_multihash()), offset); } @@ -412,9 +467,12 @@ where match maybe_cache { Ok(cache) => { let backend = cache.backend; - let free_offsets = cache.cache_free_offsets; stored_pieces.extend(cache.cache_stored_pieces.into_iter()); - dangling_free_offsets.extend(free_offsets.into_iter()); + dangling_free_offsets.extend( + cache.cache_free_offsets.into_iter().filter(|free_offset| { + free_offset.piece_offset.0 < backend.used_capacity + }), + ); backends.push(backend); } Err(_cancelled) => { @@ -500,6 +558,8 @@ where .stored_pieces .extract_if(|key, _offset| piece_indices_to_store.remove(key).is_none()) .for_each(|(_piece_index, offset)| { + // There is no need to adjust the `last_stored_offset` of the `backend` here, + // as the free_offset will be preferentially taken from the dangling free offsets caches.dangling_free_offsets.push_front(offset); }); @@ -570,11 +630,8 @@ where continue; }; - // TODO: Make the cache load as balanced as possible - // It will likely result in higher read load on one disk and lower on another now - // Find plot in which there is a place for new piece to be stored - let Some(offset) = caches.dangling_free_offsets.pop_front() else { + let Some(offset) = caches.pop_free_offset() else { error!( %piece_index, "Failed to store piece in cache, there was no space" @@ -878,7 +935,7 @@ where } // There is free space in cache, need to find a free spot and place piece there None => { - let Some(offset) = caches.dangling_free_offsets.pop_front() else { + let Some(offset) = caches.pop_free_offset() else { warn!( %piece_index, "Should have inserted piece into cache, but it didn't happen, this is an \ From 9cc17ea1e596ad131d3c78c752ec14e07ad7879e Mon Sep 17 00:00:00 2001 From: Nazar Mokrynskyi Date: Mon, 29 Jul 2024 22:29:39 +0300 Subject: [PATCH 23/41] TODO and a comment --- crates/subspace-farmer/src/farmer_cache.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/subspace-farmer/src/farmer_cache.rs b/crates/subspace-farmer/src/farmer_cache.rs index 9f54e3959b..fac97939d0 100644 --- a/crates/subspace-farmer/src/farmer_cache.rs +++ b/crates/subspace-farmer/src/farmer_cache.rs @@ -131,6 +131,8 @@ where Some(free_offset) } None => { + // Sort piece caches by number of stored pieces to fill those that are less + // populated first let mut sorted_backends = self .backends .iter_mut() @@ -426,6 +428,8 @@ where .insert(RecordKey::from(piece_index.to_multihash()), offset); } None => { + // TODO: Optimize to not store all free offsets, only dangling + // offsets are actually necessary cache_free_offsets.push(offset); } } From 8a545874937200bedde7e491005498cc66f3f448 Mon Sep 17 00:00:00 2001 From: Nazar Mokrynskyi Date: Wed, 31 Jul 2024 15:30:59 +0300 Subject: [PATCH 24/41] Switch to pre-release AES crate to remove workarounds for ARMv8 --- .cargo/config.toml | 5 -- .github/workflows/rust.yml | 4 +- .github/workflows/snapshot-build.yml | 7 +-- Cargo.lock | 62 +++++++++++++++++++++--- crates/subspace-proof-of-time/Cargo.toml | 2 +- crates/subspace-proof-of-time/src/aes.rs | 16 +++--- 6 files changed, 66 insertions(+), 30 deletions(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index d9b2e600cf..6f94bc4fba 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,8 +1,3 @@ [target.'cfg(target_arch = "x86_64")'] # Require AES-NI on x86-64 by default rustflags = ["-C", "target-feature=+aes"] - -[target.'cfg(target_arch = "aarch64")'] -# TODO: AES flag is such that we have decent performance on ARMv8, remove once `aes` crate with MSRV bump ships: -# https://github.com/RustCrypto/block-ciphers/pull/395 -rustflags = ["--cfg", "aes_armv8"] diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index d86f3ae74a..9b914d6862 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -24,9 +24,7 @@ env: CARGO_TERM_COLOR: always # Build smaller artifacts to avoid running out of space in CI # TODO: Try to remove once https://github.com/paritytech/substrate/issues/11538 is resolved - # TODO: AES flag is such that we have decent performance on ARMv8, remove once `aes` crate with MSRV bump ships: - # https://github.com/RustCrypto/block-ciphers/pull/395 - RUSTFLAGS: -C strip=symbols -C opt-level=s --cfg aes_armv8 + RUSTFLAGS: -C strip=symbols -C opt-level=s jobs: cargo-fmt: diff --git a/.github/workflows/snapshot-build.yml b/.github/workflows/snapshot-build.yml index c4c7613961..a70b3fc145 100644 --- a/.github/workflows/snapshot-build.yml +++ b/.github/workflows/snapshot-build.yml @@ -103,15 +103,10 @@ jobs: - os: ${{ fromJson(github.repository_owner == 'autonomys' && '["self-hosted", "ubuntu-20.04-x86-64"]' || '"ubuntu-20.04"') }} target: aarch64-unknown-linux-gnu suffix: ubuntu-aarch64-${{ github.ref_name }} - # TODO: AES flag is such that we have decent performance on ARMv8, remove once `aes` crate with MSRV bump ships: - # https://github.com/RustCrypto/block-ciphers/pull/395 - rustflags: "-C linker=aarch64-linux-gnu-gcc --cfg aes_armv8" + rustflags: "-C linker=aarch64-linux-gnu-gcc" - os: ${{ fromJson(github.repository_owner == 'autonomys' && '["self-hosted", "macos-14-arm64"]' || '"macos-14"') }} target: aarch64-apple-darwin suffix: macos-aarch64-${{ github.ref_name }} - # TODO: AES flag is such that we have decent performance on ARMv8, remove once `aes` crate with MSRV bump ships: - # https://github.com/RustCrypto/block-ciphers/pull/395 - rustflags: "--cfg aes_armv8" - os: ${{ fromJson(github.repository_owner == 'autonomys' && '["self-hosted", "windows-server-2022-x86-64"]' || '"windows-2022"') }} target: x86_64-pc-windows-msvc suffix: windows-x86_64-skylake-${{ github.ref_name }} diff --git a/Cargo.lock b/Cargo.lock index 1e2ebf27d3..d8fd6fa63f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -224,7 +224,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" dependencies = [ - "crypto-common", + "crypto-common 0.1.6", "generic-array 0.14.7", ] @@ -239,6 +239,17 @@ dependencies = [ "cpufeatures", ] +[[package]] +name = "aes" +version = "0.9.0-pre.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "183b3b4639f8f7237857117abb74f3dc8648b77e67ff78d9cb6959fd7e76f387" +dependencies = [ + "cfg-if", + "cipher 0.5.0-pre.6", + "cpufeatures", +] + [[package]] name = "aes-gcm" version = "0.10.3" @@ -246,7 +257,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" dependencies = [ "aead", - "aes", + "aes 0.8.4", "cipher 0.4.4", "ctr", "ghash", @@ -1824,11 +1835,21 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common", - "inout", + "crypto-common 0.1.6", + "inout 0.1.3", "zeroize", ] +[[package]] +name = "cipher" +version = "0.5.0-pre.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c71c893d5a1e8257048dbb29954d2e1f85f091a150304f1defe4ca2806da5d3f" +dependencies = [ + "crypto-common 0.2.0-rc.0", + "inout 0.2.0-rc.0", +] + [[package]] name = "clap" version = "4.5.7" @@ -2299,6 +2320,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.0-rc.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c070b79a496dccd931229780ad5bbedd535ceff6c3565605a8e440e18e1aa2b" +dependencies = [ + "hybrid-array", +] + [[package]] name = "crypto-mac" version = "0.7.0" @@ -2578,7 +2608,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", "const-oid", - "crypto-common", + "crypto-common 0.1.6", "subtle 2.6.0", ] @@ -4991,6 +5021,15 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "hybrid-array" +version = "0.2.0-rc.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d306b679262030ad8813a82d4915fc04efff97776e4db7f8eb5137039d56400" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "0.14.29" @@ -5222,6 +5261,15 @@ dependencies = [ "generic-array 0.14.7", ] +[[package]] +name = "inout" +version = "0.2.0-rc.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbc33218cf9ce7b927426ee4ad3501bcc5d8c26bf5fb4a82849a083715aca427" +dependencies = [ + "hybrid-array", +] + [[package]] name = "instant" version = "0.1.13" @@ -13047,7 +13095,7 @@ dependencies = [ name = "subspace-proof-of-time" version = "0.1.0" dependencies = [ - "aes", + "aes 0.9.0-pre.1", "core_affinity", "criterion", "rand 0.8.5", @@ -14281,7 +14329,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" dependencies = [ - "crypto-common", + "crypto-common 0.1.6", "subtle 2.6.0", ] diff --git a/crates/subspace-proof-of-time/Cargo.toml b/crates/subspace-proof-of-time/Cargo.toml index d4dae815e2..2e0117c4c3 100644 --- a/crates/subspace-proof-of-time/Cargo.toml +++ b/crates/subspace-proof-of-time/Cargo.toml @@ -15,7 +15,7 @@ include = [ bench = false [dependencies] -aes = "0.8.4" +aes = "0.9.0-pre.1" subspace-core-primitives = { version = "0.1.0", path = "../subspace-core-primitives", default-features = false } thiserror = { version = "1.0.61", optional = true } diff --git a/crates/subspace-proof-of-time/src/aes.rs b/crates/subspace-proof-of-time/src/aes.rs index 36cdc5f166..16d92caf19 100644 --- a/crates/subspace-proof-of-time/src/aes.rs +++ b/crates/subspace-proof-of-time/src/aes.rs @@ -7,8 +7,8 @@ mod x86_64; #[cfg(not(feature = "std"))] extern crate alloc; -use aes::cipher::generic_array::GenericArray; -use aes::cipher::{BlockDecrypt, BlockEncrypt, KeyInit}; +use aes::cipher::array::Array; +use aes::cipher::{BlockCipherDecrypt, BlockCipherEncrypt, KeyInit}; use aes::Aes128; use subspace_core_primitives::{PotCheckpoints, PotKey, PotOutput, PotSeed}; @@ -26,9 +26,9 @@ pub(crate) fn create(seed: PotSeed, key: PotKey, checkpoint_iterations: u32) -> #[cfg(any(not(target_arch = "x86_64"), test))] #[inline(always)] fn create_generic(seed: PotSeed, key: PotKey, checkpoint_iterations: u32) -> PotCheckpoints { - let key = GenericArray::from(*key); + let key = Array::from(*key); let cipher = Aes128::new(&key); - let mut cur_block = GenericArray::from(*seed); + let mut cur_block = Array::from(*seed); let mut checkpoints = PotCheckpoints::default(); for checkpoint in checkpoints.iter_mut() { @@ -54,17 +54,17 @@ pub(crate) fn verify_sequential( ) -> bool { assert_eq!(checkpoint_iterations % 2, 0); - let key = GenericArray::from(*key); + let key = Array::from(*key); let cipher = Aes128::new(&key); let mut inputs = Vec::with_capacity(checkpoints.len()); - inputs.push(GenericArray::from(*seed)); + inputs.push(Array::from(*seed)); for &checkpoint in checkpoints.iter().rev().skip(1).rev() { - inputs.push(GenericArray::from(*checkpoint)); + inputs.push(Array::from(*checkpoint)); } let mut outputs = checkpoints .iter() - .map(|&checkpoint| GenericArray::from(*checkpoint)) + .map(|&checkpoint| Array::from(*checkpoint)) .collect::>(); for _ in 0..checkpoint_iterations / 2 { From 9e730f742d38b222e9a54c4e9a56951509be2674 Mon Sep 17 00:00:00 2001 From: Nazar Mokrynskyi Date: Thu, 1 Aug 2024 06:14:15 +0300 Subject: [PATCH 25/41] Update Rust toolchain to latest nightly --- Dockerfile-bootstrap-node | 2 +- Dockerfile-bootstrap-node.aarch64 | 2 +- Dockerfile-farmer | 2 +- Dockerfile-farmer.aarch64 | 2 +- Dockerfile-node | 2 +- Dockerfile-node.aarch64 | 2 +- Dockerfile-runtime | 2 +- .../Cargo.toml | 5 +++ crates/pallet-subspace-mmr/Cargo.toml | 5 +++ .../src/consensus/types.rs | 18 +++++----- crates/subspace-archiving/src/lib.rs | 2 +- crates/subspace-core-primitives/src/lib.rs | 1 - crates/subspace-core-primitives/src/pieces.rs | 10 +++--- .../subspace-core-primitives/src/segments.rs | 8 +++-- crates/subspace-farmer-components/src/lib.rs | 1 - .../src/plotting.rs | 4 +-- crates/subspace-farmer/src/lib.rs | 2 -- .../unbuffered_io_file_windows.rs | 10 +++--- .../request_response_factory.rs | 14 ++++---- .../src/protocols/reserved_peers.rs | 4 +-- .../src/stateless_runtime.rs | 6 ++-- .../client/consensus-relay-chain/src/lib.rs | 4 +-- domains/client/domain-operator/src/lib.rs | 33 ++++++++++--------- domains/client/domain-operator/src/tests.rs | 1 + domains/pallets/block-fees/Cargo.toml | 5 +++ domains/pallets/domain-id/Cargo.toml | 5 +++ domains/pallets/domain-sudo/Cargo.toml | 5 +++ domains/pallets/evm_nonce_tracker/Cargo.toml | 5 +++ rust-toolchain.toml | 2 +- 29 files changed, 97 insertions(+), 67 deletions(-) diff --git a/Dockerfile-bootstrap-node b/Dockerfile-bootstrap-node index 0970ad7ab0..b51a9719d5 100644 --- a/Dockerfile-bootstrap-node +++ b/Dockerfile-bootstrap-node @@ -1,6 +1,6 @@ FROM ubuntu:20.04 -ARG RUSTC_VERSION=nightly-2024-04-22 +ARG RUSTC_VERSION=nightly-2024-08-01 ARG PROFILE=production ARG RUSTFLAGS # Workaround for https://github.com/rust-lang/cargo/issues/10583 diff --git a/Dockerfile-bootstrap-node.aarch64 b/Dockerfile-bootstrap-node.aarch64 index c6829fcf39..87cde19cd4 100644 --- a/Dockerfile-bootstrap-node.aarch64 +++ b/Dockerfile-bootstrap-node.aarch64 @@ -1,6 +1,6 @@ FROM ubuntu:20.04 -ARG RUSTC_VERSION=nightly-2024-04-22 +ARG RUSTC_VERSION=nightly-2024-08-01 ARG PROFILE=production ARG RUSTFLAGS # Workaround for https://github.com/rust-lang/cargo/issues/10583 diff --git a/Dockerfile-farmer b/Dockerfile-farmer index 5104b68319..e38f4b06dd 100644 --- a/Dockerfile-farmer +++ b/Dockerfile-farmer @@ -1,6 +1,6 @@ FROM ubuntu:20.04 -ARG RUSTC_VERSION=nightly-2024-04-22 +ARG RUSTC_VERSION=nightly-2024-08-01 ARG PROFILE=production ARG RUSTFLAGS # Workaround for https://github.com/rust-lang/cargo/issues/10583 diff --git a/Dockerfile-farmer.aarch64 b/Dockerfile-farmer.aarch64 index 074caae241..76291a3cbd 100644 --- a/Dockerfile-farmer.aarch64 +++ b/Dockerfile-farmer.aarch64 @@ -1,6 +1,6 @@ FROM ubuntu:20.04 -ARG RUSTC_VERSION=nightly-2024-04-22 +ARG RUSTC_VERSION=nightly-2024-08-01 ARG PROFILE=production ARG RUSTFLAGS # Workaround for https://github.com/rust-lang/cargo/issues/10583 diff --git a/Dockerfile-node b/Dockerfile-node index 00e52ba275..cd0fe490f7 100644 --- a/Dockerfile-node +++ b/Dockerfile-node @@ -1,6 +1,6 @@ FROM ubuntu:20.04 -ARG RUSTC_VERSION=nightly-2024-04-22 +ARG RUSTC_VERSION=nightly-2024-08-01 ARG PROFILE=production ARG RUSTFLAGS # Workaround for https://github.com/rust-lang/cargo/issues/10583 diff --git a/Dockerfile-node.aarch64 b/Dockerfile-node.aarch64 index 77f39987e9..a5c8d05fda 100644 --- a/Dockerfile-node.aarch64 +++ b/Dockerfile-node.aarch64 @@ -1,6 +1,6 @@ FROM ubuntu:20.04 -ARG RUSTC_VERSION=nightly-2024-04-22 +ARG RUSTC_VERSION=nightly-2024-08-01 ARG PROFILE=production ARG RUSTFLAGS # Workaround for https://github.com/rust-lang/cargo/issues/10583 diff --git a/Dockerfile-runtime b/Dockerfile-runtime index ebab4afeaf..e7ade29e7e 100644 --- a/Dockerfile-runtime +++ b/Dockerfile-runtime @@ -1,6 +1,6 @@ FROM ubuntu:20.04 -ARG RUSTC_VERSION=nightly-2024-04-22 +ARG RUSTC_VERSION=nightly-2024-08-01 ARG PROFILE=production ARG RUSTFLAGS # Workaround for https://github.com/rust-lang/cargo/issues/10583 diff --git a/crates/pallet-grandpa-finality-verifier/Cargo.toml b/crates/pallet-grandpa-finality-verifier/Cargo.toml index a906157640..8f829444e5 100644 --- a/crates/pallet-grandpa-finality-verifier/Cargo.toml +++ b/crates/pallet-grandpa-finality-verifier/Cargo.toml @@ -47,3 +47,8 @@ std = [ "sp-runtime/std", "sp-std/std", ] +try-runtime = [ + "frame-support/try-runtime", + "frame-system/try-runtime", + "sp-runtime/try-runtime", +] diff --git a/crates/pallet-subspace-mmr/Cargo.toml b/crates/pallet-subspace-mmr/Cargo.toml index d78dcd516b..08df60c74a 100644 --- a/crates/pallet-subspace-mmr/Cargo.toml +++ b/crates/pallet-subspace-mmr/Cargo.toml @@ -39,3 +39,8 @@ std = [ "sp-std/std", "sp-subspace-mmr/std" ] +try-runtime = [ + "frame-support/try-runtime", + "frame-system/try-runtime", + "sp-runtime/try-runtime", +] diff --git a/crates/sc-subspace-block-relay/src/consensus/types.rs b/crates/sc-subspace-block-relay/src/consensus/types.rs index b39e4e22fa..9b14b36079 100644 --- a/crates/sc-subspace-block-relay/src/consensus/types.rs +++ b/crates/sc-subspace-block-relay/src/consensus/types.rs @@ -1,4 +1,13 @@ //! Consensus related types. +//! +//! The relay protocol needs to expose these data types to be included +//! in the top level data types: +//! 1. The request/response types to be included in +//! [`ProtocolInitialRequest`]/[`ProtocolInitialResponse`]. +//! 2. The handshake message to be included in `ProtocolMessage`. +//! The corresponding handshake response is not included in the +//! top level messages, and is private to the protocol +//! implementation. use crate::protocol::compact_block::{ CompactBlockHandshake, CompactBlockInitialRequest, CompactBlockInitialResponse, @@ -24,15 +33,6 @@ const DOWNLOAD_LABEL: &str = "client_download"; const DOWNLOAD_BLOCKS: &str = "blocks"; const DOWNLOAD_BYTES: &str = "bytes"; -/// The relay protocol needs to expose these data types to be included -/// in the top level data types: -/// 1. The request/response types to be included in -/// `ProtocolInitialRequest`/`ProtocolInitialResponse`. -/// 2. The handshake message to be included in `ProtocolMessage`. -/// The corresponding handshake response is not included in the -/// top level messages, and is private to the protocol -/// implementation. - /// Client -> server request. #[derive(From, Encode, Decode)] pub(crate) enum ConsensusRequest { diff --git a/crates/subspace-archiving/src/lib.rs b/crates/subspace-archiving/src/lib.rs index 9ceae191f4..eb19b2b622 100644 --- a/crates/subspace-archiving/src/lib.rs +++ b/crates/subspace-archiving/src/lib.rs @@ -15,7 +15,7 @@ //! Collection of modules used for dealing with archived state of Subspace Network. #![cfg_attr(not(feature = "std"), no_std)] -#![feature(array_chunks, extract_if, iter_collect_into, slice_flatten)] +#![feature(array_chunks, extract_if, iter_collect_into)] pub mod archiver; pub mod piece_reconstructor; diff --git a/crates/subspace-core-primitives/src/lib.rs b/crates/subspace-core-primitives/src/lib.rs index 1a45d11a4b..625527a550 100644 --- a/crates/subspace-core-primitives/src/lib.rs +++ b/crates/subspace-core-primitives/src/lib.rs @@ -25,7 +25,6 @@ const_try, new_uninit, portable_simd, - slice_flatten, step_trait )] diff --git a/crates/subspace-core-primitives/src/pieces.rs b/crates/subspace-core-primitives/src/pieces.rs index af252a41a3..e6d864b938 100644 --- a/crates/subspace-core-primitives/src/pieces.rs +++ b/crates/subspace-core-primitives/src/pieces.rs @@ -332,14 +332,14 @@ impl Default for RawRecord { impl AsRef<[u8]> for RawRecord { #[inline] fn as_ref(&self) -> &[u8] { - self.0.as_slice().flatten() + self.0.as_slice().as_flattened() } } impl AsMut<[u8]> for RawRecord { #[inline] fn as_mut(&mut self) -> &mut [u8] { - self.0.as_mut_slice().flatten_mut() + self.0.as_mut_slice().as_flattened_mut() } } @@ -487,14 +487,14 @@ impl Default for Record { impl AsRef<[u8]> for Record { #[inline] fn as_ref(&self) -> &[u8] { - self.0.flatten() + self.0.as_flattened() } } impl AsMut<[u8]> for Record { #[inline] fn as_mut(&mut self) -> &mut [u8] { - self.0.flatten_mut() + self.0.as_flattened_mut() } } @@ -604,7 +604,7 @@ impl Record { length, ) }; - for byte in slice.flatten_mut().flatten_mut() { + for byte in slice.as_flattened_mut().as_flattened_mut() { byte.write(0); } } diff --git a/crates/subspace-core-primitives/src/segments.rs b/crates/subspace-core-primitives/src/segments.rs index d839d3fd7f..f4a49f9c36 100644 --- a/crates/subspace-core-primitives/src/segments.rs +++ b/crates/subspace-core-primitives/src/segments.rs @@ -284,7 +284,9 @@ impl Default for RecordedHistorySegment { impl AsRef<[u8]> for RecordedHistorySegment { #[inline] fn as_ref(&self) -> &[u8] { - RawRecord::slice_to_repr(&self.0).flatten().flatten() + RawRecord::slice_to_repr(&self.0) + .as_flattened() + .as_flattened() } } @@ -292,8 +294,8 @@ impl AsMut<[u8]> for RecordedHistorySegment { #[inline] fn as_mut(&mut self) -> &mut [u8] { RawRecord::slice_mut_to_repr(&mut self.0) - .flatten_mut() - .flatten_mut() + .as_flattened_mut() + .as_flattened_mut() } } diff --git a/crates/subspace-farmer-components/src/lib.rs b/crates/subspace-farmer-components/src/lib.rs index d7061b94f8..71fe389176 100644 --- a/crates/subspace-farmer-components/src/lib.rs +++ b/crates/subspace-farmer-components/src/lib.rs @@ -11,7 +11,6 @@ never_type, new_uninit, portable_simd, - slice_flatten, try_blocks )] #![warn(rust_2018_idioms, missing_debug_implementations, missing_docs)] diff --git a/crates/subspace-farmer-components/src/plotting.rs b/crates/subspace-farmer-components/src/plotting.rs index 47c5e127fd..d5c2469e12 100644 --- a/crates/subspace-farmer-components/src/plotting.rs +++ b/crates/subspace-farmer-components/src/plotting.rs @@ -672,8 +672,8 @@ async fn download_sector_internal( // Fancy way to insert value in order to avoid going through stack (if naive de-referencing // is used) and potentially causing stack overflow as the result record - .flatten_mut() - .copy_from_slice(piece.record().flatten()); + .as_flattened_mut() + .copy_from_slice(piece.record().as_flattened()); *metadata = RecordMetadata { commitment: *piece.commitment(), witness: *piece.witness(), diff --git a/crates/subspace-farmer/src/lib.rs b/crates/subspace-farmer/src/lib.rs index ef44709af4..f398c8552f 100644 --- a/crates/subspace-farmer/src/lib.rs +++ b/crates/subspace-farmer/src/lib.rs @@ -13,8 +13,6 @@ let_chains, never_type, result_flattening, - slice_flatten, - split_at_checked, trait_alias, try_blocks, type_alias_impl_trait, diff --git a/crates/subspace-farmer/src/single_disk_farm/unbuffered_io_file_windows.rs b/crates/subspace-farmer/src/single_disk_farm/unbuffered_io_file_windows.rs index 61ba9f4720..d92dc6bf43 100644 --- a/crates/subspace-farmer/src/single_disk_farm/unbuffered_io_file_windows.rs +++ b/crates/subspace-farmer/src/single_disk_farm/unbuffered_io_file_windows.rs @@ -185,13 +185,13 @@ impl UnbufferedIoFileWindows { // done with granularity of physical sector size let offset_in_buffer = (offset % self.physical_sector_size as u64) as usize; self.file.read_exact_at( - &mut scratch_buffer.flatten_mut()[..(bytes_to_read + offset_in_buffer) + &mut scratch_buffer.as_flattened_mut()[..(bytes_to_read + offset_in_buffer) .div_ceil(self.physical_sector_size) * self.physical_sector_size], offset / self.physical_sector_size as u64 * self.physical_sector_size as u64, )?; - Ok(&scratch_buffer.flatten()[offset_in_buffer..][..bytes_to_read]) + Ok(&scratch_buffer.as_flattened()[offset_in_buffer..][..bytes_to_read]) } /// Panics on writes over `MAX_READ_SIZE` (including padding on both ends) @@ -202,7 +202,7 @@ impl UnbufferedIoFileWindows { offset: u64, ) -> io::Result<()> { // This is guaranteed by `UnbufferedIoFileWindows::open()` - assert!(scratch_buffer.flatten().len() >= MAX_READ_SIZE); + assert!(scratch_buffer.as_flattened().len() >= MAX_READ_SIZE); let aligned_offset = offset / self.physical_sector_size as u64 * self.physical_sector_size as u64; @@ -212,13 +212,13 @@ impl UnbufferedIoFileWindows { * self.physical_sector_size; if padding == 0 && bytes_to_read == bytes_to_write.len() { - let scratch_buffer = &mut scratch_buffer.flatten_mut()[..bytes_to_read]; + let scratch_buffer = &mut scratch_buffer.as_flattened_mut()[..bytes_to_read]; scratch_buffer.copy_from_slice(bytes_to_write); self.file.write_all_at(scratch_buffer, offset)?; } else { // Read whole pages where `bytes_to_write` will be written self.read_exact_at_internal(scratch_buffer, bytes_to_read, aligned_offset)?; - let scratch_buffer = &mut scratch_buffer.flatten_mut()[..bytes_to_read]; + let scratch_buffer = &mut scratch_buffer.as_flattened_mut()[..bytes_to_read]; // Update contents of existing pages and write into the file scratch_buffer[padding..][..bytes_to_write.len()].copy_from_slice(bytes_to_write); self.file.write_all_at(scratch_buffer, aligned_offset)?; diff --git a/crates/subspace-networking/src/protocols/request_response/request_response_factory.rs b/crates/subspace-networking/src/protocols/request_response/request_response_factory.rs index fd1eca1e94..6859869aed 100644 --- a/crates/subspace-networking/src/protocols/request_response/request_response_factory.rs +++ b/crates/subspace-networking/src/protocols/request_response/request_response_factory.rs @@ -22,16 +22,16 @@ //! A request-response protocol works in the following way: //! //! - For every emitted request, a new substream is open and the protocol is negotiated. If the -//! remote supports the protocol, the size of the request is sent as a LEB128 number, followed -//! with the request itself. The remote then sends the size of the response as a LEB128 number, -//! followed with the response. +//! remote supports the protocol, the size of the request is sent as a LEB128 number, followed +//! with the request itself. The remote then sends the size of the response as a LEB128 number, +//! followed with the response. //! //! - Requests have a certain time limit before they time out. This time includes the time it -//! takes to send/receive the request and response. +//! takes to send/receive the request and response. //! //! - If provided, a ["requests processing"](ProtocolConfig::inbound_queue) channel -//! is used to handle incoming requests. - +//! is used to handle incoming requests. +//! //! Original file commit: #[cfg(test)] @@ -163,7 +163,7 @@ pub struct IncomingRequest { /// 1. Drop `pending_response` and thus not changing the reputation of the peer. /// /// 2. Sending an `Err(())` via `pending_response`, optionally including reputation changes for - /// the given peer. + /// the given peer. pub pending_response: oneshot::Sender, } diff --git a/crates/subspace-networking/src/protocols/reserved_peers.rs b/crates/subspace-networking/src/protocols/reserved_peers.rs index 49857472c2..b5f778319c 100644 --- a/crates/subspace-networking/src/protocols/reserved_peers.rs +++ b/crates/subspace-networking/src/protocols/reserved_peers.rs @@ -31,9 +31,9 @@ use crate::utils::strip_peer_id; /// Each `ReservedPeerState` can be in one of the following states, represented by the /// `ConnectionStatus` enum: /// 1. `NotConnected`: This state indicates that the peer is currently not connected. -/// The time for the next connection attempt is scheduled and can be queried. +/// The time for the next connection attempt is scheduled and can be queried. /// 2. `PendingConnection`: This state means that a connection attempt to the peer is currently -/// in progress. +/// in progress. /// 3. `Connected`: This state signals that the peer is currently connected. /// /// The protocol will attempt to establish a connection to a `NotConnected` peer after a set delay, diff --git a/domains/client/block-preprocessor/src/stateless_runtime.rs b/domains/client/block-preprocessor/src/stateless_runtime.rs index 37c460bbbb..6a61d91ea4 100644 --- a/domains/client/block-preprocessor/src/stateless_runtime.rs +++ b/domains/client/block-preprocessor/src/stateless_runtime.rs @@ -26,10 +26,10 @@ use subspace_runtime_primitives::Moment; /// /// NOTE: /// - This is only supposed to be used when no domain client available, i.e., when the -/// caller does not own the entire domain state. +/// caller does not own the entire domain state. /// - This perfectly fits the runtime APIs that are purely stateless, but it's also usable -/// for the stateful APIs. If some states are used inside a runtime api, these states must -/// be provided and set before dispatching otherwise [`StatelessRuntime`] may give invalid output. +/// for the stateful APIs. If some states are used inside a runtime api, these states must +/// be provided and set before dispatching otherwise [`StatelessRuntime`] may give invalid output. pub struct StatelessRuntime { executor: Arc, runtime_code: Cow<'static, [u8]>, diff --git a/domains/client/consensus-relay-chain/src/lib.rs b/domains/client/consensus-relay-chain/src/lib.rs index 323b45eb68..ab6a8f10dc 100644 --- a/domains/client/consensus-relay-chain/src/lib.rs +++ b/domains/client/consensus-relay-chain/src/lib.rs @@ -26,10 +26,10 @@ //! 2. This parachain candidate is send to the parachain validators that are part of the relay chain. //! //! 3. The parachain validators validate at most X different parachain candidates, where X is the -//! total number of parachain validators. +//! total number of parachain validators. //! //! 4. The parachain candidate that is backed by the most validators is chosen by the relay-chain -//! block producer to be added as backed candidate on chain. +//! block producer to be added as backed candidate on chain. //! //! 5. After the parachain candidate got backed and included, all collators start at 1. diff --git a/domains/client/domain-operator/src/lib.rs b/domains/client/domain-operator/src/lib.rs index 26d7f9797b..1d1b0a245e 100644 --- a/domains/client/domain-operator/src/lib.rs +++ b/domains/client/domain-operator/src/lib.rs @@ -29,30 +29,31 @@ //! on the execution layer, they provide the necessary computational resources to maintain the //! blockchain state by running domains. Some deposits as the stake are required to be an operator. //! -//! Specifically, operators have the responsibity of producing a [`Bundle`] which contains a +//! Specifically, operators have the responsibility of producing a [`Bundle`] which contains a //! number of [`ExecutionReceipt`]s on each slot notified from the consensus chain. The operators //! are primarily driven by two events from the consensus chain. //! //! - On each new slot, operators will attempt to solve a domain-specific bundle election -//! challenge derived from a global randomness provided by the consensus chain. Upon finding -//! a solution to the challenge, they will start producing a bundle: they will collect a set -//! of extrinsics from the transaction pool which are verified to be able to cover the transaction -//! fee. With these colltected extrinsics, the bundle election solution and proper receipts, a -//! [`Bundle`] can be constructed and then be submitted to the consensus chain. The transactions -//! included in each bundle are uninterpretable blob from the consensus chain's perspective. +//! challenge derived from a global randomness provided by the consensus chain. Upon finding +//! a solution to the challenge, they will start producing a bundle: they will collect a set +//! of extrinsics from the transaction pool which are verified to be able to cover the transaction +//! fee. With these collected extrinsics, the bundle election solution and proper receipts, a +//! [`Bundle`] can be constructed and then be submitted to the consensus chain. The transactions +//! included in each bundle are uninterpretable blob from the consensus chain's perspective. //! //! - On each imported consensus block, operators will extract all the needed bundles from it -//! and convert the bundles to a list of extrinsics, construct a custom [`BlockBuilder`] to -//! build a domain block. The execution trace of all the extrinsics and hooks like -//! `initialize_block`/`finalize_block` will be recorded during the domain block execution. -//! Once the domain block is imported successfully, the [`ExecutionReceipt`] of this block -//! will be generated and stored locally. +//! and convert the bundles to a list of extrinsics, construct a custom [`BlockBuilder`] to +//! build a domain block. The execution trace of all the extrinsics and hooks like +//! `initialize_block`/`finalize_block` will be recorded during the domain block execution. +//! Once the domain block is imported successfully, the [`ExecutionReceipt`] of this block +//! will be generated and stored locally. //! //! The receipt of each domain block contains all the intermediate state roots during the block -//! execution, which will be gossiped in the domain subnet (in future). All operators whether running as an -//! authority or a full node will compute each block and generate an execution receipt independently, -//! once the execution receipt received from the network does not match the one produced locally, -//! a [`FraudProof`] will be generated and reported to the consensus chain accordingly. +//! execution, which will be gossiped in the domain subnet (in future). All operators whether +//! running as an authority or a full node will compute each block and generate an execution receipt +//! independently, once the execution receipt received from the network does not match the one +//! produced locally, a [`FraudProof`] will be generated and reported to the consensus chain +//! accordingly. //! //! [`BlockBuilder`]: ../domain_block_builder/struct.BlockBuilder.html //! [`FraudProof`]: ../sp_domains/struct.FraudProof.html diff --git a/domains/client/domain-operator/src/tests.rs b/domains/client/domain-operator/src/tests.rs index c86031f9c0..c57fd0711f 100644 --- a/domains/client/domain-operator/src/tests.rs +++ b/domains/client/domain-operator/src/tests.rs @@ -1060,6 +1060,7 @@ async fn test_long_trace_for_finalize_block_proof_creation_and_verification_shou /// - 2 to test the `apply_extrinsic` state transition with the inherent `set_consensus_chain_byte_fee` extrinsic /// - 3 to test the `apply_extrinsic` state transition with regular domain extrinsic /// - 4 to test the `finalize_block` state transition +/// /// TraceDiffType can be passed as `TraceDiffType::Shorter`, `TraceDiffType::Longer` /// and `TraceDiffType::Mismatch` async fn test_invalid_state_transition_proof_creation_and_verification( diff --git a/domains/pallets/block-fees/Cargo.toml b/domains/pallets/block-fees/Cargo.toml index 558d0776a5..35e46eccf5 100644 --- a/domains/pallets/block-fees/Cargo.toml +++ b/domains/pallets/block-fees/Cargo.toml @@ -37,3 +37,8 @@ std = [ "sp-runtime/std", "sp-std/std", ] +try-runtime = [ + "frame-support/try-runtime", + "frame-system/try-runtime", + "sp-runtime/try-runtime", +] diff --git a/domains/pallets/domain-id/Cargo.toml b/domains/pallets/domain-id/Cargo.toml index 74da7e46b9..77581ce46c 100644 --- a/domains/pallets/domain-id/Cargo.toml +++ b/domains/pallets/domain-id/Cargo.toml @@ -36,3 +36,8 @@ std = [ ] runtime-benchmarks = [] +try-runtime = [ + "frame-support/try-runtime", + "frame-system/try-runtime", + "sp-runtime/try-runtime", +] diff --git a/domains/pallets/domain-sudo/Cargo.toml b/domains/pallets/domain-sudo/Cargo.toml index 0ed90f5628..6666243f84 100644 --- a/domains/pallets/domain-sudo/Cargo.toml +++ b/domains/pallets/domain-sudo/Cargo.toml @@ -32,3 +32,8 @@ std = [ "sp-runtime/std", "sp-domain-sudo/std" ] +try-runtime = [ + "frame-support/try-runtime", + "frame-system/try-runtime", + "sp-runtime/try-runtime", +] diff --git a/domains/pallets/evm_nonce_tracker/Cargo.toml b/domains/pallets/evm_nonce_tracker/Cargo.toml index d12f7ecdd2..4237048d31 100644 --- a/domains/pallets/evm_nonce_tracker/Cargo.toml +++ b/domains/pallets/evm_nonce_tracker/Cargo.toml @@ -30,3 +30,8 @@ std = [ "sp-core/std", "sp-runtime/std", ] +try-runtime = [ + "frame-support/try-runtime", + "frame-system/try-runtime", + "sp-runtime/try-runtime", +] diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 417abc8f22..602a7c8df5 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,5 +1,5 @@ [toolchain] -channel = "nightly-2024-04-22" +channel = "nightly-2024-08-01" components = ["rust-src"] targets = ["wasm32-unknown-unknown"] profile = "default" From 59c4be819112fd87bb434d692823264b60027aeb Mon Sep 17 00:00:00 2001 From: linning Date: Thu, 1 Aug 2024 05:06:54 +0800 Subject: [PATCH 26/41] Handle confirmed domain block within submit_receipt Signed-off-by: linning --- crates/pallet-domains/src/block_tree.rs | 1 - crates/pallet-domains/src/lib.rs | 44 ++++++++++++++++++++----- 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/crates/pallet-domains/src/block_tree.rs b/crates/pallet-domains/src/block_tree.rs index d4a5e01da6..d2f2f34212 100644 --- a/crates/pallet-domains/src/block_tree.rs +++ b/crates/pallet-domains/src/block_tree.rs @@ -51,7 +51,6 @@ pub enum Error { RuntimeNotFound, LastBlockNotFound, UnmatchedNewHeadReceipt, - UnexpectedConfirmedDomainBlock, } #[derive(TypeInfo, Debug, Encode, Decode, Clone, PartialEq, Eq)] diff --git a/crates/pallet-domains/src/lib.rs b/crates/pallet-domains/src/lib.rs index b7273e97db..5d23579a7d 100644 --- a/crates/pallet-domains/src/lib.rs +++ b/crates/pallet-domains/src/lib.rs @@ -1720,12 +1720,35 @@ mod pallet { ) .map_err(Error::::from)?; - // Singleton receipt are used to fill up the receipt gap and there should be no - // new domain block being confirmed before the gap is fill up - ensure!( - maybe_confirmed_domain_block_info.is_none(), - Error::::BlockTree(BlockTreeError::UnexpectedConfirmedDomainBlock), - ); + // NOTE: Skip the following staking related operations when benchmarking the + // `submit_receipt` call, these operations will be benchmarked separately. + #[cfg(not(feature = "runtime-benchmarks"))] + if let Some(confirmed_block_info) = maybe_confirmed_domain_block_info { + actual_weight = + actual_weight.saturating_add(T::WeightInfo::confirm_domain_block( + confirmed_block_info.operator_ids.len() as u32, + confirmed_block_info.invalid_bundle_authors.len() as u32, + )); + + refund_storage_fee::( + confirmed_block_info.total_storage_fee, + confirmed_block_info.paid_bundle_storage_fees, + ) + .map_err(Error::::from)?; + + do_reward_operators::( + domain_id, + confirmed_block_info.operator_ids.into_iter(), + confirmed_block_info.rewards, + ) + .map_err(Error::::from)?; + + do_mark_operators_as_slashed::( + confirmed_block_info.invalid_bundle_authors.into_iter(), + SlashedReason::InvalidBundle(confirmed_block_info.domain_block_number), + ) + .map_err(Error::::from)?; + } } } @@ -2750,7 +2773,7 @@ impl Pallet { .saturating_add( // NOTE: within `submit_bundle`, only one of (or none) `handle_bad_receipt` and // `confirm_domain_block` can happen, thus we use the `max` of them - + // // We use `MAX_BUNDLE_PER_BLOCK` number to assume the number of slashed operators. // We do not expect so many operators to be slashed but nontheless, if it did happen // we will limit the weight to 100 operators. @@ -2765,10 +2788,15 @@ impl Pallet { pub fn max_submit_receipt_weight() -> Weight { T::WeightInfo::submit_bundle() .saturating_add( + // NOTE: within `submit_bundle`, only one of (or none) `handle_bad_receipt` and + // `confirm_domain_block` can happen, thus we use the `max` of them + // // We use `MAX_BUNDLE_PER_BLOCK` number to assume the number of slashed operators. // We do not expect so many operators to be slashed but nontheless, if it did happen // we will limit the weight to 100 operators. - T::WeightInfo::handle_bad_receipt(MAX_BUNDLE_PER_BLOCK), + T::WeightInfo::handle_bad_receipt(MAX_BUNDLE_PER_BLOCK).max( + T::WeightInfo::confirm_domain_block(MAX_BUNDLE_PER_BLOCK, MAX_BUNDLE_PER_BLOCK), + ), ) .saturating_add(T::WeightInfo::slash_operator(MAX_NOMINATORS_TO_SLASH)) } From e6e7d4511b17fd3f9c40c7df02c924a46c3aac15 Mon Sep 17 00:00:00 2001 From: linning Date: Fri, 2 Aug 2024 19:07:25 +0800 Subject: [PATCH 27/41] Account the missed domain runtime upgrade into the domain best number onchain Signed-off-by: linning --- crates/pallet-domains/src/lib.rs | 32 ++++++++++++++----- crates/sp-domains/src/lib.rs | 2 +- crates/subspace-runtime/src/lib.rs | 2 +- .../src/domain_bundle_producer.rs | 8 ++--- .../src/domain_bundle_proposer.rs | 8 ++--- test/subspace-test-runtime/src/lib.rs | 2 +- 6 files changed, 35 insertions(+), 19 deletions(-) diff --git a/crates/pallet-domains/src/lib.rs b/crates/pallet-domains/src/lib.rs index 5d23579a7d..4a9d2e8554 100644 --- a/crates/pallet-domains/src/lib.rs +++ b/crates/pallet-domains/src/lib.rs @@ -654,6 +654,10 @@ mod pallet { /// successfully submitted to current consensus block, which mean a new domain block with this block /// number will be produce. Used as a pointer in `ExecutionInbox` to identify the current under building /// domain block, also used as a mapping of consensus block number to domain block number. + // + // NOTE: the `HeadDomainNumber` is lazily updated for the domain runtime upgrade block (which only include + // the runtime upgrade tx from the consensus chain and no any user submitted tx from the bundle), use + // `domain_best_number` for the actual best domain block #[pallet::storage] pub(super) type HeadDomainNumber = StorageMap<_, Identity, DomainId, DomainBlockNumberFor, ValueQuery>; @@ -763,6 +767,8 @@ mod pallet { UnexpectedReceiptGap, /// Expecting receipt gap when validating `submit_receipt` ExpectingReceiptGap, + /// Failed to get missed domain runtime upgrade count + FailedToGetMissedUpgradeCount, } #[derive(TypeInfo, Encode, Decode, PalletError, Debug, PartialEq)] @@ -2043,8 +2049,13 @@ impl Pallet { .and_then(|mut runtime_object| runtime_object.raw_genesis.take_runtime_code()) } - pub fn domain_best_number(domain_id: DomainId) -> Option> { - Some(HeadDomainNumber::::get(domain_id)) + pub fn domain_best_number(domain_id: DomainId) -> Result, BundleError> { + // The missed domain runtime upgrades will derive domain blocks thus should be accountted + // into the domain best number + let missed_upgrade = Self::missed_domain_runtime_upgrade(domain_id) + .map_err(|_| BundleError::FailedToGetMissedUpgradeCount)?; + + Ok(HeadDomainNumber::::get(domain_id) + missed_upgrade.into()) } pub fn runtime_id(domain_id: DomainId) -> Option { @@ -2288,7 +2299,7 @@ impl Pallet { // derived from the latest domain block, and the stale bundle (that verified against an old // domain block) produced by a lagging honest operator will be rejected. ensure!( - Self::receipt_gap(domain_id) <= One::one(), + Self::receipt_gap(domain_id)? <= One::one(), BundleError::UnexpectedReceiptGap, ); @@ -2324,7 +2335,7 @@ impl Pallet { // Singleton receipt is only allowed when there is a receipt gap ensure!( - Self::receipt_gap(domain_id) > One::one(), + Self::receipt_gap(domain_id)? > One::one(), BundleError::ExpectingReceiptGap, ); @@ -2698,6 +2709,11 @@ impl Pallet { // Start from the oldest non-confirmed ER to the head domain number let mut to_check = Self::latest_confirmed_domain_block_number(domain_id).saturating_add(One::one()); + + // NOTE: we use the `HeadDomainNumber` here instead of the `domain_best_number`, which include the + // missed domain runtime upgrade block, because we don't want to trigger empty bundle production + // for confirming these blocks since they only include runtime upgrade extrinsic and no any user + // submitted extrinsic. let head_number = HeadDomainNumber::::get(domain_id); while to_check <= head_number { @@ -2970,13 +2986,13 @@ impl Pallet { DomainSudoCalls::::get(domain_id).maybe_call } - // The gap between `HeadDomainNumber` and `HeadReceiptNumber` represent the number + // The gap between `domain_best_number` and `HeadReceiptNumber` represent the number // of receipt to be submitted - pub fn receipt_gap(domain_id: DomainId) -> DomainBlockNumberFor { - let head_domain_number = HeadDomainNumber::::get(domain_id); + pub fn receipt_gap(domain_id: DomainId) -> Result, BundleError> { + let domain_best_number = Self::domain_best_number(domain_id)?; let head_receipt_number = HeadReceiptNumber::::get(domain_id); - head_domain_number.saturating_sub(head_receipt_number) + Ok(domain_best_number.saturating_sub(head_receipt_number)) } } diff --git a/crates/sp-domains/src/lib.rs b/crates/sp-domains/src/lib.rs index b7014e2080..600c3a6987 100644 --- a/crates/sp-domains/src/lib.rs +++ b/crates/sp-domains/src/lib.rs @@ -780,7 +780,7 @@ impl ProofOfElection { } } -/// Singleton receipt submit along when there is a gap between `HeadDomainNumber` +/// Singleton receipt submit along when there is a gap between `domain_best_number` /// and `HeadReceiptNumber` #[derive(Debug, Decode, Encode, TypeInfo, PartialEq, Eq, Clone)] pub struct SingletonReceipt { diff --git a/crates/subspace-runtime/src/lib.rs b/crates/subspace-runtime/src/lib.rs index 6c372188e8..f5fd1ed3a2 100644 --- a/crates/subspace-runtime/src/lib.rs +++ b/crates/subspace-runtime/src/lib.rs @@ -1231,7 +1231,7 @@ impl_runtime_apis! { } fn domain_best_number(domain_id: DomainId) -> Option { - Domains::domain_best_number(domain_id) + Domains::domain_best_number(domain_id).ok() } fn execution_receipt(receipt_hash: DomainHash) -> Option> { diff --git a/domains/client/domain-operator/src/domain_bundle_producer.rs b/domains/client/domain-operator/src/domain_bundle_producer.rs index 2ebeb2c9f7..774f55ee55 100644 --- a/domains/client/domain-operator/src/domain_bundle_producer.rs +++ b/domains/client/domain-operator/src/domain_bundle_producer.rs @@ -183,7 +183,7 @@ where let domain_best_number = self.client.info().best_number; let consensus_chain_best_hash = self.consensus_client.info().best_hash; - let head_domain_number = self + let domain_best_number_onchain = self .consensus_client .runtime_api() .domain_best_number(consensus_chain_best_hash, self.domain_id)? @@ -239,7 +239,7 @@ where let receipt = self .domain_bundle_proposer - .load_next_receipt(head_domain_number, head_receipt_number)?; + .load_next_receipt(domain_best_number_onchain, head_receipt_number)?; let api_version = self .consensus_client @@ -256,10 +256,10 @@ where // instead of bundle // TODO: remove api runtime version check before next network if api_version >= 5 - && head_domain_number.saturating_sub(head_receipt_number) > 1u32.into() + && domain_best_number_onchain.saturating_sub(head_receipt_number) > 1u32.into() { info!( - ?head_domain_number, + ?domain_best_number_onchain, ?head_receipt_number, "🔖 Producing singleton receipt at slot {:?}", slot_info.slot diff --git a/domains/client/domain-operator/src/domain_bundle_proposer.rs b/domains/client/domain-operator/src/domain_bundle_proposer.rs index 3463559f3b..e6c05e7d09 100644 --- a/domains/client/domain-operator/src/domain_bundle_proposer.rs +++ b/domains/client/domain-operator/src/domain_bundle_proposer.rs @@ -397,18 +397,18 @@ where /// Returns the receipt in the next domain bundle. pub fn load_next_receipt( &self, - head_domain_number: NumberFor, + domain_best_number_onchain: NumberFor, head_receipt_number: NumberFor, ) -> sp_blockchain::Result> { tracing::trace!( - ?head_domain_number, + ?domain_best_number_onchain, ?head_receipt_number, "Collecting receipt" ); - // Both `head_domain_number` and `head_receipt_number` are zero means the domain just + // Both `domain_best_number_onchain` and `head_receipt_number` are zero means the domain just // instantiated and nothing have submitted yet so submit the genesis receipt - if head_domain_number.is_zero() && head_receipt_number.is_zero() { + if domain_best_number_onchain.is_zero() && head_receipt_number.is_zero() { let genesis_hash = self.client.info().genesis_hash; let genesis_header = self.client.header(genesis_hash)?.ok_or_else(|| { sp_blockchain::Error::Backend(format!( diff --git a/test/subspace-test-runtime/src/lib.rs b/test/subspace-test-runtime/src/lib.rs index 04b4877af2..1aa72cc470 100644 --- a/test/subspace-test-runtime/src/lib.rs +++ b/test/subspace-test-runtime/src/lib.rs @@ -1428,7 +1428,7 @@ impl_runtime_apis! { } fn domain_best_number(domain_id: DomainId) -> Option { - Domains::domain_best_number(domain_id) + Domains::domain_best_number(domain_id).ok() } fn execution_receipt(receipt_hash: DomainHash) -> Option> { From 9f7225b824c41817fa978d2dae2dba8f9a419706 Mon Sep 17 00:00:00 2001 From: linning Date: Fri, 2 Aug 2024 19:08:01 +0800 Subject: [PATCH 28/41] Bump consensus runtime spec version Signed-off-by: linning --- crates/subspace-runtime/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/subspace-runtime/src/lib.rs b/crates/subspace-runtime/src/lib.rs index f5fd1ed3a2..6c64505e67 100644 --- a/crates/subspace-runtime/src/lib.rs +++ b/crates/subspace-runtime/src/lib.rs @@ -118,7 +118,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { spec_name: create_runtime_str!("subspace"), impl_name: create_runtime_str!("subspace"), authoring_version: 0, - spec_version: 6, + spec_version: 7, impl_version: 0, apis: RUNTIME_API_VERSIONS, transaction_version: 0, From 2f048733e0a0923e68891449d59e2fcd954ef63c Mon Sep 17 00:00:00 2001 From: Nazar Mokrynskyi Date: Tue, 6 Aug 2024 02:17:40 +0300 Subject: [PATCH 29/41] Tiny cleanup --- crates/subspace-farmer/src/single_disk_farm.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/crates/subspace-farmer/src/single_disk_farm.rs b/crates/subspace-farmer/src/single_disk_farm.rs index 9a488bc3f1..234f4c65ef 100644 --- a/crates/subspace-farmer/src/single_disk_farm.rs +++ b/crates/subspace-farmer/src/single_disk_farm.rs @@ -364,7 +364,7 @@ pub enum SingleDiskFarmError { Io(#[from] io::Error), /// Failed to spawn task for blocking thread #[error("Failed to spawn task for blocking thread: {0}")] - TokioJoinError(#[from] tokio::task::JoinError), + TokioJoinError(#[from] task::JoinError), /// Piece cache error #[error("Piece cache error: {0}")] PieceCacheError(#[from] DiskPieceCacheError), @@ -803,7 +803,7 @@ impl SingleDiskFarm { create, } = options; - let single_disk_farm_init_fut = tokio::task::spawn_blocking({ + let single_disk_farm_init_fut = task::spawn_blocking({ let directory = directory.clone(); let farmer_app_info = farmer_app_info.clone(); let span = span.clone(); @@ -905,7 +905,7 @@ impl SingleDiskFarm { .spawn_handler(tokio_rayon_spawn_handler()) .build() .map_err(SingleDiskFarmError::FailedToCreateThreadPool)?; - let farming_plot_fut = tokio::task::spawn_blocking(|| { + let farming_plot_fut = task::spawn_blocking(|| { farming_thread_pool .install(move || { #[cfg(windows)] @@ -939,7 +939,7 @@ impl SingleDiskFarm { let span = span.clone(); let plot_file = Arc::clone(&plot_file); - let read_sector_record_chunks_mode_fut = tokio::task::spawn_blocking(move || { + let read_sector_record_chunks_mode_fut = task::spawn_blocking(move || { farming_thread_pool .install(move || { let _span_guard = span.enter(); @@ -960,7 +960,7 @@ impl SingleDiskFarm { faster_read_sector_record_chunks_mode_barrier.wait().await; - let plotting_join_handle = tokio::task::spawn_blocking({ + let plotting_join_handle = task::spawn_blocking({ let sectors_metadata = Arc::clone(§ors_metadata); let handlers = Arc::clone(&handlers); let sectors_being_modified = Arc::clone(§ors_being_modified); @@ -1073,7 +1073,7 @@ impl SingleDiskFarm { } })); - let farming_join_handle = tokio::task::spawn_blocking({ + let farming_join_handle = task::spawn_blocking({ let erasure_coding = erasure_coding.clone(); let handlers = Arc::clone(&handlers); let sectors_being_modified = Arc::clone(§ors_being_modified); @@ -1165,7 +1165,7 @@ impl SingleDiskFarm { global_mutex, ); - let reading_join_handle = tokio::task::spawn_blocking({ + let reading_join_handle = task::spawn_blocking({ let mut stop_receiver = stop_sender.subscribe(); let reading_fut = reading_fut.instrument(span.clone()); @@ -1197,7 +1197,7 @@ impl SingleDiskFarm { global_mutex, ); - let reading_join_handle = tokio::task::spawn_blocking({ + let reading_join_handle = task::spawn_blocking({ let mut stop_receiver = stop_sender.subscribe(); let reading_fut = reading_fut.instrument(span.clone()); From 67abebb64cf61ee2927547e1d0ee71781ae61190 Mon Sep 17 00:00:00 2001 From: Nazar Mokrynskyi Date: Tue, 6 Aug 2024 02:17:58 +0300 Subject: [PATCH 30/41] Extract logic that determines how allocated space is distributed across files of the farm into `AllocatedSpaceDistribution` struct that can be reused later --- .../subspace-farmer/src/single_disk_farm.rs | 173 +++++++++++------- .../src/single_disk_farm/plot_cache.rs | 3 +- .../src/single_disk_farm/plot_cache/tests.rs | 4 +- 3 files changed, 105 insertions(+), 75 deletions(-) diff --git a/crates/subspace-farmer/src/single_disk_farm.rs b/crates/subspace-farmer/src/single_disk_farm.rs index 234f4c65ef..be0939f7a7 100644 --- a/crates/subspace-farmer/src/single_disk_farm.rs +++ b/crates/subspace-farmer/src/single_disk_farm.rs @@ -655,6 +655,95 @@ impl ScrubTarget { } } +struct AllocatedSpaceDistribution { + piece_cache_capacity: u32, + plot_file_size: u64, + target_sector_count: u16, + metadata_file_size: u64, +} + +impl AllocatedSpaceDistribution { + fn new( + allocated_space: u64, + sector_size: u64, + cache_percentage: u8, + sector_metadata_size: u64, + ) -> Result { + let single_sector_overhead = sector_size + sector_metadata_size; + // Fixed space usage regardless of plot size + let fixed_space_usage = RESERVED_PLOT_METADATA + + RESERVED_FARM_INFO + + Identity::file_size() as u64 + + KnownPeersManager::file_size(KNOWN_PEERS_CACHE_SIZE) as u64; + // Calculate how many sectors can fit + let target_sector_count = { + let potentially_plottable_space = allocated_space.saturating_sub(fixed_space_usage) + / 100 + * (100 - u64::from(cache_percentage)); + // Do the rounding to make sure we have exactly as much space as fits whole number of + // sectors, account for disk sector size just in case + (potentially_plottable_space - DISK_SECTOR_SIZE as u64) / single_sector_overhead + }; + + if target_sector_count == 0 { + let mut single_plot_with_cache_space = + single_sector_overhead.div_ceil(100 - u64::from(cache_percentage)) * 100; + // Cache must not be empty, ensure it contains at least one element even if + // percentage-wise it will use more space + if single_plot_with_cache_space - single_sector_overhead + < DiskPieceCache::element_size() as u64 + { + single_plot_with_cache_space = + single_sector_overhead + DiskPieceCache::element_size() as u64; + } + + return Err(SingleDiskFarmError::InsufficientAllocatedSpace { + min_space: fixed_space_usage + single_plot_with_cache_space, + allocated_space, + }); + } + let plot_file_size = target_sector_count * sector_size; + // Align plot file size for disk sector size + let plot_file_size = + plot_file_size.div_ceil(DISK_SECTOR_SIZE as u64) * DISK_SECTOR_SIZE as u64; + + // Remaining space will be used for caching purposes + let piece_cache_capacity = if cache_percentage > 0 { + let cache_space = allocated_space + - fixed_space_usage + - plot_file_size + - (sector_metadata_size * target_sector_count); + (cache_space / u64::from(DiskPieceCache::element_size())) as u32 + } else { + 0 + }; + let target_sector_count = match SectorIndex::try_from(target_sector_count) { + Ok(target_sector_count) if target_sector_count < SectorIndex::MAX => { + target_sector_count + } + _ => { + // We use this for both count and index, hence index must not reach actual `MAX` + // (consensus doesn't care about this, just farmer implementation detail) + let max_sectors = SectorIndex::MAX - 1; + return Err(SingleDiskFarmError::FarmTooLarge { + allocated_space: target_sector_count * sector_size, + allocated_sectors: target_sector_count, + max_space: max_sectors as u64 * sector_size, + max_sectors, + }); + } + }; + + Ok(Self { + piece_cache_capacity, + plot_file_size, + target_sector_count, + metadata_file_size: RESERVED_PLOT_METADATA + + sector_metadata_size * u64::from(target_sector_count), + }) + } +} + type Handler = Bag, A>; #[derive(Default, Debug)] @@ -1378,72 +1467,15 @@ impl SingleDiskFarm { }; let pieces_in_sector = single_disk_farm_info.pieces_in_sector(); - let sector_size = sector_size(pieces_in_sector); + let sector_size = sector_size(pieces_in_sector) as u64; let sector_metadata_size = SectorMetadataChecksummed::encoded_size(); - let single_sector_overhead = (sector_size + sector_metadata_size) as u64; - // Fixed space usage regardless of plot size - let fixed_space_usage = RESERVED_PLOT_METADATA - + RESERVED_FARM_INFO - + Identity::file_size() as u64 - + KnownPeersManager::file_size(KNOWN_PEERS_CACHE_SIZE) as u64; - // Calculate how many sectors can fit - let target_sector_count = { - let potentially_plottable_space = allocated_space.saturating_sub(fixed_space_usage) - / 100 - * (100 - u64::from(cache_percentage)); - // Do the rounding to make sure we have exactly as much space as fits whole number of - // sectors, account for disk sector size just in case - (potentially_plottable_space - DISK_SECTOR_SIZE as u64) / single_sector_overhead - }; - - if target_sector_count == 0 { - let mut single_plot_with_cache_space = - single_sector_overhead.div_ceil(100 - u64::from(cache_percentage)) * 100; - // Cache must not be empty, ensure it contains at least one element even if - // percentage-wise it will use more space - if single_plot_with_cache_space - single_sector_overhead - < DiskPieceCache::element_size() as u64 - { - single_plot_with_cache_space = - single_sector_overhead + DiskPieceCache::element_size() as u64; - } - - return Err(SingleDiskFarmError::InsufficientAllocatedSpace { - min_space: fixed_space_usage + single_plot_with_cache_space, - allocated_space, - }); - } - let plot_file_size = target_sector_count * sector_size as u64; - // Align plot file size for disk sector size - let plot_file_size = - plot_file_size.div_ceil(DISK_SECTOR_SIZE as u64) * DISK_SECTOR_SIZE as u64; - - // Remaining space will be used for caching purposes - let piece_cache_capacity = if cache_percentage > 0 { - let cache_space = allocated_space - - fixed_space_usage - - plot_file_size - - (sector_metadata_size as u64 * target_sector_count); - (cache_space / u64::from(DiskPieceCache::element_size())) as u32 - } else { - 0 - }; - let target_sector_count = match SectorIndex::try_from(target_sector_count) { - Ok(target_sector_count) if target_sector_count < SectorIndex::MAX => { - target_sector_count - } - _ => { - // We use this for both count and index, hence index must not reach actual `MAX` - // (consensus doesn't care about this, just farmer implementation detail) - let max_sectors = SectorIndex::MAX - 1; - return Err(SingleDiskFarmError::FarmTooLarge { - allocated_space: target_sector_count * sector_size as u64, - allocated_sectors: target_sector_count, - max_space: max_sectors as u64 * sector_size as u64, - max_sectors, - }); - } - }; + let allocated_space_distribution = AllocatedSpaceDistribution::new( + allocated_space, + sector_size, + cache_percentage, + sector_metadata_size as u64, + )?; + let target_sector_count = allocated_space_distribution.target_sector_count; let metadata_file_path = directory.join(Self::METADATA_FILE); #[cfg(not(windows))] @@ -1461,8 +1493,7 @@ impl SingleDiskFarm { let metadata_file = UnbufferedIoFileWindows::open(&metadata_file_path)?; let metadata_size = metadata_file.size()?; - let expected_metadata_size = - RESERVED_PLOT_METADATA + sector_metadata_size as u64 * u64::from(target_sector_count); + let expected_metadata_size = allocated_space_distribution.metadata_file_size; // Align plot file size for disk sector size let expected_metadata_size = expected_metadata_size.div_ceil(DISK_SECTOR_SIZE as u64) * DISK_SECTOR_SIZE as u64; @@ -1563,14 +1594,14 @@ impl SingleDiskFarm { #[cfg(windows)] let plot_file = UnbufferedIoFileWindows::open(&directory.join(Self::PLOT_FILE))?; - if plot_file.size()? != plot_file_size { + if plot_file.size()? != allocated_space_distribution.plot_file_size { // Allocating the whole file (`set_len` below can create a sparse file, which will cause // writes to fail later) plot_file - .preallocate(plot_file_size) + .preallocate(allocated_space_distribution.plot_file_size) .map_err(SingleDiskFarmError::CantPreallocatePlotFile)?; // Truncating file (if necessary) - plot_file.set_len(plot_file_size)?; + plot_file.set_len(allocated_space_distribution.plot_file_size)?; } let plot_file = Arc::new(plot_file); @@ -1591,7 +1622,7 @@ impl SingleDiskFarm { metadata_header, target_sector_count, sectors_metadata, - piece_cache_capacity, + piece_cache_capacity: allocated_space_distribution.piece_cache_capacity, plot_cache, }) } diff --git a/crates/subspace-farmer/src/single_disk_farm/plot_cache.rs b/crates/subspace-farmer/src/single_disk_farm/plot_cache.rs index a8851d7815..5d3d9e33c6 100644 --- a/crates/subspace-farmer/src/single_disk_farm/plot_cache.rs +++ b/crates/subspace-farmer/src/single_disk_farm/plot_cache.rs @@ -88,10 +88,9 @@ impl DiskPlotCache { #[cfg(windows)] file: &Arc, sectors_metadata: &Arc>>, target_sector_count: SectorIndex, - sector_size: usize, + sector_size: u64, ) -> Self { info!("Checking plot cache contents, this can take a while"); - let sector_size = sector_size as u64; let cached_pieces = { let sectors_metadata = sectors_metadata.read_blocking(); let mut element = vec![0; Self::element_size() as usize]; diff --git a/crates/subspace-farmer/src/single_disk_farm/plot_cache/tests.rs b/crates/subspace-farmer/src/single_disk_farm/plot_cache/tests.rs index fdc8b15631..00b81c7868 100644 --- a/crates/subspace-farmer/src/single_disk_farm/plot_cache/tests.rs +++ b/crates/subspace-farmer/src/single_disk_farm/plot_cache/tests.rs @@ -81,7 +81,7 @@ async fn basic() { &file, §ors_metadata, TARGET_SECTOR_COUNT, - FAKE_SECTOR_SIZE, + FAKE_SECTOR_SIZE as u64, ); // Initially empty @@ -111,7 +111,7 @@ async fn basic() { &file, §ors_metadata, TARGET_SECTOR_COUNT, - FAKE_SECTOR_SIZE, + FAKE_SECTOR_SIZE as u64, ); // Successfully stores piece if not all sectors are plotted From 96c296154a483743881d68ee0f5d4543ee0f5032 Mon Sep 17 00:00:00 2001 From: Nazar Mokrynskyi Date: Tue, 6 Aug 2024 02:59:34 +0300 Subject: [PATCH 31/41] Implement effective disk usage estimation for a farm --- .../subspace-farmer/src/single_disk_farm.rs | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/crates/subspace-farmer/src/single_disk_farm.rs b/crates/subspace-farmer/src/single_disk_farm.rs index be0939f7a7..a377b404b5 100644 --- a/crates/subspace-farmer/src/single_disk_farm.rs +++ b/crates/subspace-farmer/src/single_disk_farm.rs @@ -656,6 +656,7 @@ impl ScrubTarget { } struct AllocatedSpaceDistribution { + piece_cache_file_size: u64, piece_cache_capacity: u32, plot_file_size: u64, target_sector_count: u16, @@ -735,6 +736,8 @@ impl AllocatedSpaceDistribution { }; Ok(Self { + piece_cache_file_size: u64::from(piece_cache_capacity) + * u64::from(DiskPieceCache::element_size()), piece_cache_capacity, plot_file_size, target_sector_count, @@ -1645,6 +1648,92 @@ impl SingleDiskFarm { } } + /// Effective on-disk allocation of the files related to the farm (takes some buffer space + /// into consideration). + /// + /// This is a helpful number in case some files were not allocated properly or were removed and + /// do not correspond to allocated space in the farm info accurately. + pub fn effective_disk_usage( + directory: &Path, + cache_percentage: u8, + ) -> Result { + let mut effective_disk_usage; + match SingleDiskFarmInfo::load_from(directory)? { + Some(single_disk_farm_info) => { + let allocated_space_distribution = AllocatedSpaceDistribution::new( + single_disk_farm_info.allocated_space(), + sector_size(single_disk_farm_info.pieces_in_sector()) as u64, + cache_percentage, + SectorMetadataChecksummed::encoded_size() as u64, + )?; + + effective_disk_usage = single_disk_farm_info.allocated_space(); + effective_disk_usage -= Identity::file_size() as u64; + effective_disk_usage -= allocated_space_distribution.metadata_file_size; + effective_disk_usage -= allocated_space_distribution.plot_file_size; + effective_disk_usage -= allocated_space_distribution.piece_cache_file_size; + } + None => { + // No farm info, try to collect actual file sizes is any + effective_disk_usage = 0; + } + }; + + if Identity::open(directory)?.is_some() { + effective_disk_usage += Identity::file_size() as u64; + } + + match OpenOptions::new() + .read(true) + .open(directory.join(Self::METADATA_FILE)) + { + Ok(metadata_file) => { + effective_disk_usage += metadata_file.size()?; + } + Err(error) => { + if error.kind() == io::ErrorKind::NotFound { + // File is not stored on disk + } else { + return Err(error.into()); + } + } + }; + + match OpenOptions::new() + .read(true) + .open(directory.join(Self::PLOT_FILE)) + { + Ok(plot_file) => { + effective_disk_usage += plot_file.size()?; + } + Err(error) => { + if error.kind() == io::ErrorKind::NotFound { + // File is not stored on disk + } else { + return Err(error.into()); + } + } + }; + + match OpenOptions::new() + .read(true) + .open(directory.join(DiskPieceCache::FILE_NAME)) + { + Ok(piece_cache) => { + effective_disk_usage += piece_cache.size()?; + } + Err(error) => { + if error.kind() == io::ErrorKind::NotFound { + // File is not stored on disk + } else { + return Err(error.into()); + } + } + }; + + Ok(effective_disk_usage) + } + /// Read all sectors metadata pub fn read_all_sectors_metadata( directory: &Path, From 693de5ae4cc0057b32f6f68ecc870dabf2e7734d Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 1 Aug 2024 09:54:09 +1000 Subject: [PATCH 32/41] Silence a macro style warning that only occurs in some IDEs This warning has been observed in VS Code + rust-analyzer --- crates/pallet-feeds/src/mock.rs | 4 ++++ crates/subspace-runtime/src/lib.rs | 3 +++ domains/pallets/executive/src/mock.rs | 4 ++++ test/subspace-test-runtime/src/lib.rs | 3 +++ 4 files changed, 14 insertions(+) diff --git a/crates/pallet-feeds/src/mock.rs b/crates/pallet-feeds/src/mock.rs index 224e5e1ff8..28949989f1 100644 --- a/crates/pallet-feeds/src/mock.rs +++ b/crates/pallet-feeds/src/mock.rs @@ -1,3 +1,7 @@ +// Silence a rust-analyzer warning in `construct_runtime!`. This warning isn't present in rustc output. +// TODO: remove when upstream issue is fixed: +#![allow(non_camel_case_types)] + use crate::feed_processor::{FeedObjectMapping, FeedProcessor, FeedProcessor as FeedProcessorT}; use crate::{self as pallet_feeds}; use codec::{Compact, CompactLen, Decode, Encode}; diff --git a/crates/subspace-runtime/src/lib.rs b/crates/subspace-runtime/src/lib.rs index 6c372188e8..6498503757 100644 --- a/crates/subspace-runtime/src/lib.rs +++ b/crates/subspace-runtime/src/lib.rs @@ -18,6 +18,9 @@ #![feature(const_option, const_trait_impl, variant_count)] // `construct_runtime!` does a lot of recursion and requires us to increase the limit to 256. #![recursion_limit = "256"] +// Silence a rust-analyzer warning in `construct_runtime!`. This warning isn't present in rustc output. +// TODO: remove when upstream issue is fixed: +#![allow(non_camel_case_types)] mod domains; mod fees; diff --git a/domains/pallets/executive/src/mock.rs b/domains/pallets/executive/src/mock.rs index f4be3a348b..3c6fbc5108 100644 --- a/domains/pallets/executive/src/mock.rs +++ b/domains/pallets/executive/src/mock.rs @@ -1,3 +1,7 @@ +// Silence a rust-analyzer warning in `construct_runtime!`. This warning isn't present in rustc output. +// TODO: remove when upstream issue is fixed: +#![allow(non_camel_case_types)] + use crate as pallet_executive; use crate::Config; use frame_support::dispatch::DispatchInfo; diff --git a/test/subspace-test-runtime/src/lib.rs b/test/subspace-test-runtime/src/lib.rs index 04b4877af2..0b85dc652c 100644 --- a/test/subspace-test-runtime/src/lib.rs +++ b/test/subspace-test-runtime/src/lib.rs @@ -18,6 +18,9 @@ #![feature(const_option, variant_count)] // `construct_runtime!` does a lot of recursion and requires us to increase the limit to 256. #![recursion_limit = "256"] +// Silence a rust-analyzer warning in `construct_runtime!`. This warning isn't present in rustc output. +// TODO: remove when upstream issue is fixed: +#![allow(non_camel_case_types)] // Make the WASM binary available. #[cfg(feature = "std")] From a64e60e0effd6e18e0a31629371cfbbfbccd79cc Mon Sep 17 00:00:00 2001 From: teor Date: Thu, 1 Aug 2024 09:59:48 +1000 Subject: [PATCH 33/41] Fix spurious trait resolution errors in some IDEs `rust-analyzer` appears to resolve traits in an incorrect order, prioritising imported trait methods over generic trait methods. These errors have been observed in VS Code + rust-analyzer, and are likely a bug in VS Code's config or rust-analyzer itself. --- crates/sc-consensus-subspace/src/archiver.rs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/crates/sc-consensus-subspace/src/archiver.rs b/crates/sc-consensus-subspace/src/archiver.rs index f7fc2f3399..93cbc0342b 100644 --- a/crates/sc-consensus-subspace/src/archiver.rs +++ b/crates/sc-consensus-subspace/src/archiver.rs @@ -655,7 +655,7 @@ where Ok((block, block_object_mappings)) }, ) - .collect::>>() + .collect::, _)>>>() })?; best_archived_block = @@ -740,7 +740,7 @@ fn finalize_block( }); } -/// Crate an archiver task that will listen for importing blocks and archive blocks at `K` depth, +/// Create an archiver task that will listen for importing blocks and archive blocks at `K` depth, /// producing pieces and segment headers (segment headers are then added back to the blockchain as /// `store_segment_header` extrinsic). /// @@ -874,7 +874,10 @@ where // Special sync mode where verified blocks were inserted into blockchain // directly, archiving of this block will naturally happen later continue; - } else if client.hash(importing_block_number - One::one())?.is_none() { + } else if client + .block_hash(importing_block_number - One::one())? + .is_none() + { // We may have imported some block using special sync mode and block we're about // to import is the first one after the gap at which archiver is supposed to be // initialized, but we are only about to import it, so wait for the next block @@ -942,7 +945,7 @@ where let block = client .block( client - .hash(block_number_to_archive)? + .block_hash(block_number_to_archive)? .expect("Older block by number must always exist"), )? .expect("Older block by number must always exist"); @@ -1022,7 +1025,7 @@ where if let Some(block_number_to_finalize) = maybe_block_number_to_finalize { // Block is not guaranteed to be present this deep if we have only synced recent blocks - if let Some(block_hash_to_finalize) = client.hash(block_number_to_finalize)? { + if let Some(block_hash_to_finalize) = client.block_hash(block_number_to_finalize)? { finalize_block( client.as_ref(), telemetry.clone(), From 22fe9f06fed62096bdc1234e0859e04285a97d22 Mon Sep 17 00:00:00 2001 From: Nazar Mokrynskyi Date: Wed, 7 Aug 2024 11:26:51 +0300 Subject: [PATCH 34/41] Remove `CpuCoreSet` regrouping that is no longer needed since plotting is able to take advantage of all CPU cores even with one farm --- crates/subspace-farmer/src/utils.rs | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/crates/subspace-farmer/src/utils.rs b/crates/subspace-farmer/src/utils.rs index 0714995ce0..fdc5c17da5 100644 --- a/crates/subspace-farmer/src/utils.rs +++ b/crates/subspace-farmer/src/utils.rs @@ -200,26 +200,6 @@ impl fmt::Debug for CpuCoreSet { } impl CpuCoreSet { - /// Regroup CPU core sets to contain at most `target_sets` sets, useful when there are many L3 - /// cache groups and not as many farms - pub fn regroup(cpu_core_sets: &[Self], target_sets: usize) -> Vec { - cpu_core_sets - // Chunk CPU core sets - .chunks(cpu_core_sets.len().div_ceil(target_sets)) - .map(|sets| Self { - // Combine CPU cores - cores: sets - .iter() - .flat_map(|set| set.cores.iter()) - .copied() - .collect(), - // Preserve topology object - #[cfg(feature = "numa")] - topology: sets[0].topology.clone(), - }) - .collect() - } - /// Get cpu core numbers in this set pub fn cpu_cores(&self) -> &[usize] { &self.cores From 71976a4eb8c31ae64eacd63365317e7e77f271ee Mon Sep 17 00:00:00 2001 From: Nazar Mokrynskyi Date: Wed, 7 Aug 2024 14:02:15 +0300 Subject: [PATCH 35/41] Improve CPU core truncation code to account for physical CPU cores and L2 cache sizes --- crates/subspace-farmer/src/utils.rs | 69 ++++++++++++++++++++++++++++- 1 file changed, 67 insertions(+), 2 deletions(-) diff --git a/crates/subspace-farmer/src/utils.rs b/crates/subspace-farmer/src/utils.rs index fdc5c17da5..63c6d92610 100644 --- a/crates/subspace-farmer/src/utils.rs +++ b/crates/subspace-farmer/src/utils.rs @@ -207,9 +207,74 @@ impl CpuCoreSet { /// Will truncate list of CPU cores to this number. /// + /// Truncation will take into account L2 and L3 cache topology in order to use half of the + /// actual physical cores and half of each core type in case of heterogeneous CPUs. + /// /// If `cores` is zero, call will do nothing since zero number of cores is not allowed. - pub fn truncate(&mut self, cores: usize) { - self.cores.truncate(cores.max(1)); + pub fn truncate(&mut self, num_cores: usize) { + let num_cores = num_cores.clamp(1, self.cores.len()); + + #[cfg(feature = "numa")] + if let Some(topology) = &self.topology { + use hwlocality::object::attributes::ObjectAttributes; + use hwlocality::object::types::ObjectType; + + let mut grouped_by_l2_cache_size_and_core_count = + std::collections::HashMap::<(usize, usize), Vec>::new(); + topology + .objects_with_type(ObjectType::L2Cache) + .for_each(|object| { + let l2_cache_size = + if let Some(ObjectAttributes::Cache(cache)) = object.attributes() { + cache + .size() + .map(|size| size.get() as usize) + .unwrap_or_default() + } else { + 0 + }; + if let Some(cpuset) = object.complete_cpuset() { + let cpuset = cpuset + .into_iter() + .map(usize::from) + .filter(|core| self.cores.contains(core)) + .collect::>(); + let cpuset_len = cpuset.len(); + + if !cpuset.is_empty() { + grouped_by_l2_cache_size_and_core_count + .entry((l2_cache_size, cpuset_len)) + .or_default() + .extend(cpuset); + } + } + }); + + // Make sure all CPU cores in this set were found + if grouped_by_l2_cache_size_and_core_count + .values() + .flatten() + .count() + == self.cores.len() + { + // Walk through groups of cores for each (L2 cache size + number of cores in set) + // tuple and pull number of CPU cores proportional to the fraction of the cores that + // should be returned according to function argument + self.cores = grouped_by_l2_cache_size_and_core_count + .into_values() + .flat_map(|cores| { + let limit = cores.len() * num_cores / self.cores.len(); + // At least 1 CPU core is needed + cores.into_iter().take(limit.max(1)) + }) + .collect(); + + self.cores.sort(); + + return; + } + } + self.cores.truncate(num_cores); } /// Pin current thread to this NUMA node (not just one CPU core) From dc2e36080b1c051afd9a6c806aae33e0b18ccbb9 Mon Sep 17 00:00:00 2001 From: asamuj Date: Thu, 8 Aug 2024 10:44:52 +0800 Subject: [PATCH 36/41] Pass in signers params to Eth new function --- domains/client/eth-service/src/rpc.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/domains/client/eth-service/src/rpc.rs b/domains/client/eth-service/src/rpc.rs index 6b1d977f00..78caf126a0 100644 --- a/domains/client/eth-service/src/rpc.rs +++ b/domains/client/eth-service/src/rpc.rs @@ -145,9 +145,9 @@ where .. } = full_deps; - let mut signers = Vec::new(); + let mut signers: Vec> = Vec::new(); if enable_dev_signer { - signers.push(Box::new(EthDevSigner::new()) as Box); + signers.push(Box::new(EthDevSigner::new())); } io.merge( @@ -157,7 +157,7 @@ where graph.clone(), converter, sync.clone(), - vec![], + signers, storage_override.clone(), frontier_backend.clone(), is_authority, From 7116bf7ba78d5ed9db2d6231c7b37bd53f5b7d54 Mon Sep 17 00:00:00 2001 From: linning Date: Thu, 8 Aug 2024 22:25:51 +0800 Subject: [PATCH 37/41] Running the message listener for the consensus chain Signed-off-by: linning --- crates/subspace-node/src/commands/run.rs | 82 ++++++++----------- .../subspace-node/src/commands/run/domain.rs | 6 +- 2 files changed, 39 insertions(+), 49 deletions(-) diff --git a/crates/subspace-node/src/commands/run.rs b/crates/subspace-node/src/commands/run.rs index e0d84a9f3d..23d6d6b542 100644 --- a/crates/subspace-node/src/commands/run.rs +++ b/crates/subspace-node/src/commands/run.rs @@ -222,9 +222,11 @@ pub async fn run(run_options: RunOptions) -> Result<(), Error> { let gossip_message_sink = xdm_gossip_worker_builder.gossip_msg_sink(); let (domain_message_sink, domain_message_receiver) = tracing_unbounded("domain_message_channel", 100); + let (consensus_msg_sink, consensus_msg_receiver) = + tracing_unbounded("consensus_message_channel", 100); - // Start relayer for consensus chain - let consensus_msg_receiver = { + // Start XDM related workers for consensus chain + { let span = info_span!("Consensus"); let _enter = span.enter(); let consensus_best_hash = consensus_chain_node.client.info().best_hash; @@ -251,6 +253,36 @@ pub async fn run(run_options: RunOptions) -> Result<(), Error> { ), ); + // Start cross domain message listener for Consensus chain to receive messages from domains in the network + let domain_code_executor: sc_domains::RuntimeExecutor = + sc_service::new_wasm_executor(&domain_configuration.domain_config); + consensus_chain_node + .task_manager + .spawn_essential_handle() + .spawn_essential_blocking( + "consensus-message-listener", + None, + Box::pin( + cross_domain_message_gossip::start_cross_chain_message_listener::< + _, + _, + _, + _, + _, + DomainBlock, + _, + >( + ChainId::Consensus, + consensus_chain_node.client.clone(), + consensus_chain_node.client.clone(), + consensus_chain_node.transaction_pool.clone(), + consensus_chain_node.network_service.clone(), + consensus_msg_receiver, + domain_code_executor.into(), + ), + ), + ); + consensus_chain_node .task_manager .spawn_essential_handle() @@ -272,9 +304,6 @@ pub async fn run(run_options: RunOptions) -> Result<(), Error> { ), ); - let (consensus_msg_sink, consensus_msg_receiver) = - tracing_unbounded("consensus_message_channel", 100); - xdm_gossip_worker_builder.push_chain_sink(ChainId::Consensus, consensus_msg_sink); xdm_gossip_worker_builder.push_chain_sink( ChainId::Domain(domain_configuration.domain_id), @@ -296,16 +325,8 @@ pub async fn run(run_options: RunOptions) -> Result<(), Error> { None, Box::pin(cross_domain_message_gossip_worker.run()), ); - - consensus_msg_receiver }; - let consensus_client = consensus_chain_node.client.clone(); - let consensus_network_service = consensus_chain_node.network_service.clone(); - let consensus_tx_pool = consensus_chain_node.transaction_pool.clone(); - let consensus_task_essential_handler = - consensus_chain_node.task_manager.spawn_essential_handle(); - let domain_start_options = DomainStartOptions { consensus_client: consensus_chain_node.client, consensus_offchain_tx_pool_factory: OffchainTransactionPoolFactory::new( @@ -349,39 +370,8 @@ pub async fn run(run_options: RunOptions) -> Result<(), Error> { domain_start_options, ); - match start_domain.await { - Ok(domain_code_executor) => { - let span = info_span!("Consensus"); - let _enter = span.enter(); - // Start cross domain message listener for Consensus chain to receive messages from domains in the network - consensus_task_essential_handler - .spawn_essential_blocking( - "consensus-message-listener", - None, - Box::pin( - cross_domain_message_gossip::start_cross_chain_message_listener::< - _, - _, - _, - _, - _, - DomainBlock, - _, - >( - ChainId::Consensus, - consensus_client.clone(), - consensus_client.clone(), - consensus_tx_pool, - consensus_network_service, - consensus_msg_receiver, - domain_code_executor - ), - ), - ); - } - Err(err) => { - error!(%err, "Domain starter exited with an error"); - } + if let Err(error) = start_domain.await { + error!(%error, "Domain starter exited with an error"); } }), ); diff --git a/crates/subspace-node/src/commands/run/domain.rs b/crates/subspace-node/src/commands/run/domain.rs index 18137c04dd..a92f7be574 100644 --- a/crates/subspace-node/src/commands/run/domain.rs +++ b/crates/subspace-node/src/commands/run/domain.rs @@ -388,7 +388,7 @@ pub(super) async fn run_domain( bootstrap_result: BootstrapResult, domain_configuration: DomainConfiguration, domain_start_options: DomainStartOptions, -) -> Result, Error> { +) -> Result<(), Error> { let BootstrapResult { domain_instance_data, domain_created_at, @@ -514,7 +514,7 @@ pub(super) async fn run_domain( domain_node.task_manager.future().await?; - Ok(domain_node.code_executor.clone()) + Ok(()) } RuntimeType::AutoId => { let domain_params = domain_service::DomainParams { @@ -553,7 +553,7 @@ pub(super) async fn run_domain( domain_node.task_manager.future().await?; - Ok(domain_node.code_executor.clone()) + Ok(()) } } } From e98d0cca8e725e3a83aec7612014092f6c0c9993 Mon Sep 17 00:00:00 2001 From: asamuj Date: Thu, 8 Aug 2024 22:39:55 +0800 Subject: [PATCH 38/41] Opt code by comment --- domains/client/eth-service/src/rpc.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/domains/client/eth-service/src/rpc.rs b/domains/client/eth-service/src/rpc.rs index 78caf126a0..d6f820c8f4 100644 --- a/domains/client/eth-service/src/rpc.rs +++ b/domains/client/eth-service/src/rpc.rs @@ -145,7 +145,7 @@ where .. } = full_deps; - let mut signers: Vec> = Vec::new(); + let mut signers = Vec::>::new(); if enable_dev_signer { signers.push(Box::new(EthDevSigner::new())); } From 7f9131472f46b83ccbdf6d8d2f5d8497e6763c72 Mon Sep 17 00:00:00 2001 From: Nazar Mokrynskyi Date: Thu, 8 Aug 2024 03:55:23 +0300 Subject: [PATCH 39/41] Remove .cargo/config and `+aes` feature on x86-64 --- .cargo/config.toml | 3 --- .dockerignore | 1 - .github/workflows/snapshot-build.yml | 4 ++-- Dockerfile-bootstrap-node | 1 - Dockerfile-bootstrap-node.aarch64 | 1 - Dockerfile-farmer | 1 - Dockerfile-farmer.aarch64 | 1 - Dockerfile-node | 1 - Dockerfile-node.aarch64 | 1 - Dockerfile-runtime | 1 - crates/subspace-proof-of-time/src/aes.rs | 8 ++++---- 11 files changed, 6 insertions(+), 17 deletions(-) delete mode 100644 .cargo/config.toml diff --git a/.cargo/config.toml b/.cargo/config.toml deleted file mode 100644 index 6f94bc4fba..0000000000 --- a/.cargo/config.toml +++ /dev/null @@ -1,3 +0,0 @@ -[target.'cfg(target_arch = "x86_64")'] -# Require AES-NI on x86-64 by default -rustflags = ["-C", "target-feature=+aes"] diff --git a/.dockerignore b/.dockerignore index 68d5d1fb0c..983f1aefff 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,4 @@ * -!/.cargo !/crates !/domains !/orml diff --git a/.github/workflows/snapshot-build.yml b/.github/workflows/snapshot-build.yml index a70b3fc145..c5f85f468e 100644 --- a/.github/workflows/snapshot-build.yml +++ b/.github/workflows/snapshot-build.yml @@ -99,7 +99,7 @@ jobs: - os: ${{ fromJson(github.repository_owner == 'autonomys' && '["self-hosted", "ubuntu-20.04-x86-64"]' || '"ubuntu-20.04"') }} target: x86_64-unknown-linux-gnu suffix: ubuntu-x86_64-v2-${{ github.ref_name }} - rustflags: "-C target-cpu=x86-64-v2 -C target-feature=+aes" + rustflags: "-C target-cpu=x86-64-v2" - os: ${{ fromJson(github.repository_owner == 'autonomys' && '["self-hosted", "ubuntu-20.04-x86-64"]' || '"ubuntu-20.04"') }} target: aarch64-unknown-linux-gnu suffix: ubuntu-aarch64-${{ github.ref_name }} @@ -114,7 +114,7 @@ jobs: - os: ${{ fromJson(github.repository_owner == 'autonomys' && '["self-hosted", "windows-server-2022-x86-64"]' || '"windows-2022"') }} target: x86_64-pc-windows-msvc suffix: windows-x86_64-v2-${{ github.ref_name }} - rustflags: "-C target-cpu=x86-64-v2 -C target-feature=+aes" + rustflags: "-C target-cpu=x86-64-v2" fail-fast: false runs-on: ${{ matrix.build.os }} env: diff --git a/Dockerfile-bootstrap-node b/Dockerfile-bootstrap-node index b51a9719d5..088ee5af07 100644 --- a/Dockerfile-bootstrap-node +++ b/Dockerfile-bootstrap-node @@ -27,7 +27,6 @@ RUN \ RUN /root/.cargo/bin/rustup target add wasm32-unknown-unknown -COPY .cargo /code/.cargo COPY Cargo.lock /code/Cargo.lock COPY Cargo.toml /code/Cargo.toml COPY rust-toolchain.toml /code/rust-toolchain.toml diff --git a/Dockerfile-bootstrap-node.aarch64 b/Dockerfile-bootstrap-node.aarch64 index 87cde19cd4..9b715e14ab 100644 --- a/Dockerfile-bootstrap-node.aarch64 +++ b/Dockerfile-bootstrap-node.aarch64 @@ -27,7 +27,6 @@ RUN \ RUN /root/.cargo/bin/rustup target add wasm32-unknown-unknown -COPY .cargo /code/.cargo COPY Cargo.lock /code/Cargo.lock COPY Cargo.toml /code/Cargo.toml COPY rust-toolchain.toml /code/rust-toolchain.toml diff --git a/Dockerfile-farmer b/Dockerfile-farmer index e38f4b06dd..6522be9951 100644 --- a/Dockerfile-farmer +++ b/Dockerfile-farmer @@ -27,7 +27,6 @@ RUN \ RUN /root/.cargo/bin/rustup target add wasm32-unknown-unknown -COPY .cargo /code/.cargo COPY Cargo.lock /code/Cargo.lock COPY Cargo.toml /code/Cargo.toml COPY rust-toolchain.toml /code/rust-toolchain.toml diff --git a/Dockerfile-farmer.aarch64 b/Dockerfile-farmer.aarch64 index 76291a3cbd..0fc6d467cb 100644 --- a/Dockerfile-farmer.aarch64 +++ b/Dockerfile-farmer.aarch64 @@ -27,7 +27,6 @@ RUN \ RUN /root/.cargo/bin/rustup target add wasm32-unknown-unknown -COPY .cargo /code/.cargo COPY Cargo.lock /code/Cargo.lock COPY Cargo.toml /code/Cargo.toml COPY rust-toolchain.toml /code/rust-toolchain.toml diff --git a/Dockerfile-node b/Dockerfile-node index cd0fe490f7..46ffd787ee 100644 --- a/Dockerfile-node +++ b/Dockerfile-node @@ -27,7 +27,6 @@ RUN \ RUN /root/.cargo/bin/rustup target add wasm32-unknown-unknown -COPY .cargo /code/.cargo COPY Cargo.lock /code/Cargo.lock COPY Cargo.toml /code/Cargo.toml COPY rust-toolchain.toml /code/rust-toolchain.toml diff --git a/Dockerfile-node.aarch64 b/Dockerfile-node.aarch64 index a5c8d05fda..0b0b6fecf0 100644 --- a/Dockerfile-node.aarch64 +++ b/Dockerfile-node.aarch64 @@ -27,7 +27,6 @@ RUN \ RUN /root/.cargo/bin/rustup target add wasm32-unknown-unknown -COPY .cargo /code/.cargo COPY Cargo.lock /code/Cargo.lock COPY Cargo.toml /code/Cargo.toml COPY rust-toolchain.toml /code/rust-toolchain.toml diff --git a/Dockerfile-runtime b/Dockerfile-runtime index e7ade29e7e..b0afc621fa 100644 --- a/Dockerfile-runtime +++ b/Dockerfile-runtime @@ -27,7 +27,6 @@ RUN \ RUN /root/.cargo/bin/rustup target add wasm32-unknown-unknown -COPY .cargo /code/.cargo COPY Cargo.lock /code/Cargo.lock COPY Cargo.toml /code/Cargo.toml COPY rust-toolchain.toml /code/rust-toolchain.toml diff --git a/crates/subspace-proof-of-time/src/aes.rs b/crates/subspace-proof-of-time/src/aes.rs index 16d92caf19..4713b3b49c 100644 --- a/crates/subspace-proof-of-time/src/aes.rs +++ b/crates/subspace-proof-of-time/src/aes.rs @@ -1,7 +1,7 @@ //! AES related functionality. // TODO: Similarly optimized version for aarch64 -#[cfg(target_arch = "x86_64")] +#[cfg(all(target_arch = "x86_64", target_feature = "aes"))] mod x86_64; #[cfg(not(feature = "std"))] @@ -15,15 +15,15 @@ use subspace_core_primitives::{PotCheckpoints, PotKey, PotOutput, PotSeed}; /// Creates the AES based proof. #[inline(always)] pub(crate) fn create(seed: PotSeed, key: PotKey, checkpoint_iterations: u32) -> PotCheckpoints { - #[cfg(target_arch = "x86_64")] + #[cfg(all(target_arch = "x86_64", target_feature = "aes"))] { unsafe { x86_64::create(seed.as_ref(), key.as_ref(), checkpoint_iterations) } } - #[cfg(not(target_arch = "x86_64"))] + #[cfg(not(all(target_arch = "x86_64", target_feature = "aes")))] create_generic(seed, key, checkpoint_iterations) } -#[cfg(any(not(target_arch = "x86_64"), test))] +#[cfg(any(not(all(target_arch = "x86_64", target_feature = "aes")), test))] #[inline(always)] fn create_generic(seed: PotSeed, key: PotKey, checkpoint_iterations: u32) -> PotCheckpoints { let key = Array::from(*key); From 6a8131b768c227c0fcc576228c03182e1b3ed3f5 Mon Sep 17 00:00:00 2001 From: dastansam Date: Fri, 9 Aug 2024 20:41:57 +0600 Subject: [PATCH 40/41] Remove already executed domains migration --- crates/pallet-domains/src/lib.rs | 1 - crates/pallet-domains/src/migrations.rs | 294 ------------------------ crates/subspace-runtime/src/lib.rs | 13 -- 3 files changed, 308 deletions(-) delete mode 100644 crates/pallet-domains/src/migrations.rs diff --git a/crates/pallet-domains/src/lib.rs b/crates/pallet-domains/src/lib.rs index 4a9d2e8554..b1761eecd2 100644 --- a/crates/pallet-domains/src/lib.rs +++ b/crates/pallet-domains/src/lib.rs @@ -27,7 +27,6 @@ mod tests; pub mod block_tree; mod bundle_storage_fund; pub mod domain_registry; -pub mod migrations; pub mod runtime_registry; mod staking; mod staking_epoch; diff --git a/crates/pallet-domains/src/migrations.rs b/crates/pallet-domains/src/migrations.rs deleted file mode 100644 index 11a90ee95b..0000000000 --- a/crates/pallet-domains/src/migrations.rs +++ /dev/null @@ -1,294 +0,0 @@ -//! Migration module for pallet-domains - -use crate::Config; -use frame_support::traits::UncheckedOnRuntimeUpgrade; -use frame_support::weights::Weight; - -pub struct VersionUncheckedMigrateV0ToV1(sp_std::marker::PhantomData); -impl UncheckedOnRuntimeUpgrade for VersionUncheckedMigrateV0ToV1 { - fn on_runtime_upgrade() -> Weight { - runtime_registry_instance_count_migration::migrate_runtime_registry_storages::() - } -} - -pub(super) mod runtime_registry_instance_count_migration { - use crate::pallet::{ - DomainRegistry, LatestConfirmedDomainExecutionReceipt, RuntimeRegistry as RuntimeRegistryV1, - }; - use crate::{Config, DomainBlockNumberFor, DomainSudoCalls, ExecutionReceiptOf}; - #[cfg(not(feature = "std"))] - use alloc::string::String; - #[cfg(not(feature = "std"))] - use alloc::vec; - use codec::{Codec, Decode, Encode}; - use frame_support::pallet_prelude::{OptionQuery, TypeInfo, Weight}; - use frame_support::{storage_alias, Identity}; - use frame_system::pallet_prelude::BlockNumberFor; - use sp_core::Get; - use sp_domains::storage::RawGenesis; - use sp_domains::{ - BlockFees, DomainId, DomainSudoCall, RuntimeId, RuntimeObject as RuntimeObjectV1, - RuntimeType, Transfers, - }; - use sp_version::RuntimeVersion; - - #[derive(TypeInfo, Debug, Encode, Decode, Clone, PartialEq, Eq)] - pub struct RuntimeObject { - pub runtime_name: String, - pub runtime_type: RuntimeType, - pub runtime_upgrades: u32, - pub hash: Hash, - // The raw genesis storage that contains the runtime code. - // NOTE: don't use this field directly but `into_complete_raw_genesis` instead - pub raw_genesis: RawGenesis, - pub version: RuntimeVersion, - pub created_at: Number, - pub updated_at: Number, - } - - /// Type holding the block details of confirmed domain block. - #[derive(TypeInfo, Encode, Decode, Debug, Clone, PartialEq, Eq)] - pub struct ConfirmedDomainBlock { - /// Block number of the confirmed domain block. - pub block_number: DomainBlockNumber, - /// Block hash of the confirmed domain block. - pub block_hash: DomainHash, - /// Parent block hash of the confirmed domain block. - pub parent_block_receipt_hash: DomainHash, - /// State root of the domain block. - pub state_root: DomainHash, - /// Extrinsic root of the domain block. - pub extrinsics_root: DomainHash, - } - - #[storage_alias] - pub type RuntimeRegistry = StorageMap< - crate::Pallet, - Identity, - RuntimeId, - RuntimeObject, ::Hash>, - OptionQuery, - >; - - #[storage_alias] - pub(super) type LatestConfirmedDomainBlock = StorageMap< - crate::Pallet, - Identity, - DomainId, - ConfirmedDomainBlock, ::DomainHash>, - OptionQuery, - >; - - // Return the number of domain instance that instantiated with the given runtime - fn domain_instance_count(runtime_id: RuntimeId) -> (u32, (u64, u64)) { - let (mut read_count, mut write_count) = (0, 0); - ( - DomainRegistry::::iter() - .filter(|(domain_id, domain_obj)| { - read_count += 1; - - // migrate domain sudo call - DomainSudoCalls::::insert(domain_id, DomainSudoCall { maybe_call: None }); - write_count += 1; - - // migrate domain latest confirmed domain block - read_count += 1; - match LatestConfirmedDomainBlock::::take(domain_id) { - None => {} - Some(confirmed_domain_block) => { - write_count += 1; - LatestConfirmedDomainExecutionReceipt::::insert( - domain_id, - ExecutionReceiptOf:: { - domain_block_number: confirmed_domain_block.block_number, - domain_block_hash: confirmed_domain_block.block_hash, - domain_block_extrinsic_root: confirmed_domain_block - .extrinsics_root, - parent_domain_block_receipt_hash: confirmed_domain_block - .parent_block_receipt_hash, - consensus_block_number: Default::default(), - consensus_block_hash: Default::default(), - inboxed_bundles: vec![], - final_state_root: confirmed_domain_block.state_root, - execution_trace: vec![], - execution_trace_root: Default::default(), - block_fees: BlockFees::default(), - transfers: Transfers::default(), - }, - ) - } - } - domain_obj.domain_config.runtime_id == runtime_id - }) - .count() as u32, - (read_count, write_count), - ) - } - - /// Indexes the currently used operator's signing keys into v2 domains storage. - pub(super) fn migrate_runtime_registry_storages() -> Weight { - let (mut read_count, mut write_count) = (0, 0); - RuntimeRegistry::::drain().for_each(|(runtime_id, runtime_obj)| { - let (instance_count, (domain_read_count, domain_write_count)) = - domain_instance_count::(runtime_id); - RuntimeRegistryV1::::set( - runtime_id, - Some(RuntimeObjectV1 { - runtime_name: runtime_obj.runtime_name, - runtime_type: runtime_obj.runtime_type, - runtime_upgrades: runtime_obj.runtime_upgrades, - instance_count, - hash: runtime_obj.hash, - raw_genesis: runtime_obj.raw_genesis, - version: runtime_obj.version, - created_at: runtime_obj.created_at, - updated_at: runtime_obj.updated_at, - }), - ); - - // domain_read_count + 1 since we read the old runtime registry as well - read_count += domain_read_count + 1; - // 1 write to new registry and 1 for old registry + domain_write_count. - write_count += 2 + domain_write_count; - }); - - T::DbWeight::get().reads_writes(read_count, write_count) - } -} - -#[cfg(test)] -mod tests { - use crate::domain_registry::{do_instantiate_domain, DomainConfig}; - use crate::migrations::runtime_registry_instance_count_migration::{ - ConfirmedDomainBlock, LatestConfirmedDomainBlock, RuntimeObject, RuntimeRegistry, - }; - use crate::pallet::{ - LatestConfirmedDomainExecutionReceipt, RuntimeRegistry as RuntimeRegistryV1, - }; - use crate::tests::{new_test_ext, Balances, Test}; - use crate::{Config, DomainSudoCalls}; - use domain_runtime_primitives::{AccountId20, AccountId20Converter}; - use frame_support::pallet_prelude::Weight; - use frame_support::traits::Currency; - use hex_literal::hex; - use sp_domains::storage::RawGenesis; - use sp_domains::{DomainId, OperatorAllowList, RuntimeObject as RuntimeObjectV1}; - use sp_runtime::traits::Convert; - use sp_version::RuntimeVersion; - use subspace_runtime_primitives::SSC; - - #[test] - fn test_migrate_runtime_registry() { - let mut ext = new_test_ext(); - let domain_config = DomainConfig { - domain_name: "evm-domain".to_owned(), - runtime_id: 0, - max_block_size: 10, - max_block_weight: Weight::from_parts(1, 0), - bundle_slot_probability: (1, 1), - target_bundles_per_block: 1, - operator_allow_list: OperatorAllowList::Anyone, - initial_balances: vec![( - AccountId20Converter::convert(AccountId20::from(hex!( - "f24FF3a9CF04c71Dbc94D0b566f7A27B94566cac" - ))), - 1_000_000 * SSC, - )], - }; - let creator = 1u128; - - let expected_er = ext.execute_with(|| { - // create old registry - RuntimeRegistry::::insert( - domain_config.runtime_id, - RuntimeObject { - runtime_name: "evm".to_owned(), - runtime_type: Default::default(), - runtime_upgrades: 0, - hash: Default::default(), - raw_genesis: RawGenesis::dummy(vec![1, 2, 3, 4]), - version: RuntimeVersion { - spec_name: "test".into(), - spec_version: 1, - impl_version: 1, - transaction_version: 1, - ..Default::default() - }, - created_at: Default::default(), - updated_at: Default::default(), - }, - ); - - // create new registry so that domain instantiation can get through - RuntimeRegistryV1::::insert( - domain_config.runtime_id, - RuntimeObjectV1 { - runtime_name: "evm".to_owned(), - runtime_type: Default::default(), - runtime_upgrades: 0, - instance_count: 0, - hash: Default::default(), - raw_genesis: RawGenesis::dummy(vec![1, 2, 3, 4]), - version: RuntimeVersion { - spec_name: "test".into(), - spec_version: 1, - impl_version: 1, - transaction_version: 1, - ..Default::default() - }, - created_at: Default::default(), - updated_at: Default::default(), - }, - ); - - // Set enough fund to creator - Balances::make_free_balance_be( - &creator, - ::DomainInstantiationDeposit::get() - // for domain total issuance - + 1_000_000 * SSC - + ::ExistentialDeposit::get(), - ); - - do_instantiate_domain::(domain_config.clone(), creator, 0u64).unwrap(); - let er = LatestConfirmedDomainExecutionReceipt::::take(DomainId::new(0)).unwrap(); - LatestConfirmedDomainBlock::::insert( - DomainId::new(0), - ConfirmedDomainBlock { - block_number: er.domain_block_number, - block_hash: er.domain_block_hash, - parent_block_receipt_hash: er.parent_domain_block_receipt_hash, - state_root: er.final_state_root, - extrinsics_root: er.domain_block_extrinsic_root, - }, - ); - er - }); - - ext.commit_all().unwrap(); - - ext.execute_with(|| { - let weights = - crate::migrations::runtime_registry_instance_count_migration::migrate_runtime_registry_storages::(); - assert_eq!( - weights, - ::DbWeight::get().reads_writes(3, 4), - ); - - assert_eq!( - RuntimeRegistryV1::::get(0).unwrap().instance_count, - 1 - ); - - assert!(DomainSudoCalls::::contains_key(DomainId::new(0))); - - assert!(LatestConfirmedDomainBlock::::get(DomainId::new(0)).is_none()); - let er = LatestConfirmedDomainExecutionReceipt::::get(DomainId::new(0)).unwrap(); - assert_eq!(er.domain_block_number, expected_er.domain_block_number); - assert_eq!(er.domain_block_hash, expected_er.domain_block_hash); - assert_eq!(er.domain_block_extrinsic_root, expected_er.domain_block_extrinsic_root); - assert_eq!(er.final_state_root, expected_er.final_state_root); - assert_eq!(er.parent_domain_block_receipt_hash, expected_er.parent_domain_block_receipt_hash); - }); - } -} diff --git a/crates/subspace-runtime/src/lib.rs b/crates/subspace-runtime/src/lib.rs index 921ec5afff..84e01c976f 100644 --- a/crates/subspace-runtime/src/lib.rs +++ b/crates/subspace-runtime/src/lib.rs @@ -43,7 +43,6 @@ use domain_runtime_primitives::{ }; use frame_support::genesis_builder_helper::{build_state, get_preset}; use frame_support::inherent::ProvideInherent; -use frame_support::migrations::VersionedMigration; use frame_support::traits::{ ConstU16, ConstU32, ConstU64, ConstU8, Currency, Everything, Get, VariantCount, }; @@ -843,17 +842,6 @@ pub type SignedExtra = ( pub type UncheckedExtrinsic = generic::UncheckedExtrinsic; -pub type VersionCheckedMigrateDomainsV0ToV1 = VersionedMigration< - 0, - 1, - pallet_domains::migrations::VersionUncheckedMigrateV0ToV1, - pallet_domains::Pallet, - ::DbWeight, ->; - -// TODO: remove once the migrations are done. -pub type Migrations = (VersionCheckedMigrateDomainsV0ToV1,); - /// Executive: handles dispatch to the various modules. pub type Executive = frame_executive::Executive< Runtime, @@ -861,7 +849,6 @@ pub type Executive = frame_executive::Executive< frame_system::ChainContext, Runtime, AllPalletsWithSystem, - Migrations, >; fn extract_segment_headers(ext: &UncheckedExtrinsic) -> Option> { From ff5d032a0d4e1531e66f89b4e99f466c7ad20e56 Mon Sep 17 00:00:00 2001 From: Shamil Gadelshin Date: Mon, 12 Aug 2024 16:19:42 +0400 Subject: [PATCH 41/41] Remove unnecessary code. --- crates/subspace-service/src/mmr/request_handler.rs | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/crates/subspace-service/src/mmr/request_handler.rs b/crates/subspace-service/src/mmr/request_handler.rs index 470382edb4..0d774eac73 100644 --- a/crates/subspace-service/src/mmr/request_handler.rs +++ b/crates/subspace-service/src/mmr/request_handler.rs @@ -28,7 +28,6 @@ use sp_core::offchain::{DbExternalities, OffchainStorage, StorageKind}; use sp_runtime::codec; use sp_runtime::traits::Block as BlockT; use std::collections::BTreeMap; -use std::hash::{Hash, Hasher}; use std::marker::PhantomData; use std::sync::Arc; use std::time::Duration; @@ -86,22 +85,13 @@ pub fn generate_protocol_name>( } /// The key of [`BlockRequestHandler::seen_requests`]. -#[derive(Eq, PartialEq, Clone)] +#[derive(Eq, PartialEq, Clone, Hash)] struct SeenRequestsKey { peer: PeerId, starting_position: u32, } -#[allow(clippy::derived_hash_with_manual_eq)] -impl Hash for SeenRequestsKey { - fn hash(&self, state: &mut H) { - self.peer.hash(state); - self.starting_position.hash(state); - } -} - /// Request MMR data from a peer. -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, Encode, Decode, Debug)] pub struct MmrRequest { /// Starting position for MMR node. @@ -110,7 +100,6 @@ pub struct MmrRequest { pub limit: u32, } -#[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, Encode, Decode, Debug)] pub struct MmrResponse { /// MMR-nodes related to node position