Musig2 metrics (#2930)

* multithreading

* fix deadlock between workers

* change timeout to 30s

* remove some log

* remove some log

* optimize ceremony data sharing logic

* remove code for debugging

* fix fmt

* fix clippy

* change module name

* clean up code

* restore test_full_flow_with_3_ceremonies

* remove redundant match

* increase TCSNum to 64

* reorganize code structure

* fix test

* fix check_signBitcoin

* add SignBitcoinPayload to InitCeremony

* read threads count from cli (#2912)

* add CeremonyRoundCall

* change CreateSignTask to Request

* fix passing cli params

* musig2 ceremony metrics

* fmt

---------

Co-authored-by: Jayanring <[email protected]>
Co-authored-by: Jayanring <[email protected]>

bitacross-worker/bitacross/core/bc-task-processor/Cargo.toml

@@ -28,13 +28,15 @@ log = { version = "0.4", default-features = false }
 frame-support = { default-features = false, git = "https://github.com/paritytech/substrate.git", branch = "polkadot-v0.9.42" }
 
 ita-stf = { path = "../../../app-libs/stf", default-features = false }
+itp-enclave-metrics = { path = "../../../core-primitives/enclave-metrics", default-features = false }
 itp-ocall-api = { path = "../../../core-primitives/ocall-api", default-features = false }
 itp-sgx-crypto = { path = "../../../core-primitives/sgx/crypto", default-features = false }
 itp-sgx-externalities = { path = "../../../core-primitives/substrate-sgx/externalities", default-features = false }
 itp-stf-executor = { path = "../../../core-primitives/stf-executor", default-features = false }
 itp-stf-state-handler = { path = "../../../core-primitives/stf-state-handler", default-features = false }
 itp-utils = { path = "../../../core-primitives/utils", default-features = false }
 
+
 # litentry primities
 bc-enclave-registry = { path = "../bc-enclave-registry", default-features = false }
 bc-musig2-ceremony = { path = "../bc-musig2-ceremony", default-features = false }
@@ -64,6 +66,7 @@ sgx = [
     "lc-direct-call/sgx",
     "litentry-primitives/sgx",
     "ita-stf/sgx",
+    "itp-enclave-metrics/sgx",
     "itp-sgx-crypto/sgx",
     "itp-sgx-externalities/sgx",
     "itp-stf-executor/sgx",
@@ -86,6 +89,7 @@ std = [
     "lc-direct-call/std",
     "litentry-primitives/std",
     "ita-stf/std",
+    "itp-enclave-metrics/std",
     "itp-ocall-api/std",
     "itp-sgx-crypto/std",
     "itp-sgx-externalities/std",

bitacross-worker/bitacross/core/bc-task-processor/src/lib.rs

@@ -63,6 +63,7 @@ use frame_support::{ensure, sp_runtime::app_crypto::sp_core::blake2_256};
 use ita_stf::TrustedCallSigned;
 use itc_direct_rpc_client::{DirectRpcClient, DirectRpcClientFactory, RpcClientFactory};
 use itc_direct_rpc_server::SendRpcResponse;
+use itp_enclave_metrics::EnclaveMetric;
 use itp_ocall_api::{EnclaveAttestationOCallApi, EnclaveMetricsOCallApi, EnclaveOnChainOCallApi};
 use itp_sgx_crypto::{
 	ecdsa::Pair as EcdsaPair,
@@ -224,9 +225,11 @@ pub fn run_bit_across_handler_runner<SKR, SIGNINGAK, EKR, BKR, S, H, O, RRL, ERL
 	let ceremony_command_tmp = context.ceremony_command_tmp.clone();
 	let responder = context.responder.clone();
 	let time_to_live = 30u64;
+	let cloned_ocall_api = context.ocall_api.clone();
 	std::thread::spawn(move || loop {
 		std::thread::sleep(Duration::from_secs(3));
 		let now = get_current_timestamp();
+		let mut timed_out_count: u8 = 0;
 		{
 			let mut ceremony_registry_write = ceremony_registry.write().unwrap();
 			ceremony_registry_write.retain(|_, (ceremony, create_time)| {
@@ -247,6 +250,7 @@ pub fn run_bit_across_handler_runner<SKR, SIGNINGAK, EKR, BKR, S, H, O, RRL, ERL
 					) {
 						error!("Could not send response to {:?}, reason: {:?}", &hash, e);
 					}
+					timed_out_count += 1;
 				}
 				if_retain
 			});
@@ -255,6 +259,10 @@ pub fn run_bit_across_handler_runner<SKR, SIGNINGAK, EKR, BKR, S, H, O, RRL, ERL
 			let mut command_tmp_write = ceremony_command_tmp.write().unwrap();
 			command_tmp_write.retain(|_, &mut (_, create_time)| now - create_time < time_to_live);
 		}
+		if timed_out_count > 0 {
+			let _ = cloned_ocall_api
+				.update_metric(EnclaveMetric::Musig2CeremonyTimedout(timed_out_count));
+		}
 	});
 
 	let bit_across_task_receiver = init_bit_across_task_sender_storage();
@@ -369,6 +377,24 @@ fn handle_ceremony_command<SKR, SIGNINGAK, EKR, BKR, S, H, O, RRL, ERL, SRL, Res
 		let event = process_command(context.clone(), ceremony_id.clone(), command);
 
 		if let Some(event) = event {
+			// update metrics
+			match event {
+				CeremonyEvent::FirstRoundStarted(_, _, _) => {
+					let _ = context.ocall_api.update_metric(EnclaveMetric::Musig2CeremonyStarted);
+				},
+				CeremonyEvent::CeremonyError(_, _, _) => {
+					let _ = context.ocall_api.update_metric(EnclaveMetric::Musig2CeremonyFailed);
+				},
+				CeremonyEvent::CeremonyEnded(_, _, _, _) => {
+					let ceremony_start_time =
+						context.ceremony_registry.read().unwrap().get(&ceremony_id).unwrap().1;
+					let _ = context.ocall_api.update_metric(EnclaveMetric::Musig2CeremonyDuration(
+						Duration::from_millis(get_current_timestamp() - ceremony_start_time),
+					));
+				},
+				_ => {},
+			}
+
 			match event {
 				CeremonyEvent::FirstRoundStarted(_, _, _)
 				| CeremonyEvent::SecondRoundStarted(_, _, _) => {

bitacross-worker/core-primitives/enclave-metrics/src/lib.rs

@@ -26,15 +26,12 @@ extern crate sgx_tstd as std;
 
 use codec::{Decode, Encode};
 use core::time::Duration;
-use std::string::String;
 
 #[derive(Encode, Decode, Debug)]
 pub enum EnclaveMetric {
-	SetSidechainBlockHeight(u64),
-	TopPoolSizeSet(u64),
-	TopPoolSizeIncrement,
-	TopPoolSizeDecrement,
-	SuccessfulTrustedOperationIncrement(String),
-	FailedTrustedOperationIncrement(String),
 	ParentchainBlockImportTime(Duration),
+	Musig2CeremonyStarted,
+	Musig2CeremonyFailed,
+	Musig2CeremonyTimedout(u8),
+	Musig2CeremonyDuration(Duration),
 }

bitacross-worker/core-primitives/top-pool-author/Cargo.toml

@@ -9,7 +9,6 @@ edition = "2021"
 sgx_tstd = { branch = "master", git = "https://github.com/apache/teaclave-sgx-sdk.git", optional = true }
 
 # local dependencies
-itp-enclave-metrics = { path = "../enclave-metrics", default-features = false }
 itp-ocall-api = { path = "../ocall-api", default-features = false }
 itp-sgx-crypto = { path = "../sgx/crypto", default-features = false }
 itp-stf-primitives = { path = "../stf-primitives", default-features = false }
@@ -50,7 +49,6 @@ sp-keyring = { git = "https://github.com/paritytech/substrate.git", branch = "po
 default = ["std"]
 std = [
     "itp-sgx-crypto/std",
-    "itp-enclave-metrics/std",
     "itp-ocall-api/std",
     "itp-stf-state-handler/std",
     "itp-top-pool/std",
@@ -64,7 +62,6 @@ std = [
 sgx = [
     "sgx_tstd",
     "jsonrpc-core_sgx",
-    "itp-enclave-metrics/sgx",
     "itp-sgx-crypto/sgx",
     "itp-stf-state-handler/sgx",
     "itp-top-pool/sgx",

bitacross-worker/core-primitives/top-pool-author/src/author.rs

@@ -26,8 +26,6 @@ use crate::{
 	traits::{AuthorApi, OnBlockImported},
 };
 use codec::{Decode, Encode};
-use itp_enclave_metrics::EnclaveMetric;
-use itp_ocall_api::EnclaveMetricsOCallApi;
 use itp_sgx_crypto::{key_repository::AccessKey, ShieldingCryptoDecrypt};
 use itp_stf_primitives::{
 	traits::{PoolTransactionValidation, TrustedCallVerification},
@@ -66,7 +64,7 @@ pub type RequestIdWithParamsAndMethod = Option<(Hash, Vec<String>)>;
 /// Authoring API for RPC calls
 ///
 ///
-pub struct Author<TopPool, TopFilter, StateFacade, ShieldingKeyRepository, OCallApi, TCS, G>
+pub struct Author<TopPool, TopFilter, StateFacade, ShieldingKeyRepository, TCS, G>
 where
 	TopPool: TrustedOperationPool<StfTrustedOperation<TCS, G>> + Sync + Send + 'static,
 	TopFilter: Filter<Value = StfTrustedOperation<TCS, G>>,
@@ -80,18 +78,16 @@ where
 	top_filter: TopFilter,
 	state_facade: Arc<StateFacade>,
 	shielding_key_repo: Arc<ShieldingKeyRepository>,
-	ocall_api: Arc<OCallApi>,
 }
 
-impl<TopPool, TopFilter, StateFacade, ShieldingKeyRepository, OCallApi, TCS, G>
-	Author<TopPool, TopFilter, StateFacade, ShieldingKeyRepository, OCallApi, TCS, G>
+impl<TopPool, TopFilter, StateFacade, ShieldingKeyRepository, TCS, G>
+	Author<TopPool, TopFilter, StateFacade, ShieldingKeyRepository, TCS, G>
 where
 	TopPool: TrustedOperationPool<StfTrustedOperation<TCS, G>> + Sync + Send + 'static,
 	TopFilter: Filter<Value = StfTrustedOperation<TCS, G>>,
 	StateFacade: QueryShardState,
 	ShieldingKeyRepository: AccessKey,
 	<ShieldingKeyRepository as AccessKey>::KeyType: ShieldingCryptoDecrypt + 'static,
-	OCallApi: EnclaveMetricsOCallApi + Send + Sync + 'static,
 	TCS: PartialEq + Encode + Clone + Debug + Send + Sync,
 	G: PartialEq + Encode + Clone + PoolTransactionValidation + Debug + Send + Sync,
 {
@@ -101,9 +97,8 @@ where
 		top_filter: TopFilter,
 		state_facade: Arc<StateFacade>,
 		encryption_key: Arc<ShieldingKeyRepository>,
-		ocall_api: Arc<OCallApi>,
 	) -> Self {
-		Author { top_pool, top_filter, state_facade, shielding_key_repo: encryption_key, ocall_api }
+		Author { top_pool, top_filter, state_facade, shielding_key_repo: encryption_key }
 	}
 }
 
@@ -112,15 +107,14 @@ enum TopSubmissionMode {
 	SubmitWatch,
 }
 
-impl<TopPool, TopFilter, StateFacade, ShieldingKeyRepository, OCallApi, TCS, G>
-	Author<TopPool, TopFilter, StateFacade, ShieldingKeyRepository, OCallApi, TCS, G>
+impl<TopPool, TopFilter, StateFacade, ShieldingKeyRepository, TCS, G>
+	Author<TopPool, TopFilter, StateFacade, ShieldingKeyRepository, TCS, G>
 where
 	TopPool: TrustedOperationPool<StfTrustedOperation<TCS, G>> + Sync + Send + 'static,
 	TopFilter: Filter<Value = StfTrustedOperation<TCS, G>>,
 	StateFacade: QueryShardState,
 	ShieldingKeyRepository: AccessKey,
 	<ShieldingKeyRepository as AccessKey>::KeyType: ShieldingCryptoDecrypt + 'static,
-	OCallApi: EnclaveMetricsOCallApi + Send + Sync + 'static,
 	TCS: PartialEq
 		+ Encode
 		+ Decode
@@ -185,11 +179,6 @@ where
 		// dummy block hash
 		let best_block_hash = Default::default();
 
-		// Update metric
-		if let Err(e) = self.ocall_api.update_metric(EnclaveMetric::TopPoolSizeIncrement) {
-			warn!("Failed to update metric for top pool size: {:?}", e);
-		}
-
 		if let Some(trusted_call_signed) = trusted_operation.to_call() {
 			debug!(
 				"Submitting trusted call to TOP pool: {:?}, TOP hash: {:?}",
@@ -251,11 +240,6 @@ where
 
 		debug!("removing {:?} from top pool", hash);
 
-		// Update metric
-		if let Err(e) = self.ocall_api.update_metric(EnclaveMetric::TopPoolSizeDecrement) {
-			warn!("Failed to update metric for top pool size: {:?}", e);
-		}
-
 		let removed_op_hash = self
 			.top_pool
 			.remove_invalid(&[hash], shard, inblock)
@@ -284,16 +268,15 @@ where
 	.into()
 }
 
-impl<TopPool, TopFilter, StateFacade, ShieldingKeyRepository, OCallApi, TCS, G>
+impl<TopPool, TopFilter, StateFacade, ShieldingKeyRepository, TCS, G>
 	AuthorApi<TxHash, BlockHash, TCS, G>
-	for Author<TopPool, TopFilter, StateFacade, ShieldingKeyRepository, OCallApi, TCS, G>
+	for Author<TopPool, TopFilter, StateFacade, ShieldingKeyRepository, TCS, G>
 where
 	TopPool: TrustedOperationPool<StfTrustedOperation<TCS, G>> + Sync + Send + 'static,
 	TopFilter: Filter<Value = StfTrustedOperation<TCS, G>>,
 	StateFacade: QueryShardState,
 	ShieldingKeyRepository: AccessKey,
 	<ShieldingKeyRepository as AccessKey>::KeyType: ShieldingCryptoDecrypt + 'static,
-	OCallApi: EnclaveMetricsOCallApi + Send + Sync + 'static,
 	G: PartialEq
 		+ Encode
 		+ Decode
@@ -407,15 +390,14 @@ where
 	}
 }
 
-impl<TopPool, TopFilter, StateFacade, ShieldingKeyRepository, OCallApi, TCS, G> OnBlockImported
-	for Author<TopPool, TopFilter, StateFacade, ShieldingKeyRepository, OCallApi, TCS, G>
+impl<TopPool, TopFilter, StateFacade, ShieldingKeyRepository, TCS, G> OnBlockImported
+	for Author<TopPool, TopFilter, StateFacade, ShieldingKeyRepository, TCS, G>
 where
 	TopPool: TrustedOperationPool<StfTrustedOperation<TCS, G>> + Sync + Send + 'static,
 	TopFilter: Filter<Value = StfTrustedOperation<TCS, G>>,
 	StateFacade: QueryShardState,
 	ShieldingKeyRepository: AccessKey,
 	<ShieldingKeyRepository as AccessKey>::KeyType: ShieldingCryptoDecrypt + 'static,
-	OCallApi: EnclaveMetricsOCallApi + Send + Sync + 'static,
 	G: PartialEq + Encode + Clone + PoolTransactionValidation + Debug + Send + Sync,
 	TCS: PartialEq + Encode + Clone + Debug + Send + Sync,
 {

bitacross-worker/core-primitives/top-pool-author/src/author_tests.rs

@@ -28,7 +28,6 @@ use itp_sgx_crypto::{mocks::KeyRepositoryMock, ShieldingCryptoDecrypt, Shielding
 use itp_stf_state_handler::handle_state::HandleState;
 use itp_test::mock::{
 	handle_state_mock::HandleStateMock,
-	metrics_ocall_mock::MetricsOCallMock,
 	shielding_crypto_mock::ShieldingCryptoMock,
 	stf_mock::{
 		mock_top_direct_trusted_call_signed, mock_top_indirect_trusted_call_signed,
@@ -45,7 +44,6 @@ type TestAuthor<Filter> = Author<
 	Filter,
 	HandleStateMock,
 	KeyRepositoryMock<ShieldingCryptoMock>,
-	MetricsOCallMock,
 	TrustedCallSignedMock,
 	GetterMock,
 >;
@@ -134,16 +132,9 @@ fn create_author_with_filter<F: Filter<Value = TrustedOperationMock>>(
 	let encryption_key = ShieldingCryptoMock::default();
 	let shielding_key_repo =
 		Arc::new(KeyRepositoryMock::<ShieldingCryptoMock>::new(encryption_key.clone()));
-	let ocall_mock = Arc::new(MetricsOCallMock::default());
 
 	(
-		Author::new(
-			top_pool.clone(),
-			filter,
-			Arc::new(state_facade),
-			shielding_key_repo,
-			ocall_mock,
-		),
+		Author::new(top_pool.clone(), filter, Arc::new(state_facade), shielding_key_repo),
 		top_pool,
 		encryption_key,
 	)

bitacross-worker/enclave-runtime/src/initialization/global_components.rs

@@ -306,7 +306,6 @@ pub type EnclaveTopPoolAuthor = Author<
 	AuthorTopFilter<EnclaveTrustedCallSigned, EnclaveGetter>,
 	EnclaveStateHandler,
 	EnclaveShieldingKeyRepository,
-	EnclaveOCallApi,
 	EnclaveTrustedCallSigned,
 	EnclaveGetter,
 >;

bitacross-worker/enclave-runtime/src/initialization/mod.rs

@@ -24,7 +24,7 @@ use crate::{
 	get_node_metadata_repository_from_integritee_solo_or_parachain,
 	get_validator_accessor_from_integritee_solo_or_parachain,
 	initialization::global_components::{
-		EnclaveGetterExecutor, EnclaveLightClientSeal, EnclaveOCallApi, EnclaveRpcResponder,
+		EnclaveGetterExecutor, EnclaveLightClientSeal, EnclaveRpcResponder,
 		EnclaveShieldingKeyRepository, EnclaveSidechainApi, EnclaveStateFileIo,
 		EnclaveStateHandler, EnclaveStateInitializer, EnclaveStateObserver,
 		EnclaveStateSnapshotRepository, EnclaveStfEnclaveSigner, EnclaveTopPool,
@@ -211,7 +211,6 @@ pub(crate) fn init_enclave(
 	let top_pool_author = create_top_pool_author(
 		rpc_responder.clone(),
 		state_handler,
-		ocall_api.clone(),
 		shielding_key_repository.clone(),
 	);
 	GLOBAL_TOP_POOL_AUTHOR_COMPONENT.initialize(top_pool_author.clone());
@@ -470,7 +469,6 @@ pub(crate) fn migrate_shard(new_shard: ShardIdentifier) -> EnclaveResult<()> {
 pub fn create_top_pool_author(
 	rpc_responder: Arc<EnclaveRpcResponder>,
 	state_handler: Arc<EnclaveStateHandler>,
-	ocall_api: Arc<EnclaveOCallApi>,
 	shielding_key_repository: Arc<EnclaveShieldingKeyRepository>,
 ) -> Arc<EnclaveTopPoolAuthor> {
 	let side_chain_api = Arc::new(EnclaveSidechainApi::new());
@@ -482,6 +480,5 @@ pub fn create_top_pool_author(
 		AuthorTopFilter::<TrustedCallSigned, Getter>::new(),
 		state_handler,
 		shielding_key_repository,
-		ocall_api,
 	))
 }