Fixed output batching

libtonode-tests/Cargo.toml

@@ -6,14 +6,13 @@ edition = "2021"
 [features]
 chain_generic_tests = []
 ci = ["zingolib/ci"]
-sync = ["dep:zingo-sync"]
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 [dependencies]
 zingolib = { path = "../zingolib", features = [ "deprecations", "test-elevation" ] }
 zingo-status = { path = "../zingo-status" }
 zingo-netutils = { path = "../zingo-netutils" }
-zingo-sync = { path = "../zingo-sync", optional = true }
+zingo-sync = { path = "../zingo-sync" }
 testvectors = { path = "../testvectors" }
 
 bip0039.workspace = true

libtonode-tests/tests/sync.rs

@@ -1,7 +1,9 @@
+use std::time::Duration;
+
 use tempfile::TempDir;
 use testvectors::seeds::HOSPITAL_MUSEUM_SEED;
 use zingo_netutils::GrpcConnector;
-use zingo_sync::sync::sync;
+use zingo_sync::sync::{self, sync};
 use zingolib::{
     config::{construct_lightwalletd_uri, load_clientconfig, DEFAULT_LIGHTWALLETD_SERVER},
     get_base_address_macro,
@@ -10,7 +12,7 @@ use zingolib::{
     wallet::WalletBase,
 };
 
-#[ignore = "too slow, and flakey"]
+#[ignore = "temporary mainnet test for sync development"]
 #[tokio::test]
 async fn sync_mainnet_test() {
     rustls::crypto::ring::default_provider()
@@ -49,6 +51,55 @@ async fn sync_mainnet_test() {
     dbg!(&wallet.sync_state);
 }
 
+#[ignore = "mainnet test for large chain"]
+#[tokio::test]
+async fn sync_status() {
+    rustls::crypto::ring::default_provider()
+        .install_default()
+        .expect("Ring to work as a default");
+    tracing_subscriber::fmt().init();
+
+    let uri = construct_lightwalletd_uri(Some(DEFAULT_LIGHTWALLETD_SERVER.to_string()));
+    let temp_dir = TempDir::new().unwrap();
+    let temp_path = temp_dir.path().to_path_buf();
+    let config = load_clientconfig(
+        uri.clone(),
+        Some(temp_path),
+        zingolib::config::ChainType::Mainnet,
+        true,
+    )
+    .unwrap();
+    let lightclient = LightClient::create_from_wallet_base_async(
+        WalletBase::from_string(HOSPITAL_MUSEUM_SEED.to_string()),
+        &config,
+        2_750_000,
+        // 2_670_000,
+        true,
+    )
+    .await
+    .unwrap();
+
+    let client = GrpcConnector::new(uri).get_client().await.unwrap();
+
+    let wallet = lightclient.wallet.clone();
+    let sync_handle = tokio::spawn(async move {
+        sync(client, &config.chain, wallet).await.unwrap();
+    });
+
+    let wallet = lightclient.wallet.clone();
+    tokio::spawn(async move {
+        loop {
+            let wallet = wallet.clone();
+            let sync_status = sync::sync_status(wallet).await;
+            dbg!(sync_status);
+            tokio::time::sleep(Duration::from_secs(1)).await;
+        }
+    });
+
+    sync_handle.await.unwrap();
+}
+
+// temporary test for sync development
 #[tokio::test]
 async fn sync_test() {
     tracing_subscriber::fmt().init();

zingo-sync/src/client.rs

@@ -29,7 +29,10 @@ pub enum FetchRequest {
     /// Gets the height of the blockchain from the server.
     ChainTip(oneshot::Sender<BlockId>),
     /// Gets the specified range of compact blocks from the server (end exclusive).
-    CompactBlockRange(oneshot::Sender<Vec<CompactBlock>>, Range<BlockHeight>),
+    CompactBlockRange(
+        oneshot::Sender<tonic::Streaming<CompactBlock>>,
+        Range<BlockHeight>,
+    ),
     /// Gets the tree states for a specified block height.
     TreeState(oneshot::Sender<TreeState>, BlockHeight),
     /// Get a full transaction by txid.
@@ -74,14 +77,14 @@ pub async fn get_chain_height(
 pub async fn get_compact_block_range(
     fetch_request_sender: UnboundedSender<FetchRequest>,
     block_range: Range<BlockHeight>,
-) -> Result<Vec<CompactBlock>, ()> {
+) -> Result<tonic::Streaming<CompactBlock>, ()> {
     let (reply_sender, reply_receiver) = oneshot::channel();
     fetch_request_sender
         .send(FetchRequest::CompactBlockRange(reply_sender, block_range))
         .unwrap();
-    let compact_blocks = reply_receiver.await.unwrap();
+    let block_stream = reply_receiver.await.unwrap();
 
-    Ok(compact_blocks)
+    Ok(block_stream)
 }
 
 /// Gets the stream of shards (subtree roots)
@@ -93,7 +96,7 @@ pub async fn get_subtree_roots(
     start_index: u32,
     shielded_protocol: i32,
     max_entries: u32,
-) -> Result<tonic::Streaming<SubtreeRoot>, ()> {
+) -> Result<Vec<SubtreeRoot>, ()> {
     let (reply_sender, reply_receiver) = oneshot::channel();
     fetch_request_sender
         .send(FetchRequest::GetSubtreeRoots(
@@ -103,8 +106,13 @@ pub async fn get_subtree_roots(
             max_entries,
         ))
         .unwrap();
-    let shards = reply_receiver.await.unwrap();
-    Ok(shards)
+    let mut subtree_root_stream = reply_receiver.await.unwrap();
+    let mut subtree_roots = Vec::new();
+    while let Some(subtree_root) = subtree_root_stream.message().await.unwrap() {
+        subtree_roots.push(subtree_root);
+    }
+
+    Ok(subtree_roots)
 }
 
 /// Gets the frontiers for a specified block height.
@@ -188,7 +196,7 @@ pub async fn get_transparent_address_transactions(
 pub async fn get_mempool_transaction_stream(
     client: &mut CompactTxStreamerClient<zingo_netutils::UnderlyingService>,
 ) -> Result<tonic::Streaming<RawTransaction>, ()> {
-    tracing::info!("Fetching mempool stream");
+    tracing::debug!("Fetching mempool stream");
     let mempool_stream = fetch::get_mempool_stream(client).await.unwrap();
 
     Ok(mempool_stream)

zingo-sync/src/client/fetch.rs

@@ -103,17 +103,17 @@ async fn fetch_from_server(
 ) -> Result<(), ()> {
     match fetch_request {
         FetchRequest::ChainTip(sender) => {
-            tracing::info!("Fetching chain tip.");
+            tracing::debug!("Fetching chain tip.");
             let block_id = get_latest_block(client).await.unwrap();
             sender.send(block_id).unwrap();
         }
         FetchRequest::CompactBlockRange(sender, block_range) => {
-            tracing::info!("Fetching compact blocks. {:?}", &block_range);
-            let compact_blocks = get_block_range(client, block_range).await.unwrap();
-            sender.send(compact_blocks).unwrap();
+            tracing::debug!("Fetching compact blocks. {:?}", &block_range);
+            let block_stream = get_block_range(client, block_range).await.unwrap();
+            sender.send(block_stream).unwrap();
         }
         FetchRequest::GetSubtreeRoots(sender, start_index, shielded_protocol, max_entries) => {
-            tracing::info!(
+            tracing::debug!(
                 "Fetching subtree roots. start index: {}. shielded protocol: {}",
                 start_index,
                 shielded_protocol
@@ -124,19 +124,19 @@ async fn fetch_from_server(
             sender.send(shards).unwrap();
         }
         FetchRequest::TreeState(sender, block_height) => {
-            tracing::info!("Fetching tree state. {:?}", &block_height);
+            tracing::debug!("Fetching tree state. {:?}", &block_height);
             let tree_state = get_tree_state(client, block_height).await.unwrap();
             sender.send(tree_state).unwrap();
         }
         FetchRequest::Transaction(sender, txid) => {
-            tracing::info!("Fetching transaction. {:?}", txid);
+            tracing::debug!("Fetching transaction. {:?}", txid);
             let transaction = get_transaction(client, consensus_parameters, txid)
                 .await
                 .unwrap();
             sender.send(transaction).unwrap();
         }
         FetchRequest::UtxoMetadata(sender, (addresses, start_height)) => {
-            tracing::info!(
+            tracing::debug!(
                 "Fetching unspent transparent output metadata from {:?} for addresses:\n{:?}",
                 &start_height,
                 &addresses
@@ -147,7 +147,7 @@ async fn fetch_from_server(
             sender.send(utxo_metadata).unwrap();
         }
         FetchRequest::TransparentAddressTxs(sender, (address, block_range)) => {
-            tracing::info!(
+            tracing::debug!(
                 "Fetching raw transactions in block range {:?} for address {:?}",
                 &block_range,
                 &address
@@ -169,13 +169,11 @@ async fn get_latest_block(
 
     Ok(client.get_latest_block(request).await.unwrap().into_inner())
 }
+
 async fn get_block_range(
     client: &mut CompactTxStreamerClient<zingo_netutils::UnderlyingService>,
     block_range: Range<BlockHeight>,
-) -> Result<Vec<CompactBlock>, ()> {
-    let mut compact_blocks: Vec<CompactBlock> =
-        Vec::with_capacity(u64::from(block_range.end - block_range.start) as usize);
-
+) -> Result<tonic::Streaming<CompactBlock>, ()> {
     let request = tonic::Request::new(BlockRange {
         start: Some(BlockId {
             height: u64::from(block_range.start),
@@ -186,13 +184,8 @@ async fn get_block_range(
             hash: vec![],
         }),
     });
-    let mut block_stream = client.get_block_range(request).await.unwrap().into_inner();
-
-    while let Some(compact_block) = block_stream.message().await.unwrap() {
-        compact_blocks.push(compact_block);
-    }
 
-    Ok(compact_blocks)
+    Ok(client.get_block_range(request).await.unwrap().into_inner())
 }
 
 async fn get_subtree_roots(
@@ -206,12 +199,14 @@ async fn get_subtree_roots(
         shielded_protocol,
         max_entries,
     };
+
     Ok(client
         .get_subtree_roots(request)
         .await
         .unwrap()
         .into_inner())
 }
+
 async fn get_tree_state(
     client: &mut CompactTxStreamerClient<zingo_netutils::UnderlyingService>,
     block_height: BlockHeight,