Skip to content

Commit

Permalink
Merge pull request #1323 from Oscar-Pepper/zingo_sync_note_commitment…
Browse files Browse the repository at this point in the history 
…s_and_retention

Zingo sync note commitments and retention
  • Loading branch information
@fluidvanadium
fluidvanadium authored Aug 12, 2024
2 parents a4ab4a6 + b73c220 commit a23564e
Show file tree
Hide file tree
Showing 3 changed files with 162 additions and 84 deletions.
2 changes: 1 addition & 1 deletion Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ zip32 = "0.1"
clap = "4.4"
tempdir = "0.3"
portpicker = "0.1"
incrementalmerkletree = { version = "0.5" }
incrementalmerkletree = { version = "0.5.1" }
futures = "0.3.15"
shardtree = "0.3"
build_utils = { path = "./build_utils" }
Expand Down
236 changes: 155 additions & 81 deletions zingo-sync/src/scan.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,13 @@ use std::{
collections::{HashMap, HashSet},
};

use incrementalmerkletree::Position;
use orchard::keys::Scope;
use incrementalmerkletree::{Position, Retention};
use orchard::{keys::Scope, note_encryption::CompactAction, tree::MerkleHashOrchard};
use sapling_crypto::{note_encryption::CompactOutputDescription, Node};
use tokio::sync::mpsc::UnboundedSender;
use zcash_client_backend::{
data_api::scanning::ScanRange,
proto::compact_formats::{CompactBlock, CompactTx},
proto::compact_formats::{CompactBlock, CompactOrchardAction, CompactSaplingOutput},
scanning::ScanningKeys,
};
use zcash_note_encryption::Domain;
Expand All @@ -24,7 +25,7 @@ use crate::{
primitives::{OutputId, WalletCompactBlock},
};

use self::runners::BatchRunners;
use self::runners::{BatchRunners, DecryptedOutput};

pub(crate) mod runners;

Expand Down Expand Up @@ -141,9 +142,12 @@ where

check_continuity(&compact_blocks, initial_scan_data.previous_block).unwrap();

let (incoming_sapling_outputs, incoming_orchard_outputs) =
trial_decrypt(parameters, scanning_keys, &compact_blocks).unwrap();
let mut runners = trial_decrypt(parameters, scanning_keys, &compact_blocks).unwrap();

let mut relevent_txids: HashSet<TxId> = HashSet::new();
let mut sapling_leaves_and_retentions: Vec<(Node, Retention<BlockHeight>)> = Vec::new();
let mut orchard_leaves_and_retentions: Vec<(MerkleHashOrchard, Retention<BlockHeight>)> =
Vec::new();
let mut sapling_nullifiers_and_positions: HashMap<
OutputId,
(sapling_crypto::Nullifier, Position),
Expand All @@ -154,20 +158,54 @@ where
> = HashMap::new();
let mut sapling_tree_size = initial_scan_data.sapling_initial_tree_size;
let mut orchard_tree_size = initial_scan_data.orchard_initial_tree_size;
for block in compact_blocks {
// let zip212_enforcement = zip212_enforcement(parameters, block.height());
for transaction in block.vtx {
// TODO: add note commitment and retentions
for block in &compact_blocks {
let mut transactions = block.vtx.iter().peekable();
while let Some(transaction) = transactions.next() {
// collect trial decryption results by transaction
let incoming_sapling_outputs = runners
.sapling
.collect_results(block.hash(), transaction.txid());
let incoming_orchard_outputs = runners
.orchard
.collect_results(block.hash(), transaction.txid());

// gather the txids of all transactions relevent to the wallet
// the edge case of transactions that this capability created but did not receive change
// or create outgoing data is handled when the nullifiers are added and linked
incoming_sapling_outputs.iter().for_each(|(output_id, _)| {
relevent_txids.insert(output_id.txid());
});
incoming_orchard_outputs.iter().for_each(|(output_id, _)| {
relevent_txids.insert(output_id.txid());
});
// TODO: add outgoing outputs to relevent txids

sapling_leaves_and_retentions.extend(
calculate_sapling_leaves_and_retentions(
&transaction.outputs,
block.height(),
transactions.peek().is_none(),
&incoming_sapling_outputs,
)
.unwrap(),
);
orchard_leaves_and_retentions.extend(
calculate_orchard_leaves_and_retentions(
&transaction.actions,
block.height(),
transactions.peek().is_none(),
&incoming_orchard_outputs,
)
.unwrap(),
);

calculate_nullifiers_and_positions(
&transaction,
sapling_tree_size,
scanning_keys.sapling(),
&incoming_sapling_outputs,
&mut sapling_nullifiers_and_positions,
);
calculate_nullifiers_and_positions(
&transaction,
orchard_tree_size,
scanning_keys.orchard(),
&incoming_orchard_outputs,
Expand All @@ -179,84 +217,30 @@ where
orchard_tree_size += u32::try_from(transaction.actions.len())
.expect("should not be more than 2^32 outputs in a transaction");
}
// TODO: check tree size matches chain_metadata if available
}
// TODO: map nullifiers and write compact blocks

// gather the IDs of all transactions relevent to the wallet
// the edge case of transactions that this capability created but did not receive change
// or create outgoing data is handled when the nullifiers are added and linked
let mut relevent_txids: HashSet<TxId> = HashSet::new();
incoming_sapling_outputs
.iter()
.for_each(|(output_id, _, _)| {
relevent_txids.insert(output_id.txid());
});
incoming_orchard_outputs
.iter()
.for_each(|(output_id, _, _)| {
relevent_txids.insert(output_id.txid());
});

Ok(())
}

// TODO: reduce type complexity once trial decryption is finalised
#[allow(clippy::type_complexity)]
fn trial_decrypt<P>(
parameters: &P,
scanning_keys: &ScanningKeys<AccountId, KeyId>,
compact_blocks: &[CompactBlock],
) -> Result<
(
Vec<(OutputId, sapling_crypto::Note, KeyId)>,
Vec<(OutputId, orchard::Note, KeyId)>,
),
(),
>
) -> Result<BatchRunners<KeyId, (), ()>, ()>
where
P: Parameters + Send + 'static,
{
let mut runners = BatchRunners::<_, (), ()>::for_keys(100, scanning_keys);
// TODO: add outgoing decryption

let mut runners = BatchRunners::<KeyId, (), ()>::for_keys(100, scanning_keys);
for block in compact_blocks {
runners.add_block(parameters, block.clone()).unwrap();
}
runners.flush();

let mut incoming_sapling_outputs: Vec<(OutputId, sapling_crypto::Note, KeyId)> = Vec::new();
let mut incoming_orchard_outputs: Vec<(OutputId, orchard::Note, KeyId)> = Vec::new();

// TODO: add outgoing decryption
for block in compact_blocks {
for transaction in block.vtx.iter() {
let decrypted_sapling_outputs = runners
.sapling
.collect_results(block.hash(), transaction.txid());
decrypted_sapling_outputs
.iter()
.for_each(|((txid, output_index), output)| {
incoming_sapling_outputs.push((
OutputId::from_parts(*txid, *output_index),
output.note.clone(),
output.ivk_tag,
));
});

let decrypted_orchard_outputs = runners
.orchard
.collect_results(block.hash(), transaction.txid());
decrypted_orchard_outputs
.iter()
.for_each(|((txid, output_index), output)| {
incoming_orchard_outputs.push((
OutputId::from_parts(*txid, *output_index),
output.note,
output.ivk_tag,
));
});
}
}

Ok((incoming_sapling_outputs, incoming_orchard_outputs))
Ok(runners)
}

// checks height and hash continuity of a batch of compact blocks.
Expand Down Expand Up @@ -293,30 +277,120 @@ fn check_continuity(
Ok(())
}

// calculates nullifiers and positions for a given compact transaction
// calculates nullifiers and positions for a given compact transaction and insert into hash map
// `tree_size` is the tree size of the corresponding shielded pool up to - and not including - the compact transaction
// being processed
fn calculate_nullifiers_and_positions<D, K, Nf>(
transaction: &CompactTx,
tx_initial_tree_size: u32,
tree_size: u32,
keys: &HashMap<KeyId, K>,
incoming_outputs: &[(OutputId, D::Note, KeyId)],
incoming_outputs: &HashMap<OutputId, DecryptedOutput<KeyId, D, ()>>,
nullifiers_and_positions: &mut HashMap<OutputId, (Nf, Position)>,
) where
D: Domain,
K: zcash_client_backend::scanning::ScanningKeyOps<D, AccountId, Nf>,
{
incoming_outputs
.iter()
.filter(|(output_id, _, _)| output_id.txid() == transaction.txid())
.for_each(|(output_id, note, key_id)| {
.for_each(|(output_id, incoming_output)| {
let position = Position::from(u64::from(
tx_initial_tree_size + u32::try_from(output_id.output_index()).unwrap(),
tree_size + u32::try_from(output_id.output_index()).unwrap(),
));
let key = keys
.get(key_id)
.get(&incoming_output.ivk_tag)
.expect("key should be available as it was used to decrypt output");
let nullifier = key
.nf(note, position)
.nf(&incoming_output.note, position)
.expect("only fvks currently supported");
nullifiers_and_positions.insert(*output_id, (nullifier, position));
});
}

// TODO: unify sapling and orchard leaf and retention fns
// calculates the sapling note commitment tree leaves and shardtree retentions for a given compact transaction
fn calculate_sapling_leaves_and_retentions<D: Domain>(
outputs: &[CompactSaplingOutput],
block_height: BlockHeight,
last_outputs_in_block: bool,
decrypted_incoming_outputs: &HashMap<OutputId, DecryptedOutput<KeyId, D, ()>>,
) -> Result<Vec<(Node, Retention<BlockHeight>)>, ()> {
let incoming_output_indices: Vec<usize> = decrypted_incoming_outputs
.keys()
.copied()
.map(|output_id| output_id.output_index())
.collect();

if outputs.is_empty() {
Ok(Vec::new())
} else {
let last_output_index = outputs.len() - 1;

let leaves_and_retentions = outputs
.iter()
.enumerate()
.map(|(output_index, output)| {
let note_commitment = CompactOutputDescription::try_from(output).unwrap().cmu;
let leaf = sapling_crypto::Node::from_cmu(&note_commitment);

let last_output_in_block: bool =
last_outputs_in_block && output_index == last_output_index;
let decrypted: bool = incoming_output_indices.contains(&output_index);
let retention = match (decrypted, last_output_in_block) {
(is_marked, true) => Retention::Checkpoint {
id: block_height,
is_marked,
},
(true, false) => Retention::Marked,
(false, false) => Retention::Ephemeral,
};

(leaf, retention)
})
.collect();

Ok(leaves_and_retentions)
}
}
// calculates the orchard note commitment tree leaves and shardtree retentions for a given compact transaction
fn calculate_orchard_leaves_and_retentions<D: Domain>(
actions: &[CompactOrchardAction],
block_height: BlockHeight,
last_outputs_in_block: bool,
decrypted_incoming_outputs: &HashMap<OutputId, DecryptedOutput<KeyId, D, ()>>,
) -> Result<Vec<(MerkleHashOrchard, Retention<BlockHeight>)>, ()> {
let incoming_output_indices: Vec<usize> = decrypted_incoming_outputs
.keys()
.copied()
.map(|output_id| output_id.output_index())
.collect();

if actions.is_empty() {
Ok(Vec::new())
} else {
let last_output_index = actions.len() - 1;

let leaves_and_retentions = actions
.iter()
.enumerate()
.map(|(output_index, output)| {
let note_commitment = CompactAction::try_from(output).unwrap().cmx();
let leaf = MerkleHashOrchard::from_cmx(&note_commitment);

let last_output_in_block: bool =
last_outputs_in_block && output_index == last_output_index;
let decrypted: bool = incoming_output_indices.contains(&output_index);
let retention = match (decrypted, last_output_in_block) {
(is_marked, true) => Retention::Checkpoint {
id: block_height,
is_marked,
},
(true, false) => Retention::Marked,
(false, false) => Retention::Ephemeral,
};

(leaf, retention)
})
.collect();

Ok(leaves_and_retentions)
}
}
8 changes: 6 additions & 2 deletions zingo-sync/src/scan/runners.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ use zcash_primitives::{block::BlockHash, transaction::TxId};

use memuse::DynamicUsage;

use crate::primitives::OutputId;

type TaggedSaplingBatch<IvkTag> = Batch<
IvkTag,
SaplingDomain,
Expand Down Expand Up @@ -593,7 +595,7 @@ where
&mut self,
block_tag: BlockHash,
txid: TxId,
) -> HashMap<(TxId, usize), DecryptedOutput<IvkTag, D, Dec::Memo>> {
) -> HashMap<OutputId, DecryptedOutput<IvkTag, D, Dec::Memo>> {
self.pending_results
.remove(&ResultKey(block_tag, txid))
// We won't have a pending result if the transaction didn't have outputs of
Expand All @@ -610,7 +612,9 @@ where
|OutputIndex {
output_index,
value,
}| { ((txid, output_index), value) },
}| {
(OutputId::from_parts(txid, output_index), value)
},
)
.collect()
})
Expand Down

0 comments on commit a23564e

Please sign in to comment.