diff --git a/zcash_client_backend/CHANGELOG.md b/zcash_client_backend/CHANGELOG.md index 1d3860dcd5..917e204215 100644 --- a/zcash_client_backend/CHANGELOG.md +++ b/zcash_client_backend/CHANGELOG.md @@ -18,9 +18,19 @@ and this library adheres to Rust's notion of `map_internal_account_note` and `map_ephemeral_transparent_outpoint` and `internal_account_note_transpose_option` methods have consequently been removed. -- `zcash_client_backend::data_api::WalletRead::get_known_ephemeral_addresses` - now takes a `Range` as its - argument instead of a `Range` +- `zcash_client_backend::data_api::WalletRead`: + - `get_transparent_receivers` now takes additional `include_change` and + `include_ephemeral` arguments. + - `get_known_ephemeral_addresses` now takes a + `Range` as its argument + instead of a `Range` +- `zcash_client_backend::data_api::WalletWrite` has an added method + `get_address_for_index` + +### Removed +- `zcash_client_backend::data_api::GAP_LIMIT` gap limits are now configured + based upon the key scope that they're associated with; there is no longer a + globally applicable gap limit. ### Deprecated - `zcash_client_backend::address` (use `zcash_keys::address` instead) diff --git a/zcash_client_backend/src/data_api.rs b/zcash_client_backend/src/data_api.rs index 15f9dafbc8..dc1a5be4fd 100644 --- a/zcash_client_backend/src/data_api.rs +++ b/zcash_client_backend/src/data_api.rs @@ -81,7 +81,7 @@ use zcash_protocol::{ value::{BalanceError, Zatoshis}, ShieldedProtocol, TxId, }; -use zip32::fingerprint::SeedFingerprint; +use zip32::{fingerprint::SeedFingerprint, DiversifierIndex}; use self::{ chain::{ChainState, CommitmentTreeRoot}, @@ -127,10 +127,6 @@ pub const SAPLING_SHARD_HEIGHT: u8 = sapling::NOTE_COMMITMENT_TREE_DEPTH / 2; #[cfg(feature = "orchard")] pub const ORCHARD_SHARD_HEIGHT: u8 = { orchard::NOTE_COMMITMENT_TREE_DEPTH as u8 } / 2; -/// The number of ephemeral addresses that can be safely reserved without observing any -/// of them to be mined. This is the same as the gap limit in Bitcoin. -pub const GAP_LIMIT: u32 = 20; - /// An enumeration of constraints that can be applied when querying for nullifiers for notes /// belonging to the wallet. pub enum NullifierQuery { @@ -1369,6 +1365,8 @@ pub trait WalletRead { fn get_transparent_receivers( &self, _account: Self::AccountId, + _include_change: bool, + _include_ephemeral: bool, ) -> Result>, Self::Error> { Ok(HashMap::new()) } @@ -1393,7 +1391,7 @@ pub trait WalletRead { /// This is equivalent to (but may be implemented more efficiently than): /// ```compile_fail /// Ok( - /// if let Some(result) = self.get_transparent_receivers(account)?.get(address) { + /// if let Some(result) = self.get_transparent_receivers(account, true, true)?.get(address) { /// result.clone() /// } else { /// self.get_known_ephemeral_addresses(account, None)? @@ -1414,7 +1412,10 @@ pub trait WalletRead { ) -> Result, Self::Error> { // This should be overridden. Ok( - if let Some(result) = self.get_transparent_receivers(account)?.get(address) { + if let Some(result) = self + .get_transparent_receivers(account, true, true)? + .get(address) + { result.clone() } else { self.get_known_ephemeral_addresses(account, None)? @@ -1427,8 +1428,8 @@ pub trait WalletRead { /// Returns a vector of ephemeral transparent addresses associated with the given /// account controlled by this wallet, along with their metadata. The result includes - /// reserved addresses, and addresses for [`GAP_LIMIT`] additional indices (capped to - /// the maximum index). + /// reserved addresses, and addresses for the backend's configured gap limit worth + /// of additional indices (capped to the maximum index). /// /// If `index_range` is some `Range`, it limits the result to addresses with indices /// in that range. An `index_range` of `None` is defined to be equivalent to @@ -2360,9 +2361,10 @@ pub trait WalletWrite: WalletRead { key_source: Option<&str>, ) -> Result; - /// Generates and persists the next available diversified address for the specified account, - /// given the current addresses known to the wallet. If the `request` parameter is `None`, - /// an address should be generated using all of the available receivers for the account's UFVK. + /// Generates, persists, and marks as exposed the next available diversified address for the + /// specified account, given the current addresses known to the wallet. If the `request` + /// parameter is `None`, an address should be generated using all of the available receivers + /// for the account's UFVK. /// /// Returns `Ok(None)` if the account identifier does not correspond to a known /// account. @@ -2372,6 +2374,24 @@ pub trait WalletWrite: WalletRead { request: Option, ) -> Result, Self::Error>; + /// Generates, persists, and marks as exposed a diversified address for the specified account + /// at the provided diversifier index. If the `request` parameter is `None`, an address should + /// be generated using all of the available receivers for the account's UFVK. + /// + /// In the case that the diversifier index is outside of the range of valid transparent address + /// indexes, no transparent receiver should be generated in the resulting unified address. If a + /// transparent receiver is specifically requested for such a diversifier index, + /// implementations of this method should return an error. + /// + /// Address generation should fail if a transparent receiver would be generated that violates + /// the backend's internally configured gap limit for HD-seed-based recovery. + fn get_address_for_index( + &mut self, + account: Self::AccountId, + diversifier_index: DiversifierIndex, + request: Option, + ) -> Result, Self::Error>; + /// Updates the wallet's view of the blockchain. /// /// This method is used to provide the wallet with information about the state of the diff --git a/zcash_client_backend/src/data_api/testing.rs b/zcash_client_backend/src/data_api/testing.rs index 4b88bb032a..188ef9c83c 100644 --- a/zcash_client_backend/src/data_api/testing.rs +++ b/zcash_client_backend/src/data_api/testing.rs @@ -2599,6 +2599,8 @@ impl WalletRead for MockWalletDb { fn get_transparent_receivers( &self, _account: Self::AccountId, + _include_change: bool, + _include_ephemeral: bool, ) -> Result>, Self::Error> { Ok(HashMap::new()) } @@ -2689,6 +2691,15 @@ impl WalletWrite for MockWalletDb { Ok(None) } + fn get_address_for_index( + &mut self, + _account: Self::AccountId, + _diversifier_index: DiversifierIndex, + _request: Option, + ) -> Result, Self::Error> { + Ok(None) + } + #[allow(clippy::type_complexity)] fn put_blocks( &mut self, diff --git a/zcash_client_backend/src/data_api/testing/pool.rs b/zcash_client_backend/src/data_api/testing/pool.rs index a58e73b78f..951f92a6c2 100644 --- a/zcash_client_backend/src/data_api/testing/pool.rs +++ b/zcash_client_backend/src/data_api/testing/pool.rs @@ -60,7 +60,7 @@ use super::{DataStoreFactory, Reset, TestCache, TestFvk, TestState}; #[cfg(feature = "transparent-inputs")] use { crate::{ - data_api::{TransactionDataRequest, TransactionStatus}, + data_api::TransactionDataRequest, fees::ChangeValue, proposal::{Proposal, ProposalError, StepOutput, StepOutputIndex}, wallet::{TransparentAddressMetadata, WalletTransparentOutput}, @@ -86,6 +86,11 @@ use zcash_protocol::PoolType; #[cfg(feature = "pczt")] use pczt::roles::{prover::Prover, signer::Signer}; +/// The number of ephemeral addresses that can be safely reserved without observing any +/// of them to be mined. +#[cfg(feature = "transparent-inputs")] +pub const EPHEMERAL_ADDR_GAP_LIMIT: usize = 5; + /// Trait that exposes the pool-specific types and operations necessary to run the /// single-shielded-pool tests on a given pool. /// @@ -510,7 +515,7 @@ pub fn send_multi_step_proposed_transfer( { use ::transparent::builder::TransparentSigningSet; - use crate::data_api::{OutputOfSentTx, GAP_LIMIT}; + use crate::data_api::OutputOfSentTx; let mut st = TestBuilder::new() .with_data_store_factory(ds_factory) @@ -535,24 +540,30 @@ pub fn send_multi_step_proposed_transfer( .block_height(), h ); - assert_eq!(st.get_spendable_balance(account_id, 1), value); h }; let value = Zatoshis::const_from_u64(100000); let transfer_amount = Zatoshis::const_from_u64(50000); - let run_test = |st: &mut TestState<_, DSF::DataStore, _>, expected_index| { + let run_test = |st: &mut TestState<_, DSF::DataStore, _>, expected_index, prior_balance| { // Add funds to the wallet. add_funds(st, value); + let initial_balance: Option = prior_balance + value; + assert_eq!( + st.get_spendable_balance(account_id, 1), + initial_balance.unwrap() + ); let expected_step0_fee = (zip317::MARGINAL_FEE * 3u64).unwrap(); let expected_step1_fee = zip317::MINIMUM_FEE; let expected_ephemeral = (transfer_amount + expected_step1_fee).unwrap(); let expected_step0_change = - (value - expected_ephemeral - expected_step0_fee).expect("sufficient funds"); + (initial_balance - expected_ephemeral - expected_step0_fee).expect("sufficient funds"); assert!(expected_step0_change.is_positive()); + let total_sent = (expected_step0_fee + expected_step1_fee + transfer_amount).unwrap(); + // Generate a ZIP 320 proposal, sending to the wallet's default transparent address // expressed as a TEX address. let tex_addr = match default_addr { @@ -598,6 +609,12 @@ pub fn send_multi_step_proposed_transfer( assert_matches!(&create_proposed_result, Ok(txids) if txids.len() == 2); let txids = create_proposed_result.unwrap(); + // Mine the created transactions. + for txid in txids.iter() { + let (h, _) = st.generate_next_block_including(*txid); + st.scan_cached_blocks(h, 1); + } + // Check that there are sent outputs with the correct values. let confirmed_sent: Vec> = txids .iter() @@ -661,12 +678,15 @@ pub fn send_multi_step_proposed_transfer( -ZatBalance::from(expected_ephemeral), ); - (ephemeral_address.unwrap().0, txids) + let ending_balance = st.get_spendable_balance(account_id, 1); + assert_eq!(initial_balance - total_sent, ending_balance.into()); + + (ephemeral_address.unwrap().0, txids, ending_balance) }; // Each transfer should use a different ephemeral address. - let (ephemeral0, txids0) = run_test(&mut st, 0); - let (ephemeral1, txids1) = run_test(&mut st, 1); + let (ephemeral0, _, bal_0) = run_test(&mut st, 0, Zatoshis::ZERO); + let (ephemeral1, _, _) = run_test(&mut st, 1, bal_0); assert_ne!(ephemeral0, ephemeral1); let height = add_funds(&mut st, value); @@ -707,7 +727,7 @@ pub fn send_multi_step_proposed_transfer( .wallet() .get_known_ephemeral_addresses(account_id, None) .unwrap(); - assert_eq!(known_addrs.len(), (GAP_LIMIT as usize) + 2); + assert_eq!(known_addrs.len(), EPHEMERAL_ADDR_GAP_LIMIT + 2); // Check that the addresses are all distinct. let known_set: HashSet<_> = known_addrs.iter().map(|(addr, _)| addr).collect(); @@ -732,7 +752,7 @@ pub fn send_multi_step_proposed_transfer( }, ); let mut transparent_signing_set = TransparentSigningSet::new(); - let (colliding_addr, _) = &known_addrs[10]; + let (colliding_addr, _) = &known_addrs[EPHEMERAL_ADDR_GAP_LIMIT - 1]; let utxo_value = (value - zip317::MINIMUM_FEE).unwrap(); assert_matches!( builder.add_transparent_output(colliding_addr, utxo_value), @@ -773,9 +793,15 @@ pub fn send_multi_step_proposed_transfer( ) .unwrap(); let txid = build_result.transaction().txid(); + + // Now, store the transaction, pretending it has been mined (we will actually mine the block + // next). This will cause the the gap start to move & a new `EPHEMERAL_ADDR_GAP_LIMIT` of + + // addresses to be created. + let target_height = st.latest_cached_block().unwrap().height() + 1; st.wallet_mut() .store_decrypted_tx(DecryptedTransaction::new( - None, + Some(target_height), build_result.transaction(), vec![], #[cfg(feature = "orchard")] @@ -783,25 +809,20 @@ pub fn send_multi_step_proposed_transfer( )) .unwrap(); - // Verify that storing the fully transparent transaction causes a transaction - // status request to be generated. - let tx_data_requests = st.wallet().transaction_data_requests().unwrap(); - assert!(tx_data_requests.contains(&TransactionDataRequest::GetStatus(txid))); - - // We call get_transparent_output with `allow_unspendable = true` to verify - // storage because the decrypted transaction has not yet been mined. - let utxo = st - .wallet() - .get_transparent_output(&OutPoint::new(txid.into(), 0), true) - .unwrap(); - assert_matches!(utxo, Some(v) if v.value() == utxo_value); + // Mine the transaction & scan it so that it is will be detected as mined. Note that + // `generate_next_block_including` does not actually do anything with fully-transparent + // transactions; we're doing this just to get the mined block that we added via + // `store_decrypted_tx` into the database. + let (h, _) = st.generate_next_block_including(txid); + st.scan_cached_blocks(h, 1); - // That should have advanced the start of the gap to index 11. + // At this point the start of the gap should be at index `EPHEMERAL_ADDR_GAP_LIMIT` and the new + // size of the known address set should be `EPHEMERAL_ADDR_GAP_LIMIT * 2`. let new_known_addrs = st .wallet() .get_known_ephemeral_addresses(account_id, None) .unwrap(); - assert_eq!(new_known_addrs.len(), (GAP_LIMIT as usize) + 11); + assert_eq!(new_known_addrs.len(), EPHEMERAL_ADDR_GAP_LIMIT * 2); assert!(new_known_addrs.starts_with(&known_addrs)); let reservation_should_succeed = |st: &mut TestState<_, DSF::DataStore, _>, n| { @@ -821,85 +842,20 @@ pub fn send_multi_step_proposed_transfer( }; let next_reserved = reservation_should_succeed(&mut st, 1); - assert_eq!(next_reserved[0], known_addrs[11]); - - // Calling `reserve_next_n_ephemeral_addresses(account_id, 1)` will have advanced - // the start of the gap to index 12. This also tests the `index_range` parameter. - let newer_known_addrs = st - .wallet() - .get_known_ephemeral_addresses( - account_id, - Some( - NonHardenedChildIndex::from_index(5).unwrap() - ..NonHardenedChildIndex::from_index(100).unwrap(), - ), - ) - .unwrap(); - assert_eq!(newer_known_addrs.len(), (GAP_LIMIT as usize) + 12 - 5); - assert!(newer_known_addrs.starts_with(&new_known_addrs[5..])); - - // None of the five transactions created above (two from each proposal and the - // one built manually) have been mined yet. So, the range of address indices - // that are safe to reserve is still 0..20, and we have already reserved 12 - // addresses, so trying to reserve another 9 should fail. - reservation_should_fail(&mut st, 9, 20); - reservation_should_succeed(&mut st, 8); - reservation_should_fail(&mut st, 1, 20); - - // Now mine the transaction with the ephemeral output at index 1. - // We already reserved 20 addresses, so this should allow 2 more (..22). - // It does not matter that the transaction with ephemeral output at index 0 - // remains unmined. - let (h, _) = st.generate_next_block_including(txids1.head); - st.scan_cached_blocks(h, 1); - reservation_should_succeed(&mut st, 2); - reservation_should_fail(&mut st, 1, 22); - - // Mining the transaction with the ephemeral output at index 0 at this point - // should make no difference. - let (h, _) = st.generate_next_block_including(txids0.head); - st.scan_cached_blocks(h, 1); - reservation_should_fail(&mut st, 1, 22); - - // Now mine the transaction with the ephemeral output at index 10. - let tx = build_result.transaction(); - let tx_index = 1; - let (h, _) = st.generate_next_block_from_tx(tx_index, tx); - st.scan_cached_blocks(h, 1); - - // The above `scan_cached_blocks` does not detect `tx` as interesting to the - // wallet. If a transaction is in the database with a null `mined_height`, - // as in this case, its `mined_height` will remain null unless either - // `put_tx_meta` or `set_transaction_status` is called on it. The former - // is normally called internally via `put_blocks` as a result of scanning, - // but not for the case of a fully transparent transaction. The latter is - // called by the wallet implementation in response to processing the - // `transaction_data_requests` queue. - - // The reservation should fail because `tx` is not yet seen as mined. - reservation_should_fail(&mut st, 1, 22); - - // Simulate the wallet processing the `transaction_data_requests` queue. - let tx_data_requests = st.wallet().transaction_data_requests().unwrap(); - assert!(tx_data_requests.contains(&TransactionDataRequest::GetStatus(tx.txid()))); - - // Respond to the GetStatus request. - st.wallet_mut() - .set_transaction_status(tx.txid(), TransactionStatus::Mined(h)) - .unwrap(); - - // We already reserved 22 addresses, so mining the transaction with the - // ephemeral output at index 10 should allow 9 more (..31). - reservation_should_succeed(&mut st, 9); - reservation_should_fail(&mut st, 1, 31); - - let newest_known_addrs = st - .wallet() - .get_known_ephemeral_addresses(account_id, None) - .unwrap(); - assert_eq!(newest_known_addrs.len(), (GAP_LIMIT as usize) + 31); - assert!(newest_known_addrs.starts_with(&known_addrs)); - assert!(newest_known_addrs[5..].starts_with(&newer_known_addrs)); + assert_eq!(next_reserved[0], known_addrs[EPHEMERAL_ADDR_GAP_LIMIT]); + + // The range of address indices that are safe to reserve now is + // 0..(EPHEMERAL_ADDR_GAP_LIMIT * 2 - 1)`, and we have already reserved or used + // `EPHEMERAL_ADDR_GAP_LIMIT + 1`, addresses, so trying to reserve another + // `EPHEMERAL_ADDR_GAP_LIMIT` should fail. + reservation_should_fail( + &mut st, + EPHEMERAL_ADDR_GAP_LIMIT, + (EPHEMERAL_ADDR_GAP_LIMIT * 2) as u32, + ); + reservation_should_succeed(&mut st, EPHEMERAL_ADDR_GAP_LIMIT - 1); + // Now we've reserved everything we can, we can't reserve one more + reservation_should_fail(&mut st, 1, (EPHEMERAL_ADDR_GAP_LIMIT * 2) as u32); } #[cfg(feature = "transparent-inputs")] diff --git a/zcash_client_backend/src/sync.rs b/zcash_client_backend/src/sync.rs index e2a8681544..8519d19fc4 100644 --- a/zcash_client_backend/src/sync.rs +++ b/zcash_client_backend/src/sync.rs @@ -135,7 +135,7 @@ where "Refreshing UTXOs for {:?} from height {}", account_id, start_height, ); - refresh_utxos(params, client, db_data, account_id, start_height).await?; + refresh_utxos(params, client, db_data, account_id, start_height, false).await?; } // 5) Get the suggested scan ranges from the wallet database @@ -498,6 +498,7 @@ async fn refresh_utxos( db_data: &mut DbT, account_id: DbT::AccountId, start_height: BlockHeight, + include_ephemeral: bool, ) -> Result<(), Error::Error, TrErr>> where P: Parameters + Send + 'static, @@ -510,7 +511,7 @@ where { let request = service::GetAddressUtxosArg { addresses: db_data - .get_transparent_receivers(account_id) + .get_transparent_receivers(account_id, true, include_ephemeral) .map_err(Error::Wallet)? .into_keys() .map(|addr| addr.encode(params)) diff --git a/zcash_client_sqlite/CHANGELOG.md b/zcash_client_sqlite/CHANGELOG.md index 22b31c43d2..e8cd3542b9 100644 --- a/zcash_client_sqlite/CHANGELOG.md +++ b/zcash_client_sqlite/CHANGELOG.md @@ -9,6 +9,14 @@ and this library adheres to Rust's notion of ### Changed - Migrated to `nonempty 0.11` +- `zcash_client_sqlite::error::SqliteClientError` variants have changed: + - The `EphemeralAddressReuse` variant has been removed and replaced + by a new generalized `AddressReuse` error variant. + - The `ReachedGapLimit` variant no longer includes the account UUID + for the account that reached the limit in its payload. + - Each row returned from the `v_received_outputs` view now exposes an + internal identifier for the address that received that output. This should + be ignored by external consumers of this view. ## [0.14.0] - 2024-12-16 diff --git a/zcash_client_sqlite/src/error.rs b/zcash_client_sqlite/src/error.rs index 7c9e48cc39..05958e435d 100644 --- a/zcash_client_sqlite/src/error.rs +++ b/zcash_client_sqlite/src/error.rs @@ -3,19 +3,18 @@ use std::error; use std::fmt; +use nonempty::NonEmpty; use shardtree::error::ShardTreeError; + use zcash_address::ParseError; use zcash_client_backend::data_api::NoteFilter; use zcash_keys::keys::AddressGenerationError; -use zcash_protocol::{consensus::BlockHeight, value::BalanceError, PoolType}; +use zcash_protocol::{consensus::BlockHeight, value::BalanceError, PoolType, TxId}; use crate::{wallet::commitment_tree, AccountUuid}; #[cfg(feature = "transparent-inputs")] -use { - ::transparent::address::TransparentAddress, zcash_keys::encoding::TransparentCodecError, - zcash_primitives::transaction::TxId, -}; +use {::transparent::address::TransparentAddress, zcash_keys::encoding::TransparentCodecError}; /// The primary error type for the SQLite wallet backend. #[derive(Debug)] @@ -119,16 +118,16 @@ pub enum SqliteClientError { NoteFilterInvalid(NoteFilter), /// The proposal cannot be constructed until transactions with previously reserved - /// ephemeral address outputs have been mined. The parameters are the account UUID and - /// the index that could not safely be reserved. + /// ephemeral address outputs have been mined. The error contains the index that could not + /// safely be reserved. #[cfg(feature = "transparent-inputs")] - ReachedGapLimit(AccountUuid, u32), + ReachedGapLimit(u32), - /// An ephemeral address would be reused. The parameters are the address in string - /// form, and the txid of the earliest transaction in which it is known to have been - /// used. - #[cfg(feature = "transparent-inputs")] - EphemeralAddressReuse(String, TxId), + /// The wallet attempted to create a transaction that would use of one of the wallet's + /// previously-used addresses, potentially creating a problem with on-chain transaction + /// linkability. The returned value contains the string encoding of the address and the txid(s) + /// of the transactions in which it is known to have been used. + AddressReuse(String, NonEmpty), } impl error::Error for SqliteClientError { @@ -185,12 +184,13 @@ impl fmt::Display for SqliteClientError { SqliteClientError::BalanceError(e) => write!(f, "Balance error: {}", e), SqliteClientError::NoteFilterInvalid(s) => write!(f, "Could not evaluate filter query: {:?}", s), #[cfg(feature = "transparent-inputs")] - SqliteClientError::ReachedGapLimit(account_id, bad_index) => write!(f, - "The proposal cannot be constructed until transactions with previously reserved ephemeral address outputs have been mined. \ - The ephemeral address in account {account_id:?} at index {bad_index} could not be safely reserved.", + SqliteClientError::ReachedGapLimit(bad_index) => write!(f, + "The proposal cannot be constructed until transactions with outputs to previously reserved ephemeral addresses have been mined. \ + The ephemeral address at index {bad_index} could not be safely reserved.", ), - #[cfg(feature = "transparent-inputs")] - SqliteClientError::EphemeralAddressReuse(address_str, txid) => write!(f, "The ephemeral address {address_str} previously used in txid {txid} would be reused."), + SqliteClientError::AddressReuse(address_str, txids) => { + write!(f, "The address {address_str} previously used in txid(s) {:?} would be reused.", txids) + } } } } diff --git a/zcash_client_sqlite/src/lib.rs b/zcash_client_sqlite/src/lib.rs index bcf2771974..6a464e7611 100644 --- a/zcash_client_sqlite/src/lib.rs +++ b/zcash_client_sqlite/src/lib.rs @@ -79,9 +79,12 @@ use zcash_protocol::{ use zip32::{fingerprint::SeedFingerprint, DiversifierIndex}; use crate::{error::SqliteClientError, wallet::commitment_tree::SqliteShardStore}; - -#[cfg(any(test, feature = "test-dependencies", not(feature = "orchard")))] -use zcash_protocol::PoolType; +use wallet::{ + chain_tip_height, + commitment_tree::{self, put_shard_roots}, + common::spendable_notes_meta, + SubtreeProgressEstimator, +}; #[cfg(feature = "orchard")] use { @@ -94,6 +97,7 @@ use { #[cfg(feature = "transparent-inputs")] use { ::transparent::{address::TransparentAddress, bundle::OutPoint, keys::NonHardenedChildIndex}, + std::collections::BTreeSet, zcash_client_backend::wallet::TransparentAddressMetadata, zcash_keys::encoding::AddressCodec, }; @@ -106,10 +110,17 @@ use maybe_rayon::{ #[cfg(any(test, feature = "test-dependencies"))] use { + rusqlite::named_params, zcash_client_backend::data_api::{testing::TransactionSummary, OutputOfSentTx, WalletTest}, zcash_keys::address::Address, }; +#[cfg(any(test, feature = "test-dependencies", feature = "transparent-inputs"))] +use wallet::KeyScope; + +#[cfg(any(test, feature = "test-dependencies", not(feature = "orchard")))] +use zcash_protocol::PoolType; + /// `maybe-rayon` doesn't provide this as a fallback, so we have to. #[cfg(not(feature = "multicore"))] trait ParallelSliceMut { @@ -131,15 +142,9 @@ use { pub mod chain; pub mod error; -pub mod wallet; -use wallet::{ - commitment_tree::{self, put_shard_roots}, - common::spendable_notes_meta, - SubtreeProgressEstimator, -}; - #[cfg(test)] mod testing; +pub mod wallet; /// The maximum number of blocks the wallet is allowed to rewind. This is /// consistent with the bound in zcashd, and allows block data deeper than @@ -181,7 +186,7 @@ pub(crate) const UA_TRANSPARENT: ReceiverRequirement = ReceiverRequirement::Requ /// events". Examples of these include: /// - Restoring a wallet from a backed-up seed. /// - Importing the same viewing key into two different wallet instances. -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Default)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Default, PartialOrd, Ord)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub struct AccountUuid(#[cfg_attr(feature = "serde", serde(with = "uuid::serde::compact"))] Uuid); @@ -212,7 +217,7 @@ impl AccountUuid { /// /// This is an ephemeral value for efficiently and generically working with accounts in a /// [`WalletDb`]. To reference accounts in external contexts, use [`AccountUuid`]. -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Default)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Default, PartialOrd, Ord)] pub(crate) struct AccountRef(u32); /// This implementation is retained under `#[cfg(test)]` for pre-AccountUuid testing. @@ -242,13 +247,63 @@ impl fmt::Display for ReceivedNoteId { pub struct UtxoId(pub i64); /// A newtype wrapper for sqlite primary key values for the transactions table. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord)] struct TxRef(pub i64); +/// A newtype wrapper for sqlite primary key values for the addresses table. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +struct AddressRef(pub(crate) i64); + +/// A data structure that can be used to configure custom gap limits for use in transparent address +/// rotation. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[cfg(feature = "transparent-inputs")] +pub struct GapLimits { + external: u32, + transparent_internal: u32, + ephemeral: u32, +} + +#[cfg(feature = "transparent-inputs")] +impl GapLimits { + pub fn new(external: u32, transparent_internal: u32, ephemeral: u32) -> Self { + Self { + external, + transparent_internal, + ephemeral, + } + } + + pub(crate) fn external(&self) -> u32 { + self.external + } + + pub(crate) fn transparent_internal(&self) -> u32 { + self.transparent_internal + } + + pub(crate) fn ephemeral(&self) -> u32 { + self.ephemeral + } +} + +#[cfg(feature = "transparent-inputs")] +impl Default for GapLimits { + fn default() -> Self { + Self { + external: 20, + transparent_internal: 5, + ephemeral: 5, + } + } +} + /// A wrapper for the SQLite connection to the wallet database. pub struct WalletDb { conn: C, params: P, + #[cfg(feature = "transparent-inputs")] + gap_limits: GapLimits, } /// A wrapper for a SQLite transaction affecting the wallet database. @@ -265,10 +320,21 @@ impl WalletDb { pub fn for_path>(path: F, params: P) -> Result { Connection::open(path).and_then(move |conn| { rusqlite::vtab::array::load_module(&conn)?; - Ok(WalletDb { conn, params }) + Ok(WalletDb { + conn, + params, + #[cfg(feature = "transparent-inputs")] + gap_limits: GapLimits::default(), + }) }) } + #[cfg(feature = "transparent-inputs")] + pub fn with_gap_limits(mut self, gap_limits: GapLimits) -> Self { + self.gap_limits = gap_limits; + self + } + pub fn transactionally>(&mut self, f: F) -> Result where F: FnOnce(&mut WalletDb, P>) -> Result, @@ -277,6 +343,8 @@ impl WalletDb { let mut wdb = WalletDb { conn: SqlTransaction(&tx), params: self.params.clone(), + #[cfg(feature = "transparent-inputs")] + gap_limits: self.gap_limits, }; let result = f(&mut wdb)?; tx.commit()?; @@ -624,8 +692,21 @@ impl, P: consensus::Parameters> WalletRead for W fn get_transparent_receivers( &self, account: Self::AccountId, + include_change: bool, + include_ephemeral: bool, ) -> Result>, Self::Error> { - wallet::transparent::get_transparent_receivers(self.conn.borrow(), &self.params, account) + let key_scopes: &[KeyScope] = match (include_change, include_ephemeral) { + (true, true) => &[KeyScope::EXTERNAL, KeyScope::INTERNAL, KeyScope::Ephemeral], + (true, false) => &[KeyScope::EXTERNAL, KeyScope::INTERNAL], + (false, true) => &[KeyScope::EXTERNAL, KeyScope::Ephemeral], + (false, false) => &[KeyScope::EXTERNAL], + }; + wallet::transparent::get_transparent_receivers( + self.conn.borrow(), + &self.params, + account, + key_scopes, + ) } #[cfg(feature = "transparent-inputs")] @@ -667,7 +748,7 @@ impl, P: consensus::Parameters> WalletRead for W self.conn.borrow(), &self.params, account_id, - index_range.map(|i| i.start.index()..i.end.index()), + index_range, ) } @@ -710,7 +791,6 @@ impl, P: consensus::Parameters> WalletTest for W protocol: ShieldedProtocol, ) -> Result, ::Error> { use crate::wallet::pool_code; - use rusqlite::named_params; let mut stmt_sent_notes = self.conn.borrow().prepare( "SELECT output_index @@ -737,29 +817,35 @@ impl, P: consensus::Parameters> WalletTest for W &self, txid: &TxId, ) -> Result, ::Error> { - let mut stmt_sent = self - .conn.borrow() - .prepare( - "SELECT value, to_address, ephemeral_addresses.address, ephemeral_addresses.address_index - FROM sent_notes - JOIN transactions ON transactions.id_tx = sent_notes.tx - LEFT JOIN ephemeral_addresses ON ephemeral_addresses.used_in_tx = sent_notes.tx - WHERE transactions.txid = ? - ORDER BY value", - )?; + let mut stmt_sent = self.conn.borrow().prepare( + "SELECT value, to_address, + a.cached_transparent_receiver_address, a.transparent_child_index + FROM sent_notes + JOIN transactions t ON t.id_tx = sent_notes.tx + LEFT JOIN transparent_received_outputs tro ON tro.transaction_id = t.id_tx + LEFT JOIN addresses a ON a.id = tro.address_id AND a.key_scope = :key_scope + WHERE t.txid = :txid + ORDER BY value", + )?; let sends = stmt_sent - .query_map(rusqlite::params![txid.as_ref()], |row| { - let v = row.get(0)?; - let to_address = row - .get::<_, Option>(1)? - .and_then(|s| Address::decode(&self.params, &s)); - let ephemeral_address = row - .get::<_, Option>(2)? - .and_then(|s| Address::decode(&self.params, &s)); - let address_index: Option = row.get(3)?; - Ok((v, to_address, ephemeral_address.zip(address_index))) - })? + .query_map( + named_params![ + ":txid": txid.as_ref(), + ":key_scope": KeyScope::Ephemeral.encode() + ], + |row| { + let v = row.get(0)?; + let to_address = row + .get::<_, Option>(1)? + .and_then(|s| Address::decode(&self.params, &s)); + let ephemeral_address = row + .get::<_, Option>(2)? + .and_then(|s| Address::decode(&self.params, &s)); + let address_index: Option = row.get(3)?; + Ok((v, to_address, ephemeral_address.zip(address_index))) + }, + )? .map(|res| { let (amount, external_recipient, ephemeral_address) = res?; Ok::<_, ::Error>(OutputOfSentTx::from_parts( @@ -871,6 +957,8 @@ impl WalletWrite for WalletDb }, wallet::ViewingKey::Full(Box::new(ufvk)), birthday, + #[cfg(feature = "transparent-inputs")] + &wdb.gap_limits, )?; Ok((account.id(), usk)) @@ -908,6 +996,8 @@ impl WalletWrite for WalletDb }, wallet::ViewingKey::Full(Box::new(ufvk)), birthday, + #[cfg(feature = "transparent-inputs")] + &wdb.gap_limits, )?; Ok((account, usk)) @@ -933,6 +1023,8 @@ impl WalletWrite for WalletDb }, wallet::ViewingKey::Full(Box::new(ufvk.to_owned())), birthday, + #[cfg(feature = "transparent-inputs")] + &wdb.gap_limits, ) }) } @@ -957,14 +1049,16 @@ impl WalletWrite for WalletDb }; let (addr, diversifier_index) = ufvk.find_address(search_from, request)?; - let account_id = wallet::get_account_ref(wdb.conn.0, account_uuid)?; - wallet::insert_address( + let chain_tip_height = chain_tip_height(wdb.conn.0)? + .ok_or(SqliteClientError::ChainHeightUnknown)?; + wallet::upsert_address( wdb.conn.0, &wdb.params, account_id, diversifier_index, &addr, + Some(chain_tip_height), )?; Ok(Some(addr)) @@ -974,6 +1068,15 @@ impl WalletWrite for WalletDb ) } + fn get_address_for_index( + &mut self, + _account: Self::AccountId, + _diversifier_index: DiversifierIndex, + _request: Option, + ) -> Result, Self::Error> { + todo!() + } + fn update_chain_tip(&mut self, tip_height: BlockHeight) -> Result<(), Self::Error> { let tx = self.conn.transaction()?; wallet::scanning::update_chain_tip(&tx, &self.params, tip_height)?; @@ -1021,6 +1124,10 @@ impl WalletWrite for WalletDb let mut orchard_commitments = vec![]; let mut last_scanned_height = None; let mut note_positions = vec![]; + + #[cfg(feature = "transparent-inputs")] + let mut tx_refs = BTreeSet::new(); + for block in blocks.into_iter() { if last_scanned_height .iter() @@ -1044,16 +1151,20 @@ impl WalletWrite for WalletDb )?; for tx in block.transactions() { - let tx_row = wallet::put_tx_meta(wdb.conn.0, tx, block.height())?; + let tx_ref = wallet::put_tx_meta(wdb.conn.0, tx, block.height())?; + + #[cfg(feature = "transparent-inputs")] + tx_refs.insert(tx_ref); + wallet::queue_tx_retrieval(wdb.conn.0, std::iter::once(tx.txid()), None)?; // Mark notes as spent and remove them from the scanning cache for spend in tx.sapling_spends() { - wallet::sapling::mark_sapling_note_spent(wdb.conn.0, tx_row, spend.nf())?; + wallet::sapling::mark_sapling_note_spent(wdb.conn.0, tx_ref, spend.nf())?; } #[cfg(feature = "orchard")] for spend in tx.orchard_spends() { - wallet::orchard::mark_orchard_note_spent(wdb.conn.0, tx_row, spend.nf())?; + wallet::orchard::mark_orchard_note_spent(wdb.conn.0, tx_ref, spend.nf())?; } for output in tx.sapling_outputs() { @@ -1071,7 +1182,14 @@ impl WalletWrite for WalletDb .transpose()? .flatten(); - wallet::sapling::put_received_note(wdb.conn.0, output, tx_row, spent_in)?; + wallet::sapling::put_received_note( + wdb.conn.0, + &wdb.params, + output, + tx_ref, + Some(block.height()), + spent_in, + )?; } #[cfg(feature = "orchard")] for output in tx.orchard_outputs() { @@ -1089,7 +1207,14 @@ impl WalletWrite for WalletDb .transpose()? .flatten(); - wallet::orchard::put_received_note(wdb.conn.0, output, tx_row, spent_in)?; + wallet::orchard::put_received_note( + wdb.conn.0, + &wdb.params, + output, + tx_ref, + Some(block.height()), + spent_in, + )?; } } @@ -1160,6 +1285,18 @@ impl WalletWrite for WalletDb orchard_commitments.extend(block_commitments.orchard.into_iter().map(Some)); } + #[cfg(feature = "transparent-inputs")] + for (account_id, key_scope) in wallet::involved_accounts(wdb.conn.0, tx_refs)? { + wallet::transparent::generate_gap_addresses( + wdb.conn.0, + &wdb.params, + account_id, + key_scope, + &wdb.gap_limits, + None, + )?; + } + // Prune the nullifier map of entries we no longer need. if let Some(meta) = wdb.block_fully_scanned()? { wallet::prune_nullifier_map( @@ -1414,11 +1551,27 @@ impl WalletWrite for WalletDb _output: &WalletTransparentOutput, ) -> Result { #[cfg(feature = "transparent-inputs")] - return wallet::transparent::put_received_transparent_utxo( - &self.conn, - &self.params, - _output, - ); + { + self.transactionally(|wdb| { + let (account_id, key_scope, utxo_id) = + wallet::transparent::put_received_transparent_utxo( + wdb.conn.0, + &wdb.params, + _output, + )?; + + wallet::transparent::generate_gap_addresses( + wdb.conn.0, + &wdb.params, + account_id, + key_scope, + &wdb.gap_limits, + None, + )?; + + Ok(utxo_id) + }) + } #[cfg(not(feature = "transparent-inputs"))] panic!( @@ -1430,7 +1583,15 @@ impl WalletWrite for WalletDb &mut self, d_tx: DecryptedTransaction, ) -> Result<(), Self::Error> { - self.transactionally(|wdb| wallet::store_decrypted_tx(wdb.conn.0, &wdb.params, d_tx)) + self.transactionally(|wdb| { + wallet::store_decrypted_tx( + wdb.conn.0, + &wdb.params, + d_tx, + #[cfg(feature = "transparent-inputs")] + &wdb.gap_limits, + ) + }) } fn store_transactions_to_be_sent( @@ -1457,12 +1618,16 @@ impl WalletWrite for WalletDb ) -> Result, Self::Error> { self.transactionally(|wdb| { let account_id = wallet::get_account_ref(wdb.conn.0, account_id)?; - wallet::transparent::ephemeral::reserve_next_n_ephemeral_addresses( + let reserved = wallet::transparent::reserve_next_n_addresses( wdb.conn.0, &wdb.params, account_id, + wallet::KeyScope::Ephemeral, + wdb.gap_limits.ephemeral(), n, - ) + )?; + + Ok(reserved.into_iter().map(|(_, a, m)| (a, m)).collect()) }) } @@ -1986,6 +2151,12 @@ mod tests { .build(); let account = st.test_account().cloned().unwrap(); + // We have to have the chain tip height in order to allocate new addresses, to record the + // exposed-at height. + st.wallet_mut() + .update_chain_tip(account.birthday().height()) + .unwrap(); + let current_addr = st.wallet().get_current_address(account.id()).unwrap(); assert!(current_addr.is_some()); @@ -2238,7 +2409,10 @@ mod tests { let ufvk = account.usk().to_unified_full_viewing_key(); let (taddr, _) = account.usk().default_transparent_address(); - let receivers = st.wallet().get_transparent_receivers(account.id()).unwrap(); + let receivers = st + .wallet() + .get_transparent_receivers(account.id(), false, false) + .unwrap(); // The receiver for the default UA should be in the set. assert!(receivers.contains_key( diff --git a/zcash_client_sqlite/src/testing/db.rs b/zcash_client_sqlite/src/testing/db.rs index e7fa1d300e..1ec0a82f6d 100644 --- a/zcash_client_sqlite/src/testing/db.rs +++ b/zcash_client_sqlite/src/testing/db.rs @@ -9,7 +9,6 @@ use tempfile::NamedTempFile; use rusqlite::{self}; use secrecy::SecretVec; use shardtree::{error::ShardTreeError, ShardTree}; -use zip32::fingerprint::SeedFingerprint; use zcash_client_backend::{ data_api::{ @@ -32,6 +31,7 @@ use zcash_protocol::{ consensus::BlockHeight, local_consensus::LocalNetwork, memo::Memo, value::Zatoshis, ShieldedProtocol, }; +use zip32::{fingerprint::SeedFingerprint, DiversifierIndex}; use crate::{ error::SqliteClientError, @@ -153,15 +153,6 @@ pub(crate) struct TestDbFactory { target_migrations: Option>, } -impl TestDbFactory { - #[allow(dead_code)] - pub(crate) fn new(target_migrations: Vec) -> Self { - Self { - target_migrations: Some(target_migrations), - } - } -} - impl DataStoreFactory for TestDbFactory { type Error = (); type AccountId = AccountUuid; diff --git a/zcash_client_sqlite/src/testing/pool.rs b/zcash_client_sqlite/src/testing/pool.rs index 1b5132a36f..95edb8be85 100644 --- a/zcash_client_sqlite/src/testing/pool.rs +++ b/zcash_client_sqlite/src/testing/pool.rs @@ -48,11 +48,11 @@ pub(crate) fn send_multi_step_proposed_transfer() { zcash_client_backend::data_api::testing::pool::send_multi_step_proposed_transfer::( TestDbFactory::default(), BlockCache::new(), - |e, account_id, expected_bad_index| { + |e, _, expected_bad_index| { matches!( e, - crate::error::SqliteClientError::ReachedGapLimit(acct, bad_index) - if acct == &account_id && bad_index == &expected_bad_index) + crate::error::SqliteClientError::ReachedGapLimit(bad_index) + if bad_index == &expected_bad_index) }, ) } diff --git a/zcash_client_sqlite/src/wallet.rs b/zcash_client_sqlite/src/wallet.rs index 99db16e477..23f0c57172 100644 --- a/zcash_client_sqlite/src/wallet.rs +++ b/zcash_client_sqlite/src/wallet.rs @@ -64,32 +64,29 @@ //! wallet. //! - `memo` the shielded memo associated with the output, if any. -use incrementalmerkletree::{Marking, Retention}; +use std::{ + collections::{HashMap, HashSet}, + convert::TryFrom, + io::{self, Cursor}, + num::NonZeroU32, + ops::RangeInclusive, +}; +use incrementalmerkletree::{Marking, Retention}; use rusqlite::{self, named_params, params, Connection, OptionalExtension}; use secrecy::{ExposeSecret, SecretVec}; use shardtree::{error::ShardTreeError, store::ShardStore, ShardTree}; -use uuid::Uuid; -use zcash_client_backend::data_api::{ - AccountPurpose, DecryptedTransaction, Progress, TransactionDataRequest, TransactionStatus, - Zip32Derivation, -}; -use zip32::fingerprint::SeedFingerprint; - -use std::collections::{HashMap, HashSet}; -use std::convert::TryFrom; -use std::io::{self, Cursor}; -use std::num::NonZeroU32; -use std::ops::RangeInclusive; - use tracing::{debug, warn}; +use uuid::Uuid; use zcash_address::ZcashAddress; use zcash_client_backend::{ data_api::{ scanning::{ScanPriority, ScanRange}, - Account as _, AccountBalance, AccountBirthday, AccountSource, BlockMetadata, Ratio, - SentTransaction, SentTransactionOutput, WalletSummary, SAPLING_SHARD_HEIGHT, + Account as _, AccountBalance, AccountBirthday, AccountPurpose, AccountSource, + BlockMetadata, DecryptedTransaction, Progress, Ratio, SentTransaction, + SentTransactionOutput, TransactionDataRequest, TransactionStatus, WalletSummary, + Zip32Derivation, SAPLING_SHARD_HEIGHT, }, wallet::{Note, NoteId, Recipient, WalletTx}, DecryptedOutput, @@ -105,27 +102,33 @@ use zcash_keys::{ use zcash_primitives::{ block::BlockHash, merkle_tree::read_commitment_tree, - transaction::{Transaction, TransactionData, TxId}, + transaction::{Transaction, TransactionData}, }; use zcash_protocol::{ - consensus::{self, BlockHeight, BranchId, NetworkUpgrade, Parameters}, + consensus::{self, BlockHeight, BranchId, NetworkConstants as _, NetworkUpgrade, Parameters}, memo::{Memo, MemoBytes}, value::{ZatBalance, Zatoshis}, - PoolType, ShieldedProtocol, + PoolType, ShieldedProtocol, TxId, }; -use zip32::{DiversifierIndex, Scope}; +use zip32::{fingerprint::SeedFingerprint, DiversifierIndex}; use self::scanning::{parse_priority_code, priority_code, replace_queue_entries}; use crate::{ error::SqliteClientError, wallet::commitment_tree::{get_max_checkpointed_height, SqliteShardStore}, - AccountRef, SqlTransaction, TransferType, WalletCommitmentTrees, WalletDb, PRUNING_DEPTH, - SAPLING_TABLES_PREFIX, + AccountRef, AccountUuid, AddressRef, SqlTransaction, TransferType, TxRef, + WalletCommitmentTrees, WalletDb, PRUNING_DEPTH, SAPLING_TABLES_PREFIX, VERIFY_LOOKAHEAD, }; -use crate::{AccountUuid, TxRef, VERIFY_LOOKAHEAD}; #[cfg(feature = "transparent-inputs")] -use ::transparent::bundle::{OutPoint, TxOut}; +use { + crate::GapLimits, + ::transparent::{ + bundle::{OutPoint, TxOut}, + keys::{NonHardenedChildIndex, TransparentKeyScope}, + }, + std::collections::BTreeMap, +}; #[cfg(feature = "orchard")] use {crate::ORCHARD_TABLES_PREFIX, zcash_client_backend::data_api::ORCHARD_SHARD_HEIGHT}; @@ -211,6 +214,7 @@ pub(crate) enum ViewingKey { /// An account stored in a `zcash_client_sqlite` database. #[derive(Debug, Clone)] pub struct Account { + id: AccountRef, uuid: AccountUuid, name: Option, kind: AccountSource, @@ -229,6 +233,10 @@ impl Account { ) -> Result<(UnifiedAddress, DiversifierIndex), AddressGenerationError> { self.uivk().default_address(request) } + + pub(crate) fn internal_id(&self) -> AccountRef { + self.id + } } impl zcash_client_backend::data_api::Account for Account { @@ -340,21 +348,91 @@ pub(crate) fn pool_code(pool_type: PoolType) -> i64 { } } -pub(crate) fn scope_code(scope: Scope) -> i64 { - match scope { - Scope::External => 0i64, - Scope::Internal => 1i64, +/// An enumeration of the scopes of keys that are generated by the `zcash_client_sqlite` +/// implementation of the `WalletWrite` trait. +/// +/// This extends the [`zip32::Scope`] type to include the custom scope used to generate keys for +/// ephemeral transparent addresses. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum KeyScope { + /// A key scope corresponding to a [`zip32::Scope`]. + Zip32(zip32::Scope), + /// An ephemeral transparent address, which is derived from an account's transparent + /// [`AccountPubKey`] with the BIP 44 path `change` level index set to the value `2`. + /// + /// [`AccountPubKey`]: zcash_primitives::legacy::keys::AccountPubKey + Ephemeral, +} + +impl KeyScope { + pub(crate) const EXTERNAL: KeyScope = KeyScope::Zip32(zip32::Scope::External); + pub(crate) const INTERNAL: KeyScope = KeyScope::Zip32(zip32::Scope::Internal); + + pub(crate) fn encode(&self) -> i64 { + match self { + KeyScope::Zip32(zip32::Scope::External) => 0i64, + KeyScope::Zip32(zip32::Scope::Internal) => 1i64, + KeyScope::Ephemeral => 2i64, + } + } + + pub(crate) fn decode(code: i64) -> Result { + match code { + 0i64 => Ok(KeyScope::EXTERNAL), + 1i64 => Ok(KeyScope::INTERNAL), + 2i64 => Ok(KeyScope::Ephemeral), + other => Err(SqliteClientError::CorruptedData(format!( + "Invalid key scope code: {}", + other + ))), + } + } +} + +impl From for KeyScope { + fn from(value: zip32::Scope) -> Self { + KeyScope::Zip32(value) + } +} + +#[cfg(feature = "transparent-inputs")] +impl From for TransparentKeyScope { + fn from(value: KeyScope) -> Self { + match value { + KeyScope::Zip32(scope) => scope.into(), + KeyScope::Ephemeral => TransparentKeyScope::custom(2).expect("valid scope"), + } } } -pub(crate) fn parse_scope(code: i64) -> Option { - match code { - 0i64 => Some(Scope::External), - 1i64 => Some(Scope::Internal), - _ => None, +impl TryFrom for zip32::Scope { + type Error = (); + + fn try_from(value: KeyScope) -> Result { + match value { + KeyScope::Zip32(scope) => Ok(scope), + KeyScope::Ephemeral => Err(()), + } } } +pub(crate) fn encode_diversifier_index_be(idx: DiversifierIndex) -> [u8; 11] { + let mut di_be = *idx.as_bytes(); + di_be.reverse(); + di_be +} + +pub(crate) fn decode_diversifier_index_be( + di_be: &[u8], +) -> Result { + let mut di_be: [u8; 11] = di_be.try_into().map_err(|_| { + SqliteClientError::CorruptedData("Diversifier index is not an 11-byte value".to_owned()) + })?; + di_be.reverse(); + + Ok(DiversifierIndex::from(di_be)) +} + pub(crate) fn memo_repr(memo: Option<&MemoBytes>) -> Option<&[u8]> { memo.map(|m| { if m == &MemoBytes::empty() { @@ -390,6 +468,7 @@ pub(crate) fn add_account( kind: &AccountSource, viewing_key: ViewingKey, birthday: &AccountBirthday, + #[cfg(feature = "transparent-inputs")] gap_limits: &GapLimits, ) -> Result { if let Some(ufvk) = viewing_key.ufvk() { // Check whether any component of this UFVK collides with an existing imported or derived FVK. @@ -508,6 +587,7 @@ pub(crate) fn add_account( })?; let account = Account { + id: account_id, name: Some(account_name.to_owned()), uuid: account_uuid, kind: kind.clone(), @@ -611,11 +691,21 @@ pub(crate) fn add_account( // key has fewer components than the wallet supports (most likely due to this being an // imported viewing key), derive an address containing the common subset of receivers. let (address, d_idx) = account.default_address(None)?; - insert_address(conn, params, account_id, d_idx, &address)?; + upsert_address( + conn, + params, + account_id, + d_idx, + &address, + Some(birthday.height()), + )?; - // Initialize the `ephemeral_addresses` table. + // Pre-generate transparent addresses up to the gap limits for the external, internal, + // and ephemeral key scopes. #[cfg(feature = "transparent-inputs")] - transparent::ephemeral::init_account(conn, params, account_id)?; + for key_scope in [KeyScope::EXTERNAL, KeyScope::INTERNAL, KeyScope::Ephemeral] { + transparent::generate_gap_addresses(conn, params, account_id, key_scope, gap_limits, None)?; + } Ok(account) } @@ -625,26 +715,31 @@ pub(crate) fn get_current_address( params: &P, account_uuid: AccountUuid, ) -> Result, SqliteClientError> { + let ua_prefix = params.network_type().hrp_unified_address(); // This returns the most recently generated address. let addr: Option<(String, Vec)> = conn .query_row( - "SELECT address, diversifier_index_be - FROM addresses - JOIN accounts ON addresses.account_id = accounts.id - WHERE accounts.uuid = :account_uuid - ORDER BY diversifier_index_be DESC - LIMIT 1", - named_params![":account_uuid": account_uuid.0], + &format!( + "SELECT address, diversifier_index_be + FROM addresses + JOIN accounts ON addresses.account_id = accounts.id + WHERE accounts.uuid = :account_uuid + AND key_scope = :key_scope + AND address LIKE '{ua_prefix}%' + AND exposed_at_height IS NOT NULL + ORDER BY diversifier_index_be DESC + LIMIT 1" + ), + named_params![ + ":account_uuid": account_uuid.0, + ":key_scope": KeyScope::EXTERNAL.encode() + ], |row| Ok((row.get(0)?, row.get(1)?)), ) .optional()?; addr.map(|(addr_str, di_vec)| { - let mut di_be: [u8; 11] = di_vec.try_into().map_err(|_| { - SqliteClientError::CorruptedData("Diversifier index is not an 11-byte value".to_owned()) - })?; - di_be.reverse(); - + let diversifier_index = decode_diversifier_index_be(&di_vec)?; Address::decode(params, &addr_str) .ok_or_else(|| { SqliteClientError::CorruptedData("Not a valid Zcash recipient address".to_owned()) @@ -656,47 +751,103 @@ pub(crate) fn get_current_address( addr_str, ))), }) - .map(|addr| (addr, DiversifierIndex::from(di_be))) + .map(|addr| (addr, diversifier_index)) }) .transpose() } -/// Adds the given address and diversifier index to the addresses table. +/// Adds the given external address and diversifier index to the addresses table. /// -/// Returns the database row for the newly-inserted address. -pub(crate) fn insert_address( +/// Returns the primary key identifier for the newly-inserted address. +pub(crate) fn upsert_address( conn: &rusqlite::Connection, params: &P, account_id: AccountRef, diversifier_index: DiversifierIndex, address: &UnifiedAddress, -) -> Result<(), SqliteClientError> { + exposed_at_height: Option, +) -> Result { let mut stmt = conn.prepare_cached( "INSERT INTO addresses ( account_id, diversifier_index_be, + key_scope, address, - cached_transparent_receiver_address + transparent_child_index, + cached_transparent_receiver_address, + exposed_at_height ) VALUES ( :account_id, :diversifier_index_be, + :key_scope, :address, - :cached_transparent_receiver_address - )", + :transparent_child_index, + :cached_transparent_receiver_address, + :exposed_at_height + ) + ON CONFLICT (account_id, diversifier_index_be, key_scope) DO UPDATE + SET exposed_at_height = COALESCE( + MIN(exposed_at_height, :exposed_at_height), + exposed_at_height, + :exposed_at_height + ) + RETURNING id", )?; - // the diversifier index is stored in big-endian order to allow sorting - let mut di_be = *diversifier_index.as_bytes(); - di_be.reverse(); - stmt.execute(named_params![ - ":account_id": account_id.0, - ":diversifier_index_be": &di_be[..], - ":address": &address.encode(params), - ":cached_transparent_receiver_address": &address.transparent().map(|r| r.encode(params)), - ])?; + #[cfg(feature = "transparent-inputs")] + let transparent_child_index = NonHardenedChildIndex::try_from(diversifier_index) + .ok() + .map(|i| i.index()); + #[cfg(not(feature = "transparent-inputs"))] + let transparent_child_index: Option = None; - Ok(()) + stmt.query_row( + named_params![ + ":account_id": account_id.0, + // the diversifier index is stored in big-endian order to allow sorting + ":diversifier_index_be": encode_diversifier_index_be(diversifier_index), + ":key_scope": KeyScope::EXTERNAL.encode(), + ":address": &address.encode(params), + ":transparent_child_index": transparent_child_index, + ":cached_transparent_receiver_address": &address.transparent().map(|r| r.encode(params)), + ":exposed_at_height": exposed_at_height.map(u32::from) + ], + |row| row.get(0).map(AddressRef) + ).map_err(SqliteClientError::from) +} + +#[cfg(feature = "transparent-inputs")] +pub(crate) fn involved_accounts( + conn: &rusqlite::Connection, + tx_refs: impl IntoIterator, +) -> Result, SqliteClientError> { + use rusqlite::types::Value; + use std::rc::Rc; + + let mut stmt = conn.prepare_cached( + "SELECT account_id, key_scope + FROM v_address_uses + WHERE transaction_id IN rarray(:tx_refs_ptr)", + )?; + + let tx_refs_values: Vec = tx_refs.into_iter().map(|r| Value::Integer(r.0)).collect(); + let tx_refs_ptr = Rc::new(tx_refs_values); + let result = stmt + .query_and_then( + named_params! { + ":tx_refs_ptr": &tx_refs_ptr + }, + |row| { + Ok::<_, SqliteClientError>(( + row.get(0).map(AccountRef)?, + KeyScope::decode(row.get(1)?)?, + )) + }, + )? + .collect::, _>>()?; + + Ok(result) } /// Returns the [`UnifiedFullViewingKey`]s for the wallet. @@ -732,6 +883,7 @@ fn parse_account_row( row: &rusqlite::Row<'_>, params: &P, ) -> Result { + let account_id = AccountRef(row.get("id")?); let account_name = row.get("name")?; let account_uuid = AccountUuid(row.get("uuid")?); let kind = parse_account_source( @@ -765,6 +917,7 @@ fn parse_account_row( }; Ok(Account { + id: account_id, name: account_name, uuid: account_uuid, kind, @@ -779,7 +932,7 @@ pub(crate) fn get_account( ) -> Result, SqliteClientError> { let mut stmt = conn.prepare_cached( r#" - SELECT name, uuid, account_kind, + SELECT id, name, uuid, account_kind, hd_seed_fingerprint, hd_account_index, key_source, ufvk, uivk, has_spend_key FROM accounts @@ -795,6 +948,30 @@ pub(crate) fn get_account( rows.next().transpose() } +#[cfg(feature = "transparent-inputs")] +pub(crate) fn get_account_internal( + conn: &rusqlite::Connection, + params: &P, + account_id: AccountRef, +) -> Result, SqliteClientError> { + let mut stmt = conn.prepare_cached( + r#" + SELECT id, name, uuid, account_kind, + hd_seed_fingerprint, hd_account_index, key_source, + ufvk, uivk, has_spend_key + FROM accounts + WHERE id = :account_id + "#, + )?; + + let mut rows = stmt.query_and_then::<_, SqliteClientError, _, _>( + named_params![":account_id": account_id.0], + |row| parse_account_row(row, params), + )?; + + rows.next().transpose() +} + /// Returns the account id corresponding to a given [`UnifiedFullViewingKey`], /// if any. pub(crate) fn get_account_for_ufvk( @@ -815,7 +992,7 @@ pub(crate) fn get_account_for_ufvk( let transparent_item: Option> = None; let mut stmt = conn.prepare( - "SELECT name, uuid, account_kind, + "SELECT id, name, uuid, account_kind, hd_seed_fingerprint, hd_account_index, key_source, ufvk, uivk, has_spend_key FROM accounts @@ -853,7 +1030,7 @@ pub(crate) fn get_derived_account( account_index: zip32::AccountId, ) -> Result, SqliteClientError> { let mut stmt = conn.prepare( - "SELECT name, key_source, uuid, ufvk + "SELECT id, name, key_source, uuid, ufvk FROM accounts WHERE hd_seed_fingerprint = :hd_seed_fingerprint AND hd_account_index = :hd_account_index", @@ -865,6 +1042,7 @@ pub(crate) fn get_derived_account( ":hd_account_index": u32::from(account_index), ], |row| { + let account_id = AccountRef(row.get("id")?); let account_name = row.get("name")?; let key_source = row.get("key_source")?; let account_uuid = AccountUuid(row.get("uuid")?); @@ -881,6 +1059,7 @@ pub(crate) fn get_derived_account( }), }?; Ok(Account { + id: account_id, name: account_name, uuid: account_uuid, kind: AccountSource::Derived { @@ -1933,20 +2112,6 @@ pub(crate) fn get_account_ref( .ok_or(SqliteClientError::AccountUnknown) } -#[cfg(feature = "transparent-inputs")] -pub(crate) fn get_account_uuid( - conn: &rusqlite::Connection, - account_id: AccountRef, -) -> Result { - conn.query_row( - "SELECT uuid FROM accounts WHERE id = :account_id", - named_params! {":account_id": account_id.0}, - |row| row.get("uuid").map(AccountUuid), - ) - .optional()? - .ok_or(SqliteClientError::AccountUnknown) -} - /// Returns the minimum and maximum heights of blocks in the chain which may be scanned. pub(crate) fn chain_tip_height( conn: &rusqlite::Connection, @@ -2262,6 +2427,13 @@ pub(crate) fn store_transaction_to_be_sent( )?; match output.recipient() { + Recipient::External { + recipient_address: _addr, + output_pool: _pool, + .. + } => { + // Nothing to do for external recipients. + } Recipient::InternalAccount { receiving_account, note, @@ -2270,6 +2442,7 @@ pub(crate) fn store_transaction_to_be_sent( Note::Sapling(note) => { sapling::put_received_note( wdb.conn.0, + &wdb.params, &DecryptedOutput::new( output.output_index(), note.clone(), @@ -2280,6 +2453,7 @@ pub(crate) fn store_transaction_to_be_sent( TransferType::WalletInternal, ), tx_ref, + Some(sent_tx.target_height()), None, )?; } @@ -2287,6 +2461,7 @@ pub(crate) fn store_transaction_to_be_sent( Note::Orchard(note) => { orchard::put_received_note( wdb.conn.0, + &wdb.params, &DecryptedOutput::new( output.output_index(), *note, @@ -2297,16 +2472,25 @@ pub(crate) fn store_transaction_to_be_sent( TransferType::WalletInternal, ), tx_ref, + Some(sent_tx.target_height()), None, )?; } }, #[cfg(feature = "transparent-inputs")] Recipient::EphemeralTransparent { - receiving_account, ephemeral_address, outpoint, + .. } => { + // First check to verify that creation of this output does not result in reuse of + // an ephemeral address. + transparent::check_address_reuse( + wdb.conn.0, + &wdb.params, + &Address::Transparent(*ephemeral_address), + )?; + transparent::put_transparent_output( wdb.conn.0, &wdb.params, @@ -2317,17 +2501,9 @@ pub(crate) fn store_transaction_to_be_sent( }, None, ephemeral_address, - *receiving_account, true, )?; - transparent::ephemeral::mark_ephemeral_address_as_used( - wdb.conn.0, - &wdb.params, - ephemeral_address, - tx_ref, - )?; } - _ => {} } } @@ -2536,6 +2712,8 @@ pub(crate) fn truncate_to_height( let mut wdb = WalletDb { conn: SqlTransaction(conn), params: params.clone(), + #[cfg(feature = "transparent-inputs")] + gap_limits: GapLimits::default(), }; wdb.with_sapling_tree_mut(|tree| { tree.truncate_to_checkpoint(&truncation_height)?; @@ -2693,6 +2871,7 @@ pub(crate) fn store_decrypted_tx( conn: &rusqlite::Transaction, params: &P, d_tx: DecryptedTransaction, + #[cfg(feature = "transparent-inputs")] gap_limits: &GapLimits, ) -> Result<(), SqliteClientError> { let tx_ref = put_tx_data(conn, d_tx.tx(), None, None, None)?; if let Some(height) = d_tx.mined_height() { @@ -2717,6 +2896,9 @@ pub(crate) fn store_decrypted_tx( #[cfg(feature = "transparent-inputs")] let mut tx_has_wallet_outputs = false; + #[cfg(feature = "transparent-inputs")] + let mut receiving_accounts = BTreeMap::new(); + for output in d_tx.sapling_outputs() { #[cfg(feature = "transparent-inputs")] { @@ -2748,7 +2930,14 @@ pub(crate) fn store_decrypted_tx( )?; } TransferType::WalletInternal => { - sapling::put_received_note(conn, output, tx_ref, None)?; + sapling::put_received_note( + conn, + params, + output, + tx_ref, + d_tx.mined_height(), + None, + )?; let recipient = Recipient::InternalAccount { receiving_account: *output.account(), @@ -2768,7 +2957,17 @@ pub(crate) fn store_decrypted_tx( )?; } TransferType::Incoming => { - sapling::put_received_note(conn, output, tx_ref, None)?; + let _account_id = sapling::put_received_note( + conn, + params, + output, + tx_ref, + d_tx.mined_height(), + None, + )?; + + #[cfg(feature = "transparent-inputs")] + receiving_accounts.insert(_account_id, KeyScope::EXTERNAL); if let Some(account_id) = funding_account { let recipient = Recipient::InternalAccount { @@ -2837,7 +3036,14 @@ pub(crate) fn store_decrypted_tx( )?; } TransferType::WalletInternal => { - orchard::put_received_note(conn, output, tx_ref, None)?; + orchard::put_received_note( + conn, + params, + output, + tx_ref, + d_tx.mined_height(), + None, + )?; let recipient = Recipient::InternalAccount { receiving_account: *output.account(), @@ -2857,7 +3063,17 @@ pub(crate) fn store_decrypted_tx( )?; } TransferType::Incoming => { - orchard::put_received_note(conn, output, tx_ref, None)?; + let _account_id = orchard::put_received_note( + conn, + params, + output, + tx_ref, + d_tx.mined_height(), + None, + )?; + + #[cfg(feature = "transparent-inputs")] + receiving_accounts.insert(_account_id, KeyScope::EXTERNAL); if let Some(account_id) = funding_account { // Even if the recipient address is external, record the send as internal. @@ -2923,18 +3139,9 @@ pub(crate) fn store_decrypted_tx( address.encode(params) ); - // The transaction is not necessarily mined yet, but we want to record - // that an output to the address was seen in this tx anyway. This will - // advance the gap regardless of whether it is mined, but an output in - // an unmined transaction won't advance the range of safe indices. - #[cfg(feature = "transparent-inputs")] - transparent::ephemeral::mark_ephemeral_address_as_seen( - conn, params, &address, tx_ref, - )?; - // If the output belongs to the wallet, add it to `transparent_received_outputs`. #[cfg(feature = "transparent-inputs")] - if let Some(account_uuid) = + if let Some((account_uuid, key_scope)) = transparent::find_account_uuid_for_transparent_address(conn, params, &address)? { debug!( @@ -2943,7 +3150,7 @@ pub(crate) fn store_decrypted_tx( output_index, account_uuid ); - transparent::put_transparent_output( + let (account_id, _, _) = transparent::put_transparent_output( conn, params, &OutPoint::new( @@ -2953,10 +3160,11 @@ pub(crate) fn store_decrypted_tx( txout, d_tx.mined_height(), &address, - account_uuid, false, )?; + receiving_accounts.insert(account_id, key_scope); + // Since the wallet created the transparent output, we need to ensure // that any transparent inputs belonging to the wallet will be // discovered. @@ -3028,6 +3236,11 @@ pub(crate) fn store_decrypted_tx( } } + #[cfg(feature = "transparent-inputs")] + for (account_id, key_scope) in receiving_accounts { + transparent::generate_gap_addresses(conn, params, account_id, key_scope, gap_limits, None)?; + } + // If the transaction has outputs that belong to the wallet as well as transparent // inputs, we may need to download the transactions corresponding to the transparent // prevout references to determine whether the transaction was created (at least in @@ -3106,10 +3319,14 @@ pub(crate) fn select_receiving_address( "SELECT address FROM addresses JOIN accounts ON accounts.id = addresses.account_id - WHERE accounts.uuid = :account_uuid", + WHERE accounts.uuid = :account_uuid + AND key_scope = :key_scope", )?; - let mut result = stmt.query(named_params! { ":account_uuid": account.0 })?; + let mut result = stmt.query(named_params! { + ":account_uuid": account.0, + ":key_scope": KeyScope::EXTERNAL.encode(), + })?; while let Some(row) = result.next()? { let addr_str = row.get::<_, String>(0)?; let decoded = addr_str.parse::()?; @@ -3360,7 +3577,7 @@ fn flag_previously_received_change( ), named_params! { ":tx": tx_ref.0, - ":internal_scope": scope_code(Scope::Internal) + ":internal_scope": KeyScope::INTERNAL.encode() }, ) }; diff --git a/zcash_client_sqlite/src/wallet/db.rs b/zcash_client_sqlite/src/wallet/db.rs index 6520ac1834..0eed093dc4 100644 --- a/zcash_client_sqlite/src/wallet/db.rs +++ b/zcash_client_sqlite/src/wallet/db.rs @@ -13,9 +13,7 @@ // from showing up in `cargo doc --document-private-items`. #![allow(dead_code)] -use static_assertions::const_assert_eq; - -use zcash_client_backend::data_api::{scanning::ScanPriority, GAP_LIMIT}; +use zcash_client_backend::data_api::scanning::ScanPriority; use zcash_protocol::consensus::{NetworkUpgrade, Parameters}; use crate::wallet::scanning::priority_code; @@ -63,85 +61,57 @@ pub(super) const INDEX_ACCOUNTS_UIVK: &str = pub(super) const INDEX_HD_ACCOUNT: &str = r#"CREATE UNIQUE INDEX hd_account ON accounts (hd_seed_fingerprint, hd_account_index)"#; -/// Stores diversified Unified Addresses that have been generated from accounts in the -/// wallet. +/// Stores addresses that have been generated from accounts in the wallet. +/// +/// ### Columns /// -/// - The `cached_transparent_receiver_address` column contains the transparent receiver component -/// of the UA. It is cached directly in the table to make account lookups for transparent outputs -/// more efficient, enabling joins to [`TABLE_TRANSPARENT_RECEIVED_OUTPUTS`]. +/// - `account_id`: the account whose IVK was used to derive this address. +/// - `diversifier_index_be`: the diversifier index at which this address was derived. +/// - `key_scope`: the key scope for which this address was derived. +/// - `address`: The Unified, Sapling, or transparent address. For Unified and Sapling addresses, +/// only external-key scoped addresses should be stored in this table; for purely transparent +/// addresses, this may be an internal-scope (change) address, so that we can provide +/// compatibility with HD-derived change addresses produced by transparent-only wallets. +/// - `transparent_child_index`: the diversifier index, if it is in the range of a non-hardened +/// transparent address index. This is used for gap limit handling and is always populated if the +/// diversifier index is in that range; since the diversifier index is stored as a byte array we +/// cannot use SQL integer operations on it and thus need it as an integer as well. +/// - `cached_transparent_receiver_address`: the transparent receiver component of address (which +/// may be the same as `address` in the case of an internal-scope transparent change address or a +/// ZIP 320 interstitial address). It is cached directly in the table to make account lookups for +/// transparent outputs more efficient, enabling joins to [`TABLE_TRANSPARENT_RECEIVED_OUTPUTS`]. +/// - `exposed_at_height`: The chain tip height at the time that the address was generated by an +/// explicit request by the user or reserved for use in a ZIP 320 transaction. In the case of an +/// address with its first use discovered in a transaction obtained by scanning the chain, this +/// will be set to the mined height of that transaction. pub(super) const TABLE_ADDRESSES: &str = r#" CREATE TABLE "addresses" ( + id INTEGER NOT NULL PRIMARY KEY, account_id INTEGER NOT NULL, + key_scope INTEGER NOT NULL DEFAULT 0, diversifier_index_be BLOB NOT NULL, address TEXT NOT NULL, + transparent_child_index INTEGER, cached_transparent_receiver_address TEXT, + exposed_at_height INTEGER, FOREIGN KEY (account_id) REFERENCES accounts(id), - CONSTRAINT diversification UNIQUE (account_id, diversifier_index_be) + CONSTRAINT diversification UNIQUE (account_id, key_scope, diversifier_index_be), + CONSTRAINT transparent_index_consistency CHECK ( + (transparent_child_index IS NOT NULL) == (cached_transparent_receiver_address IS NOT NULL) + ) )"#; pub(super) const INDEX_ADDRESSES_ACCOUNTS: &str = r#" -CREATE INDEX "addresses_accounts" ON "addresses" ( - "account_id" ASC +CREATE INDEX idx_addresses_accounts ON addresses ( + account_id ASC +)"#; +pub(super) const INDEX_ADDRESSES_INDICES: &str = r#" +CREATE INDEX idx_addresses_indices ON addresses ( + diversifier_index_be ASC +)"#; +pub(super) const INDEX_ADDRESSES_T_INDICES: &str = r#" +CREATE INDEX idx_addresses_t_indices ON addresses ( + transparent_child_index ASC )"#; - -/// Stores ephemeral transparent addresses used for ZIP 320. -/// -/// For each account, these addresses are allocated sequentially by address index under scope 2 -/// (`TransparentKeyScope::EPHEMERAL`) at the "change" level of the BIP 32 address hierarchy. -/// The ephemeral addresses stored in the table are exactly the "reserved" ephemeral addresses -/// (that is addresses that have been allocated for use in a ZIP 320 transaction proposal), plus -/// the addresses at the next [`GAP_LIMIT`] indices. -/// -/// Addresses are never removed. New ones should only be reserved via the -/// `WalletWrite::reserve_next_n_ephemeral_addresses` API. All of the addresses in the table -/// should be scanned for incoming funds. -/// -/// ### Columns -/// - `address` contains the string (Base58Check) encoding of a transparent P2PKH address. -/// - `used_in_tx` indicates that the address has been used by this wallet in a transaction (which -/// has not necessarily been mined yet). This should only be set once, when the txid is known. -/// - `seen_in_tx` is non-null iff an output to the address has been seed in a transaction observed -/// on the network and passed to `store_decrypted_tx`. The transaction may have been sent by this -// wallet or another one using the same seed, or by a TEX address recipient sending back the -/// funds. This is used to advance the "gap", as well as to heuristically reduce the chance of -/// address reuse collisions with another wallet using the same seed. -/// -/// It is an external invariant that within each account: -/// - the address indices are contiguous and start from 0; -/// - the last [`GAP_LIMIT`] addresses have `used_in_tx` and `seen_in_tx` both NULL. -/// -/// All but the last [`GAP_LIMIT`] addresses are defined to be "reserved" addresses. Since the next -/// index to reserve is determined by dead reckoning from the last stored address, we use dummy -/// entries having `NULL` for the value of the `address` column after the maximum valid index in -/// order to allow the last [`GAP_LIMIT`] addresses at the end of the index range to be used. -/// -/// Note that the fact that `used_in_tx` references a specific transaction is just a debugging aid. -/// The same is mostly true of `seen_in_tx`, but we also take into account whether the referenced -/// transaction is unmined in order to determine the last index that is safe to reserve. -pub(super) const TABLE_EPHEMERAL_ADDRESSES: &str = r#" -CREATE TABLE ephemeral_addresses ( - account_id INTEGER NOT NULL, - address_index INTEGER NOT NULL, - -- nullability of this column is controlled by the index_range_and_address_nullity check - address TEXT, - used_in_tx INTEGER, - seen_in_tx INTEGER, - FOREIGN KEY (account_id) REFERENCES accounts(id), - FOREIGN KEY (used_in_tx) REFERENCES transactions(id_tx), - FOREIGN KEY (seen_in_tx) REFERENCES transactions(id_tx), - PRIMARY KEY (account_id, address_index), - CONSTRAINT ephemeral_addr_uniq UNIQUE (address), - CONSTRAINT used_implies_seen CHECK ( - used_in_tx IS NULL OR seen_in_tx IS NOT NULL - ), - CONSTRAINT index_range_and_address_nullity CHECK ( - (address_index BETWEEN 0 AND 0x7FFFFFFF AND address IS NOT NULL) OR - (address_index BETWEEN 0x80000000 AND 0x7FFFFFFF + 20 AND address IS NULL AND used_in_tx IS NULL AND seen_in_tx IS NULL) - ) -) WITHOUT ROWID"#; -// Hexadecimal integer literals were added in SQLite version 3.8.6 (2014-08-15). -// libsqlite3-sys requires at least version 3.14.0. -// "WITHOUT ROWID" tells SQLite to use a clustered index on the (composite) primary key. -const_assert_eq!(GAP_LIMIT, 20); /// Stores information about every block that the wallet has scanned. /// @@ -216,6 +186,7 @@ CREATE TABLE "sapling_received_notes" ( memo BLOB, commitment_tree_position INTEGER, recipient_key_scope INTEGER, + address_id INTEGER REFERENCES addresses(id), FOREIGN KEY (tx) REFERENCES transactions(id_tx), FOREIGN KEY (account_id) REFERENCES accounts(id), CONSTRAINT tx_output UNIQUE (tx, output_index) @@ -269,6 +240,7 @@ CREATE TABLE orchard_received_notes ( memo BLOB, commitment_tree_position INTEGER, recipient_key_scope INTEGER, + address_id INTEGER REFERENCES addresses(id), FOREIGN KEY (tx) REFERENCES transactions(id_tx), FOREIGN KEY (account_id) REFERENCES accounts(id), CONSTRAINT tx_output UNIQUE (tx, action_index) @@ -339,6 +311,7 @@ CREATE TABLE transparent_received_outputs ( script BLOB NOT NULL, value_zat INTEGER NOT NULL, max_observed_unspent_height INTEGER, + address_id INTEGER REFERENCES addresses(id), FOREIGN KEY (transaction_id) REFERENCES transactions(id_tx), FOREIGN KEY (account_id) REFERENCES accounts(id), CONSTRAINT transparent_output_unique UNIQUE (transaction_id, output_index) @@ -690,7 +663,8 @@ CREATE VIEW v_received_outputs AS sapling_received_notes.value, is_change, sapling_received_notes.memo, - sent_notes.id AS sent_note_id + sent_notes.id AS sent_note_id, + sapling_received_notes.address_id FROM sapling_received_notes LEFT JOIN sent_notes ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = @@ -705,7 +679,8 @@ UNION orchard_received_notes.value, is_change, orchard_received_notes.memo, - sent_notes.id AS sent_note_id + sent_notes.id AS sent_note_id, + orchard_received_notes.address_id FROM orchard_received_notes LEFT JOIN sent_notes ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = @@ -720,7 +695,8 @@ UNION u.value_zat AS value, 0 AS is_change, NULL AS memo, - sent_notes.id AS sent_note_id + sent_notes.id AS sent_note_id, + u.address_id FROM transparent_received_outputs u LEFT JOIN sent_notes ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = @@ -1072,3 +1048,37 @@ GROUP BY subtree_start_height, subtree_end_height, contains_marked"; + +pub(super) const VIEW_ADDRESS_USES: &str = " +CREATE VIEW v_address_uses AS + SELECT orn.address_id, orn.account_id, orn.tx AS transaction_id, t.mined_height, + a.key_scope, a.diversifier_index_be, a.transparent_child_index + FROM orchard_received_notes orn + JOIN addresses a ON a.id = orn.address_id + JOIN transactions t ON t.id_tx = orn.tx +UNION + SELECT srn.address_id, srn.account_id, srn.tx AS transaction_id, t.mined_height, + a.key_scope, a.diversifier_index_be, a.transparent_child_index + FROM sapling_received_notes srn + JOIN addresses a ON a.id = srn.address_id + JOIN transactions t ON t.id_tx = srn.tx +UNION + SELECT tro.address_id, tro.account_id, tro.transaction_id, t.mined_height, + a.key_scope, a.diversifier_index_be, a.transparent_child_index + FROM transparent_received_outputs tro + JOIN addresses a ON a.id = tro.address_id + JOIN transactions t ON t.id_tx = tro.transaction_id"; + +pub(super) const VIEW_ADDRESS_FIRST_USE: &str = " + CREATE VIEW v_address_first_use AS + SELECT + address_id, + account_id, + key_scope, + diversifier_index_be, + transparent_child_index, + MIN(mined_height) AS first_use_height + FROM v_address_uses + GROUP BY + address_id, account_id, key_scope, + diversifier_index_be, transparent_child_index"; diff --git a/zcash_client_sqlite/src/wallet/init.rs b/zcash_client_sqlite/src/wallet/init.rs index d9d25c2b93..77ab733482 100644 --- a/zcash_client_sqlite/src/wallet/init.rs +++ b/zcash_client_sqlite/src/wallet/init.rs @@ -220,12 +220,11 @@ fn sqlite_client_error_to_wallet_migration_error(e: SqliteClientError) -> Wallet unreachable!("we don't call methods that require a known chain height") } #[cfg(feature = "transparent-inputs")] - SqliteClientError::ReachedGapLimit(_, _) => { + SqliteClientError::ReachedGapLimit(_) => { unreachable!("we don't do ephemeral address tracking") } - #[cfg(feature = "transparent-inputs")] - SqliteClientError::EphemeralAddressReuse(_, _) => { - unreachable!("we don't do ephemeral address tracking") + SqliteClientError::AddressReuse(_, _) => { + unreachable!("we don't create transactions in migrations") } SqliteClientError::NoteFilterInvalid(_) => { unreachable!("we don't do note selection in migrations") @@ -487,7 +486,6 @@ mod tests { db::TABLE_ACCOUNTS, db::TABLE_ADDRESSES, db::TABLE_BLOCKS, - db::TABLE_EPHEMERAL_ADDRESSES, db::TABLE_NULLIFIER_MAP, db::TABLE_ORCHARD_RECEIVED_NOTE_SPENDS, db::TABLE_ORCHARD_RECEIVED_NOTES, @@ -529,6 +527,8 @@ mod tests { db::INDEX_ACCOUNTS_UUID, db::INDEX_HD_ACCOUNT, db::INDEX_ADDRESSES_ACCOUNTS, + db::INDEX_ADDRESSES_INDICES, + db::INDEX_ADDRESSES_T_INDICES, db::INDEX_NF_MAP_LOCATOR_IDX, db::INDEX_ORCHARD_RECEIVED_NOTES_ACCOUNT, db::INDEX_ORCHARD_RECEIVED_NOTES_TX, @@ -557,6 +557,8 @@ mod tests { } let expected_views = vec![ + db::VIEW_ADDRESS_FIRST_USE.to_owned(), + db::VIEW_ADDRESS_USES.to_owned(), db::view_orchard_shard_scan_ranges(st.network()), db::view_orchard_shard_unscanned_ranges(), db::VIEW_ORCHARD_SHARDS_SCAN_STATE.to_owned(), @@ -1079,6 +1081,11 @@ mod tests { let (account_id, _usk) = db_data .create_account("", &Secret::new(seed.to_vec()), &birthday, None) .unwrap(); + + // We have to have the chain tip height in order to allocate new addresses, to record the + // exposed-at height. + db_data.update_chain_tip(birthday.height()).unwrap(); + assert_matches!( db_data.get_account(account_id), Ok(Some(account)) if matches!( diff --git a/zcash_client_sqlite/src/wallet/init/migrations.rs b/zcash_client_sqlite/src/wallet/init/migrations.rs index 1e076736ed..8aa056ced2 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations.rs @@ -19,6 +19,7 @@ mod sent_notes_to_internal; mod shardtree_support; mod spend_key_available; mod support_legacy_sqlite; +mod transparent_gap_limit_handling; mod tx_retrieval_queue; mod ufvk_support; mod utxos_table; @@ -84,8 +85,8 @@ pub(super) fn all_migrations( // support_legacy_sqlite // / \ // fix_broken_commitment_trees add_account_uuids - // | - // fix_bad_change_flagging + // | | + // fix_bad_change_flagging transparent_gap_limit_handling vec![ Box::new(initial_setup::Migration {}), Box::new(utxos_table::Migration {}), @@ -149,6 +150,9 @@ pub(super) fn all_migrations( }), Box::new(fix_bad_change_flagging::Migration), Box::new(add_account_uuids::Migration), + Box::new(transparent_gap_limit_handling::Migration { + params: params.clone(), + }), ] } diff --git a/zcash_client_sqlite/src/wallet/init/migrations/add_utxo_account.rs b/zcash_client_sqlite/src/wallet/init/migrations/add_utxo_account.rs index 6c737247d7..213ec6e6e8 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/add_utxo_account.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/add_utxo_account.rs @@ -19,7 +19,7 @@ use { std::collections::HashMap, zcash_client_backend::wallet::TransparentAddressMetadata, zcash_keys::{address::Address, encoding::AddressCodec, keys::UnifiedFullViewingKey}, - zip32::{AccountId, DiversifierIndex, Scope}, + zip32::{AccountId, Scope}, }; /// This migration adds an account identifier column to the UTXOs table. @@ -132,6 +132,8 @@ fn get_transparent_receivers( params: &P, account: AccountId, ) -> Result>, SqliteClientError> { + use crate::wallet::decode_diversifier_index_be; + let mut ret: HashMap> = HashMap::new(); // Get all UAs derived @@ -141,11 +143,7 @@ fn get_transparent_receivers( while let Some(row) = rows.next()? { let ua_str: String = row.get(0)?; - let di_vec: Vec = row.get(1)?; - let mut di: [u8; 11] = di_vec.try_into().map_err(|_| { - SqliteClientError::CorruptedData("Diversifier index is not an 11-byte value".to_owned()) - })?; - di.reverse(); // BE -> LE conversion + let di = decode_diversifier_index_be(&row.get::<_, Vec>(1)?)?; let ua = Address::decode(params, &ua_str) .ok_or_else(|| { @@ -160,13 +158,11 @@ fn get_transparent_receivers( })?; if let Some(taddr) = ua.transparent() { - let index = NonHardenedChildIndex::from_index( - DiversifierIndex::from(di).try_into().map_err(|_| { - SqliteClientError::CorruptedData( - "Unable to get diversifier for transparent address.".to_owned(), - ) - })?, - ) + let index = NonHardenedChildIndex::from_index(u32::try_from(di).map_err(|_| { + SqliteClientError::CorruptedData( + "Unable to get diversifier for transparent address.".to_owned(), + ) + })?) .ok_or_else(|| { SqliteClientError::CorruptedData( "Unexpected hardened index for transparent address.".to_owned(), diff --git a/zcash_client_sqlite/src/wallet/init/migrations/ephemeral_addresses.rs b/zcash_client_sqlite/src/wallet/init/migrations/ephemeral_addresses.rs index 109f307fff..c752184d45 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/ephemeral_addresses.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/ephemeral_addresses.rs @@ -7,13 +7,25 @@ use zcash_protocol::consensus; use crate::wallet::init::WalletMigrationError; -#[cfg(feature = "transparent-inputs")] -use crate::{wallet::transparent::ephemeral, AccountRef}; - use super::utxos_to_txos; +#[cfg(feature = "transparent-inputs")] +use { + crate::{error::SqliteClientError, AccountRef}, + rusqlite::named_params, + transparent::keys::NonHardenedChildIndex, + zcash_keys::{ + encoding::AddressCodec, + keys::{AddressGenerationError, UnifiedFullViewingKey}, + }, + zip32::DiversifierIndex, +}; + pub(super) const MIGRATION_ID: Uuid = Uuid::from_u128(0x0e1d4274_1f8e_44e2_909d_689a4bc2967b); +#[cfg(feature = "transparent-inputs")] +const EPHEMERAL_GAP_LIMIT: u32 = 5; + const DEPENDENCIES: &[Uuid] = &[utxos_to_txos::MIGRATION_ID]; #[allow(dead_code)] @@ -35,6 +47,60 @@ impl

schemerz::Migration for Migration

{ } } +#[cfg(feature = "transparent-inputs")] +fn init_accounts( + transaction: &rusqlite::Transaction, + params: &P, +) -> Result<(), SqliteClientError> { + let mut stmt = transaction.prepare("SELECT id, ufvk FROM accounts")?; + let mut rows = stmt.query([])?; + while let Some(row) = rows.next()? { + let account_id = AccountRef(row.get(0)?); + let ufvk_str: Option = row.get(1)?; + if let Some(ufvk_str) = ufvk_str { + if let Some(tfvk) = UnifiedFullViewingKey::decode(params, &ufvk_str) + .map_err(SqliteClientError::CorruptedData)? + .transparent() + { + let ephemeral_ivk = tfvk.derive_ephemeral_ivk().map_err(|_| { + SqliteClientError::CorruptedData( + "Unexpected failure to derive ephemeral transparent IVK".to_owned(), + ) + })?; + + let mut ea_insert = transaction.prepare( + "INSERT INTO ephemeral_addresses (account_id, address_index, address) + VALUES (:account_id, :address_index, :address)", + )?; + + // NB: we have reduced the initial space of generated ephemeral addresses + // from 20 addresses to 5, as ephemeral addresses should always be used in + // a transaction immediatly after being reserved, and as a consequence + // there is no significant benefit in having a larger gap limit. + for i in 0..EPHEMERAL_GAP_LIMIT { + let address = ephemeral_ivk + .derive_ephemeral_address( + NonHardenedChildIndex::from_index(i).expect("index is valid"), + ) + .map_err(|_| { + AddressGenerationError::InvalidTransparentChildIndex( + DiversifierIndex::from(i), + ) + })?; + + ea_insert.execute(named_params! { + ":account_id": account_id.0, + ":address_index": i, + ":address": address.encode(params) + })?; + } + } + } + } + + Ok(()) +} + impl RusqliteMigration for Migration

{ type Error = WalletMigrationError; @@ -62,17 +128,13 @@ impl RusqliteMigration for Migration

{ ) WITHOUT ROWID;" )?; - // Make sure that at least `GAP_LIMIT` ephemeral transparent addresses are + // Make sure that at least `EPHEMERAL_GAP_LIMIT` ephemeral transparent addresses are // stored in each account. #[cfg(feature = "transparent-inputs")] { - let mut stmt = transaction.prepare("SELECT id FROM accounts")?; - let mut rows = stmt.query([])?; - while let Some(row) = rows.next()? { - let account_id = AccountRef(row.get(0)?); - ephemeral::init_account(transaction, &self.params, account_id)?; - } + init_accounts(transaction, &self.params)?; } + Ok(()) } diff --git a/zcash_client_sqlite/src/wallet/init/migrations/fix_bad_change_flagging.rs b/zcash_client_sqlite/src/wallet/init/migrations/fix_bad_change_flagging.rs index 38b1d49ae9..e25e0711e5 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/fix_bad_change_flagging.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/fix_bad_change_flagging.rs @@ -5,12 +5,11 @@ use std::collections::HashSet; use rusqlite::named_params; use schemerz_rusqlite::RusqliteMigration; use uuid::Uuid; -use zip32::Scope; use crate::{ wallet::{ init::{migrations::fix_broken_commitment_trees, WalletMigrationError}, - scope_code, + KeyScope, }, SAPLING_TABLES_PREFIX, }; @@ -52,7 +51,7 @@ impl RusqliteMigration for Migration { AND sn.from_account_id = {table_prefix}_received_notes.account_id AND {table_prefix}_received_notes.recipient_key_scope = :internal_scope" ), - named_params! {":internal_scope": scope_code(Scope::Internal)}, + named_params! {":internal_scope": KeyScope::INTERNAL.encode()}, ) }; diff --git a/zcash_client_sqlite/src/wallet/init/migrations/receiving_key_scopes.rs b/zcash_client_sqlite/src/wallet/init/migrations/receiving_key_scopes.rs index 1e71ac5012..d9ce72d67b 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/receiving_key_scopes.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/receiving_key_scopes.rs @@ -29,7 +29,7 @@ use crate::{ chain_tip_height, commitment_tree::SqliteShardStore, init::{migrations::shardtree_support, WalletMigrationError}, - scope_code, + KeyScope, }, PRUNING_DEPTH, SAPLING_TABLES_PREFIX, }; @@ -109,7 +109,7 @@ impl RusqliteMigration for Migration

{ transaction.execute_batch( &format!( "ALTER TABLE sapling_received_notes ADD COLUMN recipient_key_scope INTEGER NOT NULL DEFAULT {};", - scope_code(Scope::External) + KeyScope::EXTERNAL.encode() ) )?; @@ -204,7 +204,7 @@ impl RusqliteMigration for Migration

{ transaction.execute( "UPDATE sapling_received_notes SET recipient_key_scope = :scope WHERE id_note = :note_id", - named_params! {":scope": scope_code(Scope::Internal), ":note_id": note_id}, + named_params! {":scope": KeyScope::INTERNAL.encode(), ":note_id": note_id}, )?; } } else { @@ -261,7 +261,7 @@ impl RusqliteMigration for Migration

{ transaction.execute( "UPDATE sapling_received_notes SET recipient_key_scope = :scope WHERE id_note = :note_id", - named_params! {":scope": scope_code(Scope::Internal), ":note_id": note_id}, + named_params! {":scope": KeyScope::INTERNAL.encode(), ":note_id": note_id}, )?; } } @@ -326,8 +326,9 @@ mod tests { init_wallet_db_internal, migrations::{add_account_birthdays, shardtree_support, wallet_summaries}, }, - memo_repr, parse_scope, + memo_repr, sapling::ReceivedSaplingOutput, + KeyScope, }, AccountRef, TxRef, WalletDb, }; @@ -604,10 +605,10 @@ mod tests { while let Some(row) = rows.next().unwrap() { row_count += 1; let value: u64 = row.get(0).unwrap(); - let scope = parse_scope(row.get(1).unwrap()); + let scope = KeyScope::decode(row.get(1).unwrap()).unwrap(); match value { - EXTERNAL_VALUE => assert_eq!(scope, Some(Scope::External)), - INTERNAL_VALUE => assert_eq!(scope, Some(Scope::Internal)), + EXTERNAL_VALUE => assert_eq!(scope, KeyScope::EXTERNAL), + INTERNAL_VALUE => assert_eq!(scope, KeyScope::INTERNAL), _ => { panic!( "(Value, Scope) pair {:?} is not expected to exist in the wallet.", @@ -782,10 +783,10 @@ mod tests { while let Some(row) = rows.next().unwrap() { row_count += 1; let value: u64 = row.get(0).unwrap(); - let scope = parse_scope(row.get(1).unwrap()); + let scope = KeyScope::decode(row.get(1).unwrap()).unwrap(); match value { - EXTERNAL_VALUE => assert_eq!(scope, Some(Scope::External)), - INTERNAL_VALUE => assert_eq!(scope, Some(Scope::Internal)), + EXTERNAL_VALUE => assert_eq!(scope, KeyScope::EXTERNAL), + INTERNAL_VALUE => assert_eq!(scope, KeyScope::INTERNAL), _ => { panic!( "(Value, Scope) pair {:?} is not expected to exist in the wallet.", diff --git a/zcash_client_sqlite/src/wallet/init/migrations/transparent_gap_limit_handling.rs b/zcash_client_sqlite/src/wallet/init/migrations/transparent_gap_limit_handling.rs new file mode 100644 index 0000000000..5ce34db016 --- /dev/null +++ b/zcash_client_sqlite/src/wallet/init/migrations/transparent_gap_limit_handling.rs @@ -0,0 +1,493 @@ +//! Add support for general transparent gap limit handling, and unify the `addresses` and +//! `ephemeral_addresses` tables. + +use std::collections::HashSet; +use uuid::Uuid; + +use rusqlite::{named_params, Transaction}; +use schemerz_rusqlite::RusqliteMigration; + +use zcash_keys::keys::UnifiedIncomingViewingKey; +use zcash_protocol::consensus::{self, BlockHeight}; + +use super::add_account_uuids; +use crate::{ + wallet::{self, init::WalletMigrationError, KeyScope}, + AccountRef, +}; + +#[cfg(feature = "transparent-inputs")] +use { + crate::wallet::{decode_diversifier_index_be, encode_diversifier_index_be}, + ::transparent::keys::{IncomingViewingKey as _, NonHardenedChildIndex}, + zcash_keys::encoding::AddressCodec as _, + zip32::DiversifierIndex, +}; + +pub(super) const MIGRATION_ID: Uuid = Uuid::from_u128(0xc41dfc0e_e870_4859_be47_d2f572f5ca73); + +const DEPENDENCIES: &[Uuid] = &[add_account_uuids::MIGRATION_ID]; + +pub(super) struct Migration

{ + pub(super) params: P, +} + +impl

schemerz::Migration for Migration

{ + fn id(&self) -> Uuid { + MIGRATION_ID + } + + fn dependencies(&self) -> HashSet { + DEPENDENCIES.iter().copied().collect() + } + + fn description(&self) -> &'static str { + "Add support for general transparent gap limit handling, unifying the `addresses` and `ephemeral_addresses` tables." + } +} + +impl RusqliteMigration for Migration

{ + type Error = WalletMigrationError; + + fn up(&self, conn: &Transaction) -> Result<(), WalletMigrationError> { + let decode_uivk = |uivk_str: String| { + UnifiedIncomingViewingKey::decode(&self.params, &uivk_str).map_err(|e| { + WalletMigrationError::CorruptedData(format!( + "Invalid UIVK encoding {}: {}", + uivk_str, e + )) + }) + }; + + let external_scope_code = KeyScope::EXTERNAL.encode(); + + conn.execute_batch(&format!( + r#" + ALTER TABLE addresses ADD COLUMN key_scope INTEGER NOT NULL DEFAULT {external_scope_code}; + ALTER TABLE addresses ADD COLUMN transparent_child_index INTEGER; + "# + ))?; + + #[cfg(feature = "transparent-inputs")] + { + // If the diversifier index is in the valid range of non-hardened child indices, set + // `transparent_child_index` so that we can use it for gap limit handling. + // No `DISTINCT` is necessary here due to the preexisting UNIQUE(account_id, + // diversifier_index_be) constraint. + let mut di_query = conn.prepare( + r#" + SELECT account_id, accounts.uivk AS uivk, diversifier_index_be + FROM addresses + JOIN accounts ON accounts.id = account_id + "#, + )?; + let mut rows = di_query.query([])?; + while let Some(row) = rows.next()? { + let account_id: i64 = row.get("account_id")?; + let uivk = decode_uivk(row.get("uivk")?)?; + let di_be: Vec = row.get("diversifier_index_be")?; + let diversifier_index = decode_diversifier_index_be(&di_be)?; + + let transparent_external = NonHardenedChildIndex::try_from(diversifier_index) + .ok() + .and_then(|idx| { + uivk.transparent() + .as_ref() + .and_then(|external_ivk| external_ivk.derive_address(idx).ok()) + .map(|t_addr| (idx, t_addr)) + }); + + // Add transparent address index metadata and the transparent address corresponding + // to the index to the addresses table. We unconditionally set the cached + // transparent receiver address in order to simplify gap limit handling; even if a + // unified address is generated without a transparent receiver, we still assume + // that a transparent-only wallet for which we have imported the seed may have + // generated an address at that index. + if let Some((idx, t_addr)) = transparent_external { + conn.execute( + r#" + UPDATE addresses + SET transparent_child_index = :transparent_child_index, + cached_transparent_receiver_address = :t_addr + WHERE account_id = :account_id + AND diversifier_index_be = :diversifier_index_be + AND key_scope = :external_scope_code + "#, + named_params! { + ":account_id": account_id, + ":diversifier_index_be": &di_be[..], + ":external_scope_code": external_scope_code, + ":transparent_child_index": idx.index(), + ":t_addr": t_addr.encode(&self.params), + }, + )?; + } + } + } + + // We now have to re-create the `addresses` table in order to fix the constraints. + // Note that we do not include `used_in_tx` or `seen_in_tx` columns as these are + // duplicative of information that can be discovered via joins with the various + // `*_received_{notes|outputs}` tables, which we will create a view to perform below. + conn.execute_batch(&format!( + r#" + CREATE TABLE addresses_new ( + id INTEGER NOT NULL PRIMARY KEY, + account_id INTEGER NOT NULL, + key_scope INTEGER NOT NULL DEFAULT {external_scope_code}, + diversifier_index_be BLOB NOT NULL, + address TEXT NOT NULL, + transparent_child_index INTEGER, + cached_transparent_receiver_address TEXT, + exposed_at_height INTEGER, + FOREIGN KEY (account_id) REFERENCES accounts(id), + CONSTRAINT diversification UNIQUE (account_id, key_scope, diversifier_index_be), + CONSTRAINT transparent_index_consistency CHECK ( + (transparent_child_index IS NOT NULL) == (cached_transparent_receiver_address IS NOT NULL) + ) + ); + + INSERT INTO addresses_new ( + account_id, key_scope, diversifier_index_be, address, + transparent_child_index, cached_transparent_receiver_address + ) + SELECT + account_id, key_scope, diversifier_index_be, address, + transparent_child_index, cached_transparent_receiver_address + FROM addresses; + "# + ))?; + + // Now, we add the ephemeral addresses to the newly unified `addresses` table. + #[cfg(feature = "transparent-inputs")] + { + let mut ea_insert = conn.prepare( + r#" + INSERT INTO addresses_new ( + account_id, key_scope, diversifier_index_be, address, + transparent_child_index, cached_transparent_receiver_address + ) VALUES ( + :account_id, :key_scope, :diversifier_index_be, :address, + :transparent_child_index, :cached_transparent_receiver_address + ) + "#, + )?; + + let mut ea_query = conn.prepare( + r#" + SELECT account_id, address_index, address + FROM ephemeral_addresses + "#, + )?; + let mut rows = ea_query.query([])?; + while let Some(row) = rows.next()? { + let account_id: i64 = row.get("account_id")?; + let transparent_child_index = row.get::<_, i64>("address_index")?; + let diversifier_index = DiversifierIndex::from( + u32::try_from(transparent_child_index).map_err(|_| { + WalletMigrationError::CorruptedData( + "ephermeral address indices must be in the range of `u32`".to_owned(), + ) + })?, + ); + let address: String = row.get("address")?; + + // We set both the `address` column and the `cached_transparent_receiver_address` + // column to the same value here; there is no Unified address that corresponds to + // this transparent address. + ea_insert.execute(named_params! { + ":account_id": account_id, + ":key_scope": KeyScope::Ephemeral.encode(), + ":diversifier_index_be": encode_diversifier_index_be(diversifier_index), + ":address": address, + ":transparent_child_index": transparent_child_index, + ":cached_transparent_receiver_address": address + })?; + } + } + + conn.execute_batch( + r#" + PRAGMA legacy_alter_table = ON; + + DROP TABLE addresses; + ALTER TABLE addresses_new RENAME TO addresses; + CREATE INDEX idx_addresses_accounts ON addresses ( + account_id ASC + ); + CREATE INDEX idx_addresses_indices ON addresses ( + diversifier_index_be ASC + ); + CREATE INDEX idx_addresses_t_indices ON addresses ( + transparent_child_index ASC + ); + + DROP TABLE ephemeral_addresses; + + PRAGMA legacy_alter_table = OFF; + "#, + )?; + + // Add foreign key references from the *_received_{notes|outputs} tables to the addresses + // table to make it possible to identify which address was involved. These foreign key + // columns must be nullable as for shielded account-internal. Ideally the foreign key + // relationship between `transparent_received_outputs` and `addresses` would not be + // nullable, but we allow it to be so here in order to avoid having to re-create that + // table. + // + // While it would be possible to only add the address reference to + // `transparent_received_outputs`, that would mean that a note received at a shielded + // component of a diversified Unified Address would not update the position of the + // transparent "address gap". Since we will include shielded address indices in the gap + // computation, transparent-only wallets may not be able to discover all transparent funds, + // but users of shielded wallets will be guaranteed to be able to recover all of their + // funds. + conn.execute_batch( + r#" + ALTER TABLE orchard_received_notes + ADD COLUMN address_id INTEGER REFERENCES addresses(id); + ALTER TABLE sapling_received_notes + ADD COLUMN address_id INTEGER REFERENCES addresses(id); + ALTER TABLE transparent_received_outputs + ADD COLUMN address_id INTEGER REFERENCES addresses(id); + "#, + )?; + + // Ensure that an address exists for each received Orchard note, and populate the + // `address_id` column. + #[cfg(feature = "orchard")] + { + let mut stmt_rn_diversifiers = conn.prepare( + r#" + SELECT orn.id, orn.account_id, accounts.uivk, + orn.recipient_key_scope, orn.diversifier, t.mined_height + FROM orchard_received_notes orn + JOIN accounts ON accounts.id = account_id + JOIN transactions t on t.id_tx = orn.tx + "#, + )?; + + let mut rows = stmt_rn_diversifiers.query([])?; + while let Some(row) = rows.next()? { + let scope = KeyScope::decode(row.get("recipient_key_scope")?)?; + // for Orchard and Sapling, we only store addresses for externally-scoped keys. + if scope == KeyScope::EXTERNAL { + let row_id: i64 = row.get("id")?; + let account_id = AccountRef(row.get("account_id")?); + let mined_height = row + .get::<_, Option>("mined_height")? + .map(BlockHeight::from); + + let uivk = decode_uivk(row.get("uivk")?)?; + let diversifier = + orchard::keys::Diversifier::from_bytes(row.get("diversifier")?); + + // TODO: It's annoying that `IncomingViewingKey` doesn't expose the ability to + // decrypt the diversifier to find the index directly, and doesn't provide an + // accessor for `dk`. We already know we have the right IVK. + let ivk = uivk + .orchard() + .as_ref() + .expect("previously received an Orchard output"); + let di = ivk + .diversifier_index(&ivk.address(diversifier)) + .expect("roundtrip"); + let ua = uivk.address(di, None)?; + let address_id = wallet::upsert_address( + conn, + &self.params, + account_id, + di, + &ua, + mined_height, + )?; + + conn.execute( + "UPDATE orchard_received_notes + SET address_id = :address_id + WHERE id = :row_id", + named_params! { + ":address_id": address_id.0, + ":row_id": row_id + }, + )?; + } + } + } + + // Ensure that an address exists for each received Sapling note, and populate the + // `address_id` column. + { + let mut stmt_rn_diversifiers = conn.prepare( + r#" + SELECT srn.id, srn.account_id, accounts.uivk, + srn.recipient_key_scope, srn.diversifier, t.mined_height + FROM sapling_received_notes srn + JOIN accounts ON accounts.id = account_id + JOIN transactions t ON t.id_tx = srn.tx + "#, + )?; + + let mut rows = stmt_rn_diversifiers.query([])?; + while let Some(row) = rows.next()? { + let scope = KeyScope::decode(row.get("recipient_key_scope")?)?; + // for Orchard and Sapling, we only store addresses for externally-scoped keys. + if scope == KeyScope::EXTERNAL { + let row_id: i64 = row.get("id")?; + let account_id = AccountRef(row.get("account_id")?); + let mined_height = row + .get::<_, Option>("mined_height")? + .map(BlockHeight::from); + + let uivk = decode_uivk(row.get("uivk")?)?; + let diversifier = sapling::Diversifier(row.get("diversifier")?); + + // TODO: It's annoying that `IncomingViewingKey` doesn't expose the ability to + // decrypt the diversifier to find the index directly, and doesn't provide an + // accessor for `dk`. We already know we have the right IVK. + let ivk = uivk + .sapling() + .as_ref() + .expect("previously received a Sapling output"); + let di = ivk + .decrypt_diversifier( + &ivk.address(diversifier) + .expect("previously generated an address"), + ) + .expect("roundtrip"); + let ua = uivk.address(di, None)?; + let address_id = wallet::upsert_address( + conn, + &self.params, + account_id, + di, + &ua, + mined_height, + )?; + + conn.execute( + "UPDATE sapling_received_notes + SET address_id = :address_id + WHERE id = :row_id", + named_params! { + ":address_id": address_id.0, + ":row_id": row_id + }, + )?; + } + } + } + + // At this point, every address on which we've received a transparent output should have a + // corresponding row in the `addresses` table with a valid + // `cached_transparent_receiver_address` entry, because we will only have queried the light + // wallet server for outputs from exactly these addresses. So for transparent outputs, we + // join to the addresses table using the address itself in order to obtain the address index. + #[cfg(feature = "transparent-inputs")] + { + conn.execute( + r#" + UPDATE transparent_received_outputs + SET address_id = addresses.id + FROM addresses + WHERE addresses.cached_transparent_receiver_address = transparent_received_outputs.address + "#, + [] + )?; + } + + // Construct a view that identifies the minimum block height at which each address was + // first used + conn.execute_batch( + r#" + CREATE VIEW v_address_uses AS + SELECT orn.address_id, orn.account_id, orn.tx AS transaction_id, t.mined_height, + a.key_scope, a.diversifier_index_be, a.transparent_child_index + FROM orchard_received_notes orn + JOIN addresses a ON a.id = orn.address_id + JOIN transactions t ON t.id_tx = orn.tx + UNION + SELECT srn.address_id, srn.account_id, srn.tx AS transaction_id, t.mined_height, + a.key_scope, a.diversifier_index_be, a.transparent_child_index + FROM sapling_received_notes srn + JOIN addresses a ON a.id = srn.address_id + JOIN transactions t ON t.id_tx = srn.tx + UNION + SELECT tro.address_id, tro.account_id, tro.transaction_id, t.mined_height, + a.key_scope, a.diversifier_index_be, a.transparent_child_index + FROM transparent_received_outputs tro + JOIN addresses a ON a.id = tro.address_id + JOIN transactions t ON t.id_tx = tro.transaction_id; + + CREATE VIEW v_address_first_use AS + SELECT + address_id, + account_id, + key_scope, + diversifier_index_be, + transparent_child_index, + MIN(mined_height) AS first_use_height + FROM v_address_uses + GROUP BY + address_id, account_id, key_scope, + diversifier_index_be, transparent_child_index; + + DROP VIEW v_received_outputs; + CREATE VIEW v_received_outputs AS + SELECT + sapling_received_notes.id AS id_within_pool_table, + sapling_received_notes.tx AS transaction_id, + 2 AS pool, + sapling_received_notes.output_index, + account_id, + sapling_received_notes.value, + is_change, + sapling_received_notes.memo, + sent_notes.id AS sent_note_id, + sapling_received_notes.address_id + FROM sapling_received_notes + LEFT JOIN sent_notes + ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = + (sapling_received_notes.tx, 2, sapling_received_notes.output_index) + UNION + SELECT + orchard_received_notes.id AS id_within_pool_table, + orchard_received_notes.tx AS transaction_id, + 3 AS pool, + orchard_received_notes.action_index AS output_index, + account_id, + orchard_received_notes.value, + is_change, + orchard_received_notes.memo, + sent_notes.id AS sent_note_id, + orchard_received_notes.address_id + FROM orchard_received_notes + LEFT JOIN sent_notes + ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = + (orchard_received_notes.tx, 3, orchard_received_notes.action_index) + UNION + SELECT + u.id AS id_within_pool_table, + u.transaction_id, + 0 AS pool, + u.output_index, + u.account_id, + u.value_zat AS value, + 0 AS is_change, + NULL AS memo, + sent_notes.id AS sent_note_id, + u.address_id + FROM transparent_received_outputs u + LEFT JOIN sent_notes + ON (sent_notes.tx, sent_notes.output_pool, sent_notes.output_index) = + (u.transaction_id, 0, u.output_index); + "#, + )?; + + Ok(()) + } + + fn down(&self, _: &Transaction) -> Result<(), WalletMigrationError> { + Err(WalletMigrationError::CannotRevert(MIGRATION_ID)) + } +} diff --git a/zcash_client_sqlite/src/wallet/orchard.rs b/zcash_client_sqlite/src/wallet/orchard.rs index 41f9572020..57605a9480 100644 --- a/zcash_client_sqlite/src/wallet/orchard.rs +++ b/zcash_client_sqlite/src/wallet/orchard.rs @@ -5,14 +5,14 @@ use orchard::{ keys::Diversifier, note::{Note, Nullifier, RandomSeed, Rho}, }; -use rusqlite::{named_params, types::Value, Connection, Row, Transaction}; +use rusqlite::{named_params, types::Value, Connection, Row}; use zcash_client_backend::{ - data_api::NullifierQuery, + data_api::{Account as _, NullifierQuery}, wallet::{ReceivedNote, WalletOrchardOutput}, DecryptedOutput, TransferType, }; -use zcash_keys::keys::UnifiedFullViewingKey; +use zcash_keys::keys::{UnifiedAddressRequest, UnifiedFullViewingKey}; use zcash_primitives::transaction::TxId; use zcash_protocol::{ consensus::{self, BlockHeight}, @@ -22,9 +22,9 @@ use zcash_protocol::{ }; use zip32::Scope; -use crate::{error::SqliteClientError, AccountUuid, ReceivedNoteId, TxRef}; +use crate::{error::SqliteClientError, AccountRef, AccountUuid, AddressRef, ReceivedNoteId, TxRef}; -use super::{get_account_ref, memo_repr, parse_scope, scope_code}; +use super::{get_account, get_account_ref, memo_repr, upsert_address, KeyScope}; /// This trait provides a generalization over shielded output representations. pub(crate) trait ReceivedOrchardOutput { @@ -160,9 +160,14 @@ fn to_spendable_note( let ufvk = UnifiedFullViewingKey::decode(params, &ufvk_str) .map_err(SqliteClientError::CorruptedData)?; - let spending_key_scope = parse_scope(scope_code).ok_or_else(|| { - SqliteClientError::CorruptedData(format!("Invalid key scope code {}", scope_code)) - })?; + let spending_key_scope = zip32::Scope::try_from(KeyScope::decode(scope_code)?) + .map_err(|_| { + SqliteClientError::CorruptedData(format!( + "Invalid key scope code {}", + scope_code + )) + })?; + let recipient = ufvk .orchard() .map(|fvk| fvk.to_ivk(spending_key_scope).address(diversifier)) @@ -226,34 +231,82 @@ pub(crate) fn select_spendable_orchard_notes( ) } +pub(crate) fn ensure_address< + T: ReceivedOrchardOutput, + P: consensus::Parameters, +>( + conn: &rusqlite::Transaction, + params: &P, + output: &T, + exposure_height: Option, +) -> Result, SqliteClientError> { + if output.recipient_key_scope() != Some(Scope::Internal) { + let account = get_account(conn, params, output.account_id())? + .ok_or(SqliteClientError::AccountUnknown)?; + + let uivk = account.uivk(); + let ivk = uivk + .orchard() + .as_ref() + .expect("uivk decrypted this output."); + let to = output.note().recipient(); + let diversifier_index = ivk + .diversifier_index(&to) + .expect("address corresponds to account"); + + let ua = account + .uivk() + .address(diversifier_index, Some(UnifiedAddressRequest::ALLOW_ALL))?; + upsert_address( + conn, + params, + account.internal_id(), + diversifier_index, + &ua, + exposure_height, + ) + .map(Some) + } else { + Ok(None) + } +} + /// Records the specified shielded output as having been received. /// /// This implementation relies on the facts that: /// - A transaction will not contain more than 2^63 shielded outputs. /// - A note value will never exceed 2^63 zatoshis. -pub(crate) fn put_received_note>( - conn: &Transaction, +/// +/// Returns the internal account identifier of the account that received the output. +pub(crate) fn put_received_note< + T: ReceivedOrchardOutput, + P: consensus::Parameters, +>( + conn: &rusqlite::Transaction, + params: &P, output: &T, tx_ref: TxRef, + target_or_mined_height: Option, spent_in: Option, -) -> Result<(), SqliteClientError> { +) -> Result { let account_id = get_account_ref(conn, output.account_id())?; + let address_id = ensure_address(conn, params, output, target_or_mined_height)?; let mut stmt_upsert_received_note = conn.prepare_cached( - "INSERT INTO orchard_received_notes - ( - tx, action_index, account_id, + "INSERT INTO orchard_received_notes ( + tx, action_index, account_id, address_id, diversifier, value, rho, rseed, memo, nf, is_change, commitment_tree_position, recipient_key_scope ) VALUES ( - :tx, :action_index, :account_id, + :tx, :action_index, :account_id, :address_id, :diversifier, :value, :rho, :rseed, :memo, :nf, :is_change, :commitment_tree_position, :recipient_key_scope ) ON CONFLICT (tx, action_index) DO UPDATE SET account_id = :account_id, + address_id = :address_id, diversifier = :diversifier, value = :value, rho = :rho, @@ -274,6 +327,7 @@ pub(crate) fn put_received_note( let ufvk = UnifiedFullViewingKey::decode(params, &ufvk_str) .map_err(SqliteClientError::CorruptedData)?; - let spending_key_scope = parse_scope(scope_code).ok_or_else(|| { - SqliteClientError::CorruptedData(format!("Invalid key scope code {}", scope_code)) - })?; + let spending_key_scope = zip32::Scope::try_from(KeyScope::decode(scope_code)?) + .map_err(|_| { + SqliteClientError::CorruptedData(format!( + "Invalid key scope code {}", + scope_code + )) + })?; let recipient = match spending_key_scope { Scope::Internal => ufvk @@ -330,27 +333,79 @@ pub(crate) fn mark_sapling_note_spent( } } +pub(crate) fn ensure_address< + T: ReceivedSaplingOutput, + P: consensus::Parameters, +>( + conn: &rusqlite::Transaction, + params: &P, + output: &T, + exposure_height: Option, +) -> Result, SqliteClientError> { + if output.recipient_key_scope() != Some(Scope::Internal) { + let account = get_account(conn, params, output.account_id())? + .ok_or(SqliteClientError::AccountUnknown)?; + + let uivk = account.uivk(); + let ivk = uivk + .sapling() + .as_ref() + .expect("uivk decrypted this output."); + let to = output.note().recipient(); + let diversifier_index = ivk + .decrypt_diversifier(&to) + .expect("address corresponds to account"); + + let ua = account + .uivk() + .address(diversifier_index, Some(UnifiedAddressRequest::ALLOW_ALL))?; + + upsert_address( + conn, + params, + account.internal_id(), + diversifier_index, + &ua, + exposure_height, + ) + .map(Some) + } else { + Ok(None) + } +} + /// Records the specified shielded output as having been received. /// /// This implementation relies on the facts that: /// - A transaction will not contain more than 2^63 shielded outputs. /// - A note value will never exceed 2^63 zatoshis. -pub(crate) fn put_received_note>( - conn: &Transaction, +/// +/// Returns the internal account identifier of the account that received the output. +pub(crate) fn put_received_note< + T: ReceivedSaplingOutput, + P: consensus::Parameters, +>( + conn: &rusqlite::Transaction, + params: &P, output: &T, tx_ref: TxRef, + target_or_mined_height: Option, spent_in: Option, -) -> Result<(), SqliteClientError> { +) -> Result { let account_id = get_account_ref(conn, output.account_id())?; + let address_id = ensure_address(conn, params, output, target_or_mined_height)?; let mut stmt_upsert_received_note = conn.prepare_cached( - "INSERT INTO sapling_received_notes - (tx, output_index, account_id, diversifier, value, rcm, memo, nf, - is_change, commitment_tree_position, - recipient_key_scope) + "INSERT INTO sapling_received_notes ( + tx, output_index, account_id, address_id, + diversifier, value, rcm, memo, nf, + is_change, commitment_tree_position, + recipient_key_scope + ) VALUES ( :tx, :output_index, :account_id, + :address_id, :diversifier, :value, :rcm, @@ -362,6 +417,7 @@ pub(crate) fn put_received_note( fn address_index_from_diversifier_index_be( diversifier_index_be: &[u8], ) -> Result { - let mut di: [u8; 11] = diversifier_index_be.try_into().map_err(|_| { - SqliteClientError::CorruptedData("Diversifier index is not an 11-byte value".to_owned()) - })?; - di.reverse(); // BE -> LE conversion + let di = decode_diversifier_index_be(diversifier_index_be)?; - NonHardenedChildIndex::from_index(DiversifierIndex::from(di).try_into().map_err(|_| { - SqliteClientError::CorruptedData( - "Unable to get diversifier for transparent address.".to_string(), - ) - })?) - .ok_or_else(|| { + NonHardenedChildIndex::try_from(di).map_err(|_| { SqliteClientError::CorruptedData( "Unexpected hardened index for transparent address.".to_string(), ) @@ -83,45 +87,48 @@ pub(crate) fn get_transparent_receivers( conn: &rusqlite::Connection, params: &P, account_uuid: AccountUuid, + scopes: &[KeyScope], ) -> Result>, SqliteClientError> { let mut ret: HashMap> = HashMap::new(); - // Get all UAs derived - let mut ua_query = conn.prepare( - "SELECT address, diversifier_index_be - FROM addresses + // Get all addresses with the provided scopes. + let mut addr_query = conn.prepare( + "SELECT address, diversifier_index_be, key_scope + FROM addresses JOIN accounts ON accounts.id = addresses.account_id - WHERE accounts.uuid = :account_uuid", + WHERE accounts.uuid = :account_uuid + AND key_scope IN rarray(:scopes_ptr)", )?; - let mut rows = ua_query.query(named_params![":account_uuid": account_uuid.0])?; + + let scope_values: Vec = scopes.iter().map(|s| Value::Integer(s.encode())).collect(); + let scopes_ptr = Rc::new(scope_values); + let mut rows = addr_query.query(named_params![ + ":account_uuid": account_uuid.0, + ":scopes_ptr": &scopes_ptr + ])?; while let Some(row) = rows.next()? { let ua_str: String = row.get(0)?; let di_vec: Vec = row.get(1)?; + let scope = KeyScope::decode(row.get(2)?)?; - let ua = Address::decode(params, &ua_str) + let taddr = Address::decode(params, &ua_str) .ok_or_else(|| { SqliteClientError::CorruptedData("Not a valid Zcash recipient address".to_owned()) - }) - .and_then(|addr| match addr { - Address::Unified(ua) => Ok(ua), - _ => Err(SqliteClientError::CorruptedData(format!( - "Addresses table contains {} which is not a unified address", - ua_str, - ))), - })?; - - if let Some(taddr) = ua.transparent() { + })? + .to_transparent_address(); + + if let Some(taddr) = taddr { let address_index = address_index_from_diversifier_index_be(&di_vec)?; - let metadata = TransparentAddressMetadata::new(Scope::External.into(), address_index); - ret.insert(*taddr, Some(metadata)); + let metadata = TransparentAddressMetadata::new(scope.into(), address_index); + ret.insert(taddr, Some(metadata)); } } if let Some((taddr, address_index)) = get_legacy_transparent_address(params, conn, account_uuid)? { - let metadata = TransparentAddressMetadata::new(Scope::External.into(), address_index); + let metadata = TransparentAddressMetadata::new(KeyScope::EXTERNAL.into(), address_index); ret.insert(taddr, Some(metadata)); } @@ -183,6 +190,335 @@ pub(crate) fn get_legacy_transparent_address( Ok(None) } +/// Returns the transparent address index at the start of the first gap of at least `gap_limit` +/// indices in the given account, considering only addresses derived for the specified key scope. +/// +/// Returns `Ok(None)` if the gap would start at an index greater than the maximum valid +/// non-hardened transparent child index. +pub(crate) fn find_gap_start( + conn: &rusqlite::Connection, + account_id: AccountRef, + key_scope: KeyScope, + gap_limit: u32, +) -> Result, SqliteClientError> { + match conn + .query_row( + r#" + WITH offsets AS ( + SELECT + a.transparent_child_index, + LEAD(a.transparent_child_index) + OVER (ORDER BY a.transparent_child_index) + AS next_child_index + FROM v_address_first_use a + WHERE a.account_id = :account_id + AND a.key_scope = :key_scope + AND a.transparent_child_index IS NOT NULL + AND a.first_use_height IS NOT NULL + ) + SELECT + transparent_child_index + 1, + next_child_index - transparent_child_index - 1 AS gap_len + FROM offsets + WHERE gap_len >= :gap_limit OR next_child_index IS NULL + ORDER BY transparent_child_index + LIMIT 1 + "#, + named_params![ + ":account_id": account_id.0, + ":key_scope": key_scope.encode(), + ":gap_limit": gap_limit + ], + |row| row.get::<_, u32>(0), + ) + .optional()? + { + Some(i) => Ok(NonHardenedChildIndex::from_index(i)), + None => Ok(Some(NonHardenedChildIndex::ZERO)), + } +} + +pub(crate) fn decode_transparent_child_index( + value: i64, +) -> Result { + u32::try_from(value) + .ok() + .and_then(NonHardenedChildIndex::from_index) + .ok_or_else(|| { + SqliteClientError::CorruptedData(format!("Illegal transparent child index {value}")) + }) +} + +/// Returns a vector with the next `n` previously unreserved transparent addresses for +/// the given account. These addresses must have been previously generated using +/// `generate_gap_addresses`. +/// +/// # Errors +/// +/// * `SqliteClientError::AccountUnknown`, if there is no account with the given id. +/// * `SqliteClientError::ReachedGapLimit`, if it is not possible to reserve `n` addresses +/// within the gap limit after the last address in this account that is known to have an +/// output in a mined transaction. +/// * `SqliteClientError::AddressGeneration(AddressGenerationError::DiversifierSpaceExhausted)`, +/// if the limit on transparent address indices has been reached. +pub(crate) fn reserve_next_n_addresses( + conn: &rusqlite::Transaction, + params: &P, + account_id: AccountRef, + key_scope: KeyScope, + gap_limit: u32, + n: usize, +) -> Result, SqliteClientError> { + if n == 0 { + return Ok(vec![]); + } + + let gap_start = find_gap_start(conn, account_id, key_scope, gap_limit)?.ok_or( + SqliteClientError::AddressGeneration(AddressGenerationError::DiversifierSpaceExhausted), + )?; + + let mut stmt_addrs_to_reserve = conn.prepare( + "SELECT id, transparent_child_index, cached_transparent_receiver_address + FROM addresses + WHERE account_id = :account_id + AND key_scope = :key_scope + AND transparent_child_index >= :gap_start + AND transparent_child_index < :gap_end + AND exposed_at_height IS NULL + ORDER BY transparent_child_index + LIMIT :n", + )?; + + let addresses_to_reserve = stmt_addrs_to_reserve + .query_and_then( + named_params! { + ":account_id": account_id.0, + ":key_scope": key_scope.encode(), + ":gap_start": gap_start.index(), + ":gap_end": gap_start.saturating_add(gap_limit).index(), + ":n": n + }, + |row| { + let address_id = row.get("id").map(AddressRef)?; + let transparent_child_index = row + .get::<_, Option>("transparent_child_index")? + .map(decode_transparent_child_index) + .transpose()?; + let address = row + .get::<_, Option>("cached_transparent_receiver_address")? + .map(|addr_str| TransparentAddress::decode(params, &addr_str)) + .transpose()?; + + Ok::<_, SqliteClientError>(transparent_child_index.zip(address).map(|(i, a)| { + ( + address_id, + a, + TransparentAddressMetadata::new(key_scope.into(), i), + ) + })) + }, + )? + .filter_map(|r| r.transpose()) + .collect::, _>>()?; + + if addresses_to_reserve.len() < n { + return Err(SqliteClientError::ReachedGapLimit( + gap_start.index() + gap_limit, + )); + } + + let current_chain_tip = chain_tip_height(conn)?.ok_or(SqliteClientError::ChainHeightUnknown)?; + + let reserve_id_values: Vec = addresses_to_reserve + .iter() + .map(|(id, _, _)| Value::Integer(id.0)) + .collect(); + let reserved_ptr = Rc::new(reserve_id_values); + conn.execute( + "UPDATE addresses + SET exposed_at_height = :chain_tip_height + WHERE id IN rarray(:reserved_ptr)", + named_params! { + ":chain_tip_height": u32::from(current_chain_tip), + ":reserved_ptr": &reserved_ptr + }, + )?; + + Ok(addresses_to_reserve) +} + +/// Extend the range of preallocated addresses in an account to ensure that a full `gap_limit` of +/// transparent addresses is available from the first gap in existing indices of addresses at which +/// a received transaction has been observed on the chain, for each key scope. +/// +/// The provided [`UnifiedAddressRequest`] is used to pregenerate unified addresses that correspond +/// to the transparent address index in question; such unified addresses need not internally +/// contain a transparent receiver, and may be overwritten when these addresses are exposed via the +/// [`WalletWrite::get_next_available_address`] or [`WalletWrite::get_address_for_index`] methods. +/// If no request is provided, each address so generated will contain a receiver for each possible +/// pool: i.e., a recevier for each data item in the account's UFVK or UIVK where the transparent +/// child index is valid. +/// +/// [`WalletWrite::get_next_available_address`]: zcash_client_backend::data_api::WalletWrite::get_next_available_address +/// [`WalletWrite::get_address_for_index`]: zcash_client_backend::data_api::WalletWrite::get_address_for_index +pub(crate) fn generate_gap_addresses( + conn: &rusqlite::Transaction, + params: &P, + account_id: AccountRef, + key_scope: KeyScope, + gap_limits: &GapLimits, + request: Option, +) -> Result<(), SqliteClientError> { + let account = get_account_internal(conn, params, account_id)? + .ok_or_else(|| SqliteClientError::AccountUnknown)?; + + let request = request.unwrap_or_else(|| { + use ReceiverRequirement::*; + UnifiedAddressRequest::unsafe_new(Allow, Allow, Require) + }); + + let gen_addrs = |key_scope: KeyScope, index: NonHardenedChildIndex| { + Ok::<_, SqliteClientError>(match key_scope { + KeyScope::Zip32(zip32::Scope::External) => { + let ua = account.uivk().address(index.into(), Some(request)); + let transparent_address = account + .uivk() + .transparent() + .as_ref() + .ok_or(AddressGenerationError::KeyNotAvailable(Typecode::P2pkh))? + .derive_address(index)?; + ( + ua.map_or_else( + |e| { + if matches!(e, AddressGenerationError::ShieldedReceiverRequired) { + // fall back to the transparent-only address + Ok(Address::from(transparent_address).to_zcash_address(params)) + } else { + // other address generation errors are allowed to propagate + Err(e) + } + }, + |addr| Ok(Address::from(addr).to_zcash_address(params)), + )?, + transparent_address, + ) + } + KeyScope::Zip32(zip32::Scope::Internal) => { + let internal_address = account + .ufvk() + .and_then(|k| k.transparent()) + .ok_or(AddressGenerationError::KeyNotAvailable(Typecode::P2pkh))? + .derive_internal_ivk()? + .derive_address(index)?; + ( + Address::from(internal_address).to_zcash_address(params), + internal_address, + ) + } + KeyScope::Ephemeral => { + let ephemeral_address = account + .ufvk() + .and_then(|k| k.transparent()) + .ok_or(AddressGenerationError::KeyNotAvailable(Typecode::P2pkh))? + .derive_ephemeral_ivk()? + .derive_ephemeral_address(index)?; + ( + Address::from(ephemeral_address).to_zcash_address(params), + ephemeral_address, + ) + } + }) + }; + + let gap_limit = match key_scope { + KeyScope::Zip32(zip32::Scope::External) => gap_limits.external(), + KeyScope::Zip32(zip32::Scope::Internal) => gap_limits.transparent_internal(), + KeyScope::Ephemeral => gap_limits.ephemeral(), + }; + + if let Some(gap_start) = find_gap_start(conn, account_id, key_scope, gap_limit)? { + let range_to_store = gap_start.index()..gap_start.saturating_add(gap_limit).index(); + if range_to_store.is_empty() { + return Ok(()); + } + // exposed_at_height is initially NULL + let mut stmt_insert_address = conn.prepare_cached( + "INSERT INTO addresses ( + account_id, diversifier_index_be, key_scope, address, + transparent_child_index, cached_transparent_receiver_address + ) + VALUES ( + :account_id, :diversifier_index_be, :key_scope, :address, + :transparent_child_index, :transparent_address + ) + ON CONFLICT (account_id, diversifier_index_be, key_scope) DO NOTHING", + )?; + + for raw_index in range_to_store { + let transparent_child_index = NonHardenedChildIndex::from_index(raw_index) + .expect("restricted to valid range above"); + let (zcash_address, transparent_address) = + gen_addrs(key_scope, transparent_child_index)?; + + stmt_insert_address.execute(named_params![ + ":account_id": account_id.0, + ":diversifier_index_be": encode_diversifier_index_be(transparent_child_index.into()), + ":key_scope": key_scope.encode(), + ":address": zcash_address.encode(), + ":transparent_child_index": raw_index, + ":transparent_address": transparent_address.encode(params) + ])?; + } + } + + Ok(()) +} + +/// If `address` is one of our addresses, check whether it was used as the recipient address for +/// any of our received outputs. +/// +/// If the address was already used in an output we received, this method will return +/// [`SqliteClientError::AddressReuse`]. +pub(crate) fn check_address_reuse( + conn: &rusqlite::Transaction, + params: &P, + address: &Address, +) -> Result<(), SqliteClientError> { + // TODO: ideally we would do something better than string matching here - the best would be to + // have the diversifier index for the address passed to us instead of the address itself, but + // not all call sites currently have a good way to obtain the diversifier index. We could + // trial-decrypt with each of the wallet's IVKs if we wanted to do it here, but a better + // approach is to restructure the call sites so that we don't discard diversifier index + // information in the process of passing it through to here. + let addr_str = address.encode(params); + let taddr_str = address.to_transparent_address().map(|a| a.encode(params)); + + let mut stmt = conn.prepare_cached( + "SELECT t.txid + FROM transactions t + JOIN v_received_outputs vro ON vro.transaction_id = t.id_tx + JOIN addresses a ON a.id = vro.address_id + WHERE a.address = :address + OR a.cached_transparent_receiver_address = :transparent_address", + )?; + + let txids = stmt + .query_and_then( + named_params![ + ":address": addr_str, + ":transparent_address": taddr_str, + ], + |row| Ok(TxId::from_bytes(row.get::<_, [u8; 32]>(0)?)), + )? + .collect::, SqliteClientError>>()?; + + if let Some(txids) = NonEmpty::from_vec(txids) { + return Err(SqliteClientError::AddressReuse(addr_str, txids)); + } + + Ok(()) +} + fn to_unspent_transparent_output(row: &Row) -> Result { let txid: Vec = row.get("txid")?; let mut txid_bytes = [0u8; 32]; @@ -559,28 +895,19 @@ pub(crate) fn mark_transparent_utxo_spent( /// Adds the given received UTXO to the datastore. pub(crate) fn put_received_transparent_utxo( - conn: &rusqlite::Connection, + conn: &rusqlite::Transaction, params: &P, output: &WalletTransparentOutput, -) -> Result { - let address = output.recipient_address(); - if let Some(receiving_account) = - find_account_uuid_for_transparent_address(conn, params, address)? - { - put_transparent_output( - conn, - params, - output.outpoint(), - output.txout(), - output.mined_height(), - address, - receiving_account, - true, - ) - } else { - // The UTXO was not for any of our transparent addresses. - Err(SqliteClientError::AddressNotRecognized(*address)) - } +) -> Result<(AccountRef, KeyScope, UtxoId), SqliteClientError> { + put_transparent_output( + conn, + params, + output.outpoint(), + output.txout(), + output.mined_height(), + output.recipient_address(), + true, + ) } /// Returns the vector of [`TransactionDataRequest`]s that represents the information needed by the @@ -634,21 +961,29 @@ pub(crate) fn get_transparent_address_metadata( address: &TransparentAddress, ) -> Result, SqliteClientError> { let address_str = address.encode(params); - - if let Some(di_vec) = conn + let addr_meta = conn .query_row( - "SELECT diversifier_index_be FROM addresses + "SELECT diversifier_index_be, key_scope + FROM addresses JOIN accounts ON addresses.account_id = accounts.id - WHERE accounts.uuid = :account_uuid + WHERE accounts.uuid = :account_uuid AND cached_transparent_receiver_address = :address", named_params![":account_uuid": account_uuid.0, ":address": &address_str], - |row| row.get::<_, Vec>(0), + |row| { + let di_be: Vec = row.get(0)?; + let scope_code = row.get(1)?; + Ok(KeyScope::decode(scope_code).and_then(|key_scope| { + address_index_from_diversifier_index_be(&di_be).map(|address_index| { + TransparentAddressMetadata::new(key_scope.into(), address_index) + }) + })) + }, ) .optional()? - { - let address_index = address_index_from_diversifier_index_be(&di_vec)?; - let metadata = TransparentAddressMetadata::new(Scope::External.into(), address_index); - return Ok(Some(metadata)); + .transpose()?; + + if addr_meta.is_some() { + return Ok(addr_meta); } if let Some((legacy_taddr, address_index)) = @@ -660,13 +995,6 @@ pub(crate) fn get_transparent_address_metadata( } } - // Search known ephemeral addresses. - if let Some(address_index) = - ephemeral::find_index_for_ephemeral_address_str(conn, account_uuid, &address_str)? - { - return Ok(Some(ephemeral::metadata(address_index))); - } - Ok(None) } @@ -684,27 +1012,27 @@ pub(crate) fn find_account_uuid_for_transparent_address Result, SqliteClientError> { +) -> Result, SqliteClientError> { let address_str = address.encode(params); - if let Some(account_id) = conn + if let Some((account_id, key_scope_code)) = conn .query_row( - "SELECT accounts.uuid - FROM addresses + "SELECT accounts.uuid, addresses.key_scope + FROM addresses JOIN accounts ON accounts.id = addresses.account_id WHERE cached_transparent_receiver_address = :address", named_params![":address": &address_str], - |row| Ok(AccountUuid(row.get(0)?)), + |row| Ok((AccountUuid(row.get(0)?), row.get(1)?)), ) .optional()? { - return Ok(Some(account_id)); + return Ok(Some((account_id, KeyScope::decode(key_scope_code)?))); } // Search known ephemeral addresses. if let Some(account_id) = ephemeral::find_account_for_ephemeral_address_str(conn, &address_str)? { - return Ok(Some(account_id)); + return Ok(Some((account_id, KeyScope::Ephemeral))); } let account_ids = get_account_ids(conn)?; @@ -718,7 +1046,7 @@ pub(crate) fn find_account_uuid_for_transparent_address( txout: &TxOut, output_height: Option, address: &TransparentAddress, - receiving_account_uuid: AccountUuid, known_unspent: bool, -) -> Result { +) -> Result<(AccountRef, KeyScope, UtxoId), SqliteClientError> { + let addr_str = address.encode(params); + + // Unlike the shielded pools, we only can receive transparent outputs on addresses for which we + // have an `addresses` table entry, so we can just query for that here. + let (address_id, account_id, key_scope_code) = conn.query_row( + "SELECT id, account_id, key_scope + FROM addresses + WHERE cached_transparent_receiver_address = :transparent_address", + named_params! {":transparent_address": addr_str}, + |row| { + Ok(( + row.get("id").map(AddressRef)?, + row.get("account_id").map(AccountRef)?, + row.get("key_scope")?, + )) + }, + )?; + + let key_scope = KeyScope::decode(key_scope_code)?; + let output_height = output_height.map(u32::from); - let receiving_account_id = super::get_account_ref(conn, receiving_account_uuid)?; // Check whether we have an entry in the blocks table for the output height; // if not, the transaction will be updated with its mined height when the @@ -810,16 +1156,17 @@ pub(crate) fn put_transparent_output( let mut stmt_upsert_transparent_output = conn.prepare_cached( "INSERT INTO transparent_received_outputs ( transaction_id, output_index, - account_id, address, script, + account_id, address_id, address, script, value_zat, max_observed_unspent_height ) VALUES ( :transaction_id, :output_index, - :account_id, :address, :script, + :account_id, :address_id, :address, :script, :value_zat, :max_observed_unspent_height ) ON CONFLICT (transaction_id, output_index) DO UPDATE SET account_id = :account_id, + address_id = :address_id, address = :address, script = :script, value_zat = :value_zat, @@ -830,7 +1177,8 @@ pub(crate) fn put_transparent_output( let sql_args = named_params![ ":transaction_id": id_tx, ":output_index": &outpoint.n(), - ":account_id": receiving_account_id.0, + ":account_id": account_id.0, + ":address_id": address_id.0, ":address": &address.encode(params), ":script": &txout.script_pubkey.0, ":value_zat": &i64::from(ZatBalance::from(txout.value)), @@ -862,7 +1210,7 @@ pub(crate) fn put_transparent_output( mark_transparent_utxo_spent(conn, spending_transaction_id, outpoint)?; } - Ok(utxo_id) + Ok((account_id, key_scope, utxo_id)) } /// Adds a request to retrieve transactions involving the specified address to the transparent @@ -900,17 +1248,30 @@ pub(crate) fn queue_transparent_spend_detection( #[cfg(test)] mod tests { + use std::collections::BTreeMap; + use secrecy::Secret; use transparent::keys::NonHardenedChildIndex; use zcash_client_backend::{ - data_api::{testing::TestBuilder, Account as _, WalletWrite, GAP_LIMIT}, + data_api::{ + testing::{AddressType, TestBuilder}, + wallet::decrypt_and_store_transaction, + Account as _, WalletRead, WalletWrite, + }, wallet::TransparentAddressMetadata, }; + use zcash_keys::address::Address; use zcash_primitives::block::BlockHash; + use zcash_protocol::value::Zatoshis; use crate::{ + error::SqliteClientError, testing::{db::TestDbFactory, BlockCache}, - wallet::{get_account_ref, transparent::ephemeral}, + wallet::{ + get_account_ref, + transparent::{ephemeral, find_gap_start, reserve_next_n_addresses}, + KeyScope, + }, WalletDb, }; @@ -937,6 +1298,71 @@ mod tests { ); } + #[test] + fn gap_limits() { + let mut st = TestBuilder::new() + .with_data_store_factory(TestDbFactory::default()) + .with_block_cache(BlockCache::new()) + .with_account_from_sapling_activation(BlockHash([0; 32])) + .build(); + + let test_account = st.test_account().cloned().unwrap(); + let account_uuid = test_account.account().id(); + let ufvk = test_account.account().ufvk().unwrap().clone(); + + let external_taddrs = st + .wallet() + .get_transparent_receivers(account_uuid, false, false) + .unwrap(); + assert_eq!(external_taddrs.len(), 20); //20 external + let external_taddrs_sorted = external_taddrs + .into_iter() + .filter_map(|(addr, meta)| meta.map(|m| (m.address_index(), addr))) + .collect::>(); + + let all_taddrs = st + .wallet() + .get_transparent_receivers(account_uuid, true, true) + .unwrap(); + assert_eq!(all_taddrs.len(), 30); //20 external, 5 internal, 5 ephemeral + + // Add some funds to the wallet + let (h0, _, _) = st.generate_next_block( + &ufvk.sapling().unwrap(), + AddressType::DefaultExternal, + Zatoshis::const_from_u64(1000000), + ); + st.scan_cached_blocks(h0, 1); + + let to = Address::from( + *external_taddrs_sorted + .get(&NonHardenedChildIndex::from_index(9).unwrap()) + .expect("An address exists at index 9."), + ) + .to_zcash_address(st.network()); + + // Create a transaction & scan the block. Since the txid corresponds to one our wallet + // generated, this should cause the gap limit to be bumped (generating addresses with index + // 20..30 + let txids = st + .create_standard_transaction(&test_account, to, Zatoshis::const_from_u64(20000)) + .unwrap(); + let (h1, _) = st.generate_next_block_including(txids.head); + st.scan_cached_blocks(h1, 1); + + // use `decrypt_and_store_transaction` to ensure that the wallet sees the transaction as + // mined (since transparent handling doesn't get this from `scan_cached_blocks`) + let tx = st.wallet().get_transaction(txids.head).unwrap().unwrap(); + decrypt_and_store_transaction(&st.network().clone(), st.wallet_mut(), &tx, Some(h1)) + .unwrap(); + + let external_taddrs = st + .wallet() + .get_transparent_receivers(account_uuid, false, false) + .unwrap(); + assert_eq!(external_taddrs.len(), 30); + } + #[test] fn ephemeral_address_management() { let mut st = TestBuilder::new() @@ -949,21 +1375,59 @@ mod tests { let account0_uuid = st.test_account().unwrap().account().id(); let account0_id = get_account_ref(&st.wallet().db().conn, account0_uuid).unwrap(); + // The chain height must be known in order to reserve addresses, as we store the height at + // which the address was considered to be exposed. + st.wallet_mut() + .db_mut() + .update_chain_tip(birthday.height()) + .unwrap(); + let check = |db: &WalletDb<_, _>, account_id| { eprintln!("checking {account_id:?}"); - assert_matches!(ephemeral::first_unstored_index(&db.conn, account_id), Ok(addr_index) if addr_index == GAP_LIMIT); - assert_matches!(ephemeral::first_unreserved_index(&db.conn, account_id), Ok(addr_index) if addr_index == 0); + assert_matches!( + find_gap_start(&db.conn, account_id, KeyScope::Ephemeral, db.gap_limits.ephemeral()), Ok(addr_index) + if addr_index == Some(NonHardenedChildIndex::ZERO) + ); + //assert_matches!(ephemeral::first_unstored_index(&db.conn, account_id), Ok(addr_index) if addr_index == GAP_LIMIT); let known_addrs = ephemeral::get_known_ephemeral_addresses(&db.conn, &db.params, account_id, None) .unwrap(); - let expected_metadata: Vec = (0..GAP_LIMIT) + let expected_metadata: Vec = (0..db.gap_limits.ephemeral()) .map(|i| ephemeral::metadata(NonHardenedChildIndex::from_index(i).unwrap())) .collect(); let actual_metadata: Vec = known_addrs.into_iter().map(|(_, meta)| meta).collect(); assert_eq!(actual_metadata, expected_metadata); + + let transaction = &db.conn.unchecked_transaction().unwrap(); + // reserve half the addresses (rounding down) + let reserved = reserve_next_n_addresses( + transaction, + &db.params, + account_id, + KeyScope::Ephemeral, + db.gap_limits.ephemeral(), + (db.gap_limits.ephemeral() / 2) as usize, + ) + .unwrap(); + assert_eq!(reserved.len(), (db.gap_limits.ephemeral() / 2) as usize); + + // we have not yet used any of the addresses, so the maximum available address index + // should not have increased, and therefore attempting to reserve a full gap limit + // worth of addresses should fail. + assert_matches!( + reserve_next_n_addresses( + transaction, + &db.params, + account_id, + KeyScope::Ephemeral, + db.gap_limits.ephemeral(), + db.gap_limits.ephemeral() as usize + ), + Err(SqliteClientError::ReachedGapLimit(_)) + ); }; check(st.wallet().db(), account0_id); diff --git a/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs b/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs index 293eaa9550..d97706d3f7 100644 --- a/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs +++ b/zcash_client_sqlite/src/wallet/transparent/ephemeral.rs @@ -1,22 +1,17 @@ -//! Functions for wallet support of ephemeral transparent addresses. -use std::cmp::{max, min}; +//! Functikjns for wallet support of ephemeral transparent addresses. use std::ops::Range; use rusqlite::{named_params, OptionalExtension}; use ::transparent::{ address::TransparentAddress, - keys::{EphemeralIvk, NonHardenedChildIndex, TransparentKeyScope}, + keys::{NonHardenedChildIndex, TransparentKeyScope}, }; -use zcash_client_backend::{data_api::GAP_LIMIT, wallet::TransparentAddressMetadata}; -use zcash_keys::keys::UnifiedFullViewingKey; -use zcash_keys::{encoding::AddressCodec, keys::AddressGenerationError}; -use zcash_primitives::transaction::TxId; +use zcash_client_backend::wallet::TransparentAddressMetadata; +use zcash_keys::encoding::AddressCodec; use zcash_protocol::consensus; -use crate::wallet::{self, get_account_ref}; -use crate::AccountUuid; -use crate::{error::SqliteClientError, AccountRef, TxRef}; +use crate::{error::SqliteClientError, wallet::KeyScope, AccountRef, AccountUuid}; // Returns `TransparentAddressMetadata` in the ephemeral scope for the // given address index. @@ -24,161 +19,50 @@ pub(crate) fn metadata(address_index: NonHardenedChildIndex) -> TransparentAddre TransparentAddressMetadata::new(TransparentKeyScope::EPHEMERAL, address_index) } -/// Returns the first unstored ephemeral address index in the given account. -pub(crate) fn first_unstored_index( - conn: &rusqlite::Connection, - account_id: AccountRef, -) -> Result { - match conn - .query_row( - "SELECT address_index FROM ephemeral_addresses - WHERE account_id = :account_id - ORDER BY address_index DESC - LIMIT 1", - named_params![":account_id": account_id.0], - |row| row.get::<_, u32>(0), - ) - .optional()? - { - Some(i) if i >= (1 << 31) + GAP_LIMIT => { - unreachable!("violates constraint index_range_and_address_nullity") - } - Some(i) => Ok(i.checked_add(1).unwrap()), - None => Ok(0), - } -} - -/// Returns the first unreserved ephemeral address index in the given account. -pub(crate) fn first_unreserved_index( - conn: &rusqlite::Connection, - account_id: AccountRef, -) -> Result { - first_unstored_index(conn, account_id)? - .checked_sub(GAP_LIMIT) - .ok_or(SqliteClientError::CorruptedData( - "ephemeral_addresses table has not been initialized".to_owned(), - )) -} - -/// Returns the first ephemeral address index in the given account that -/// would violate the gap invariant if used. -pub(crate) fn first_unsafe_index( - conn: &rusqlite::Connection, - account_id: AccountRef, -) -> Result { - // The inner join with `transactions` excludes addresses for which - // `seen_in_tx` is NULL. The query also excludes addresses observed - // to have been mined in a transaction that we currently see as unmined. - // This is conservative in terms of avoiding violation of the gap - // invariant: it can only cause us to get to the end of the gap sooner. - // - // TODO: do we want to only consider transactions with a minimum number - // of confirmations here? - let first_unmined_index: u32 = match conn - .query_row( - "SELECT address_index FROM ephemeral_addresses - JOIN transactions t ON t.id_tx = seen_in_tx - WHERE account_id = :account_id AND t.mined_height IS NOT NULL - ORDER BY address_index DESC - LIMIT 1", - named_params![":account_id": account_id.0], - |row| row.get::<_, u32>(0), - ) - .optional()? - { - Some(i) if i >= 1 << 31 => { - unreachable!("violates constraint index_range_and_address_nullity") - } - Some(i) => i.checked_add(1).unwrap(), - None => 0, - }; - Ok(min( - 1 << 31, - first_unmined_index.checked_add(GAP_LIMIT).unwrap(), - )) -} - -/// Utility function to return an `Range` that starts at `i` -/// and is of length up to `n`. The range is truncated if necessary -/// so that it contains no elements beyond the maximum valid address -/// index, `(1 << 31) - 1`. -pub(crate) fn range_from(i: u32, n: u32) -> Range { - let first = min(1 << 31, i); - let last = min(1 << 31, i.saturating_add(n)); - first..last -} - -/// Returns the ephemeral transparent IVK for a given account ID. -pub(crate) fn get_ephemeral_ivk( - conn: &rusqlite::Connection, - params: &P, - account_id: AccountRef, -) -> Result, SqliteClientError> { - let ufvk = conn - .query_row( - "SELECT ufvk FROM accounts WHERE id = :account_id", - named_params![":account_id": account_id.0], - |row| { - let ufvk_str: Option = row.get("ufvk")?; - Ok(ufvk_str.map(|s| { - UnifiedFullViewingKey::decode(params, &s[..]) - .map_err(SqliteClientError::BadAccountData) - })) - }, - ) - .optional()? - .ok_or(SqliteClientError::AccountUnknown)? - .transpose()?; - - let eivk = ufvk - .as_ref() - .and_then(|ufvk| ufvk.transparent()) - .map(|t| t.derive_ephemeral_ivk()) - .transpose()?; - - Ok(eivk) -} - -/// Returns a vector of ephemeral transparent addresses associated with the given -/// account controlled by this wallet, along with their metadata. The result includes -/// reserved addresses, and addresses for `GAP_LIMIT` additional indices (capped to -/// the maximum index). +/// Returns a vector of ephemeral transparent addresses associated with the given account +/// controlled by this wallet, along with their metadata. The result includes reserved addresses, +/// and addresses for the wallet's configured ephemeral address gap limit of additional indices +/// (capped to the maximum index). /// -/// If `index_range` is some `Range`, it limits the result to addresses with indices -/// in that range. +/// If `index_range` is some `Range`, it limits the result to addresses with indices in that range. pub(crate) fn get_known_ephemeral_addresses( conn: &rusqlite::Connection, params: &P, account_id: AccountRef, - index_range: Option>, + index_range: Option>, ) -> Result, SqliteClientError> { - let index_range = index_range.unwrap_or(0..(1 << 31)); - let mut stmt = conn.prepare( - "SELECT address, address_index - FROM ephemeral_addresses ea - WHERE ea.account_id = :account_id - AND address_index >= :start - AND address_index < :end - ORDER BY address_index", + "SELECT cached_transparent_receiver_address, transparent_child_index + FROM addresses + WHERE account_id = :account_id + AND transparent_child_index >= :start + AND transparent_child_index < :end + AND key_scope = :key_scope + ORDER BY transparent_child_index", )?; - let mut rows = stmt.query(named_params![ - ":account_id": account_id.0, - ":start": index_range.start, - ":end": min(1 << 31, index_range.end), - ])?; - let mut result = vec![]; + let results = stmt + .query_and_then( + named_params![ + ":account_id": account_id.0, + ":start": index_range.as_ref().map_or(NonHardenedChildIndex::ZERO, |i| i.start).index(), + ":end": index_range.as_ref().map_or(NonHardenedChildIndex::MAX, |i| i.end).index(), + ":key_scope": KeyScope::Ephemeral.encode() + ], + |row| { + let addr_str: String = row.get(0)?; + let raw_index: u32 = row.get(1)?; + let address_index = NonHardenedChildIndex::from_index(raw_index) + .expect("where clause ensures this is in range"); + Ok::<_, SqliteClientError>(( + TransparentAddress::decode(params, &addr_str)?, + metadata(address_index) + )) + }, + )? + .collect::, _>>()?; - while let Some(row) = rows.next()? { - let addr_str: String = row.get(0)?; - let raw_index: u32 = row.get(1)?; - let address_index = NonHardenedChildIndex::from_index(raw_index) - .expect("where clause ensures this is in range"); - let address = TransparentAddress::decode(params, &addr_str)?; - result.push((address, metadata(address_index))); - } - Ok(result) + Ok(results) } /// If this is a known ephemeral address in any account, return its account id. @@ -189,277 +73,15 @@ pub(crate) fn find_account_for_ephemeral_address_str( Ok(conn .query_row( "SELECT accounts.uuid - FROM ephemeral_addresses ea - JOIN accounts ON accounts.id = ea.account_id - WHERE address = :address", - named_params![":address": &address_str], + FROM addresses + JOIN accounts ON accounts.id = account_id + WHERE cached_transparent_receiver_address = :address + AND key_scope = :key_scope", + named_params![ + ":address": &address_str, + ":key_scope": KeyScope::Ephemeral.encode() + ], |row| Ok(AccountUuid(row.get(0)?)), ) .optional()?) } - -/// If this is a known ephemeral address in the given account, return its index. -pub(crate) fn find_index_for_ephemeral_address_str( - conn: &rusqlite::Connection, - account_uuid: AccountUuid, - address_str: &str, -) -> Result, SqliteClientError> { - let account_id = get_account_ref(conn, account_uuid)?; - Ok(conn - .query_row( - "SELECT address_index FROM ephemeral_addresses - WHERE account_id = :account_id AND address = :address", - named_params![":account_id": account_id.0, ":address": &address_str], - |row| row.get::<_, u32>(0), - ) - .optional()? - .map(|index| { - NonHardenedChildIndex::from_index(index) - .expect("valid by constraint index_range_and_address_nullity") - })) -} - -/// Returns a vector with the next `n` previously unreserved ephemeral addresses for -/// the given account. -/// -/// # Errors -/// -/// * `SqliteClientError::AccountUnknown`, if there is no account with the given id. -/// * `SqliteClientError::UnknownZip32Derivation`, if the account is imported and -/// it is not possible to derive new addresses for it. -/// * `SqliteClientError::ReachedGapLimit`, if it is not possible to reserve `n` addresses -/// within the gap limit after the last address in this account that is known to have an -/// output in a mined transaction. -/// * `SqliteClientError::AddressGeneration(AddressGenerationError::DiversifierSpaceExhausted)`, -/// if the limit on transparent address indices has been reached. -pub(crate) fn reserve_next_n_ephemeral_addresses( - conn: &rusqlite::Transaction, - params: &P, - account_id: AccountRef, - n: usize, -) -> Result, SqliteClientError> { - if n == 0 { - return Ok(vec![]); - } - - let first_unreserved = first_unreserved_index(conn, account_id)?; - let first_unsafe = first_unsafe_index(conn, account_id)?; - let allocation = range_from( - first_unreserved, - u32::try_from(n).map_err(|_| AddressGenerationError::DiversifierSpaceExhausted)?, - ); - - if allocation.len() < n { - return Err(AddressGenerationError::DiversifierSpaceExhausted.into()); - } - if allocation.end > first_unsafe { - let account_uuid = wallet::get_account_uuid(conn, account_id)?; - return Err(SqliteClientError::ReachedGapLimit( - account_uuid, - max(first_unreserved, first_unsafe), - )); - } - reserve_until(conn, params, account_id, allocation.end)?; - get_known_ephemeral_addresses(conn, params, account_id, Some(allocation)) -} - -/// Initialize the `ephemeral_addresses` table. This must be called when -/// creating or migrating an account. -pub(crate) fn init_account( - conn: &rusqlite::Transaction, - params: &P, - account_id: AccountRef, -) -> Result<(), SqliteClientError> { - reserve_until(conn, params, account_id, 0) -} - -/// Extend the range of stored addresses in an account if necessary so that the index of the next -/// address to reserve will be *at least* `next_to_reserve`. If no transparent key exists for the -/// given account or it would already have been at least `next_to_reserve`, then do nothing. -/// -/// Note that this is called from database migration code. -/// -/// # Panics -/// -/// Panics if the precondition `next_to_reserve <= (1 << 31)` does not hold. -fn reserve_until( - conn: &rusqlite::Transaction, - params: &P, - account_id: AccountRef, - next_to_reserve: u32, -) -> Result<(), SqliteClientError> { - assert!(next_to_reserve <= 1 << 31); - - if let Some(ephemeral_ivk) = get_ephemeral_ivk(conn, params, account_id)? { - let first_unstored = first_unstored_index(conn, account_id)?; - let range_to_store = first_unstored..(next_to_reserve.checked_add(GAP_LIMIT).unwrap()); - if range_to_store.is_empty() { - return Ok(()); - } - - // used_in_tx and seen_in_tx are initially NULL - let mut stmt_insert_ephemeral_address = conn.prepare_cached( - "INSERT INTO ephemeral_addresses (account_id, address_index, address) - VALUES (:account_id, :address_index, :address)", - )?; - - for raw_index in range_to_store { - // The range to store may contain indicies that are out of the valid range of non hardened - // child indices; we still store explicit rows in the ephemeral_addresses table for these - // so that it's possible to find the first unused address using dead reckoning with the gap - // limit. - let address_str_opt = NonHardenedChildIndex::from_index(raw_index) - .map(|address_index| { - ephemeral_ivk - .derive_ephemeral_address(address_index) - .map(|addr| addr.encode(params)) - }) - .transpose()?; - - stmt_insert_ephemeral_address.execute(named_params![ - ":account_id": account_id.0, - ":address_index": raw_index, - ":address": address_str_opt, - ])?; - } - } - - Ok(()) -} - -/// Returns a `SqliteClientError::EphemeralAddressReuse` error if the address was -/// already used. -fn ephemeral_address_reuse_check( - conn: &rusqlite::Transaction, - address_str: &str, -) -> Result<(), SqliteClientError> { - // It is intentional that we don't require `t.mined_height` to be non-null. - // That is, we conservatively treat an ephemeral address as potentially - // reused even if we think that the transaction where we had evidence of - // its use is at present unmined. This should never occur in supported - // situations where only a single correctly operating wallet instance is - // using a given seed, because such a wallet will not reuse an address that - // it ever reserved. - // - // `COALESCE(used_in_tx, seen_in_tx)` can only differ from `used_in_tx` - // if the address was reserved, an error occurred in transaction creation - // before calling `mark_ephemeral_address_as_used`, and then we saw the - // address in another transaction (presumably created by another wallet - // instance, or as a result of a bug) anyway. - let res = conn - .query_row( - "SELECT t.txid FROM ephemeral_addresses - LEFT OUTER JOIN transactions t - ON t.id_tx = COALESCE(used_in_tx, seen_in_tx) - WHERE address = :address", - named_params![":address": address_str], - |row| row.get::<_, Option>>(0), - ) - .optional()? - .flatten(); - - if let Some(txid_bytes) = res { - let txid = TxId::from_bytes( - txid_bytes - .try_into() - .map_err(|_| SqliteClientError::CorruptedData("invalid txid".to_owned()))?, - ); - Err(SqliteClientError::EphemeralAddressReuse( - address_str.to_owned(), - txid, - )) - } else { - Ok(()) - } -} - -/// If `address` is one of our ephemeral addresses, mark it as having an output -/// in a transaction that we have just created. This has no effect if `address` is -/// not one of our ephemeral addresses. -/// -/// Returns a `SqliteClientError::EphemeralAddressReuse` error if the address was -/// already used. -pub(crate) fn mark_ephemeral_address_as_used( - conn: &rusqlite::Transaction, - params: &P, - ephemeral_address: &TransparentAddress, - tx_ref: TxRef, -) -> Result<(), SqliteClientError> { - let address_str = ephemeral_address.encode(params); - ephemeral_address_reuse_check(conn, &address_str)?; - - // We update both `used_in_tx` and `seen_in_tx` here, because a used address has - // necessarily been seen in a transaction. We will not treat this as extending the - // range of addresses that are safe to reserve unless and until the transaction is - // observed as mined. - let update_result = conn - .query_row( - "UPDATE ephemeral_addresses - SET used_in_tx = :tx_ref, seen_in_tx = :tx_ref - WHERE address = :address - RETURNING account_id, address_index", - named_params![":tx_ref": tx_ref.0, ":address": address_str], - |row| Ok((AccountRef(row.get::<_, u32>(0)?), row.get::<_, u32>(1)?)), - ) - .optional()?; - - // Maintain the invariant that the last `GAP_LIMIT` addresses are unused and unseen. - if let Some((account_id, address_index)) = update_result { - let next_to_reserve = address_index.checked_add(1).expect("ensured by constraint"); - reserve_until(conn, params, account_id, next_to_reserve)?; - } - Ok(()) -} - -/// If `address` is one of our ephemeral addresses, mark it as having an output -/// in the given mined transaction (which may or may not be a transaction we sent). -/// -/// `tx_ref` must be a valid transaction reference. This call has no effect if -/// `address` is not one of our ephemeral addresses. -pub(crate) fn mark_ephemeral_address_as_seen( - conn: &rusqlite::Transaction, - params: &P, - address: &TransparentAddress, - tx_ref: TxRef, -) -> Result<(), SqliteClientError> { - let address_str = address.encode(params); - - // Figure out which transaction was mined earlier: `tx_ref`, or any existing - // tx referenced by `seen_in_tx` for the given address. Prefer the existing - // reference in case of a tie or if both transactions are unmined. - // This slightly reduces the chance of unnecessarily reaching the gap limit - // too early in some corner cases (because the earlier transaction is less - // likely to be unmined). - // - // The query should always return a value if `tx_ref` is valid. - let earlier_ref = conn.query_row( - "SELECT id_tx FROM transactions - LEFT OUTER JOIN ephemeral_addresses e - ON id_tx = e.seen_in_tx - WHERE id_tx = :tx_ref OR e.address = :address - ORDER BY mined_height ASC NULLS LAST, - tx_index ASC NULLS LAST, - e.seen_in_tx ASC NULLS LAST - LIMIT 1", - named_params![":tx_ref": tx_ref.0, ":address": address_str], - |row| row.get::<_, i64>(0), - )?; - - let update_result = conn - .query_row( - "UPDATE ephemeral_addresses - SET seen_in_tx = :seen_in_tx - WHERE address = :address - RETURNING account_id, address_index", - named_params![":seen_in_tx": &earlier_ref, ":address": address_str], - |row| Ok((AccountRef(row.get::<_, u32>(0)?), row.get::<_, u32>(1)?)), - ) - .optional()?; - - // Maintain the invariant that the last `GAP_LIMIT` addresses are unused and unseen. - if let Some((account_id, address_index)) = update_result { - let next_to_reserve = address_index.checked_add(1).expect("ensured by constraint"); - reserve_until(conn, params, account_id, next_to_reserve)?; - } - Ok(()) -} diff --git a/zcash_transparent/CHANGELOG.md b/zcash_transparent/CHANGELOG.md index 46dc2a245c..b902cdd780 100644 --- a/zcash_transparent/CHANGELOG.md +++ b/zcash_transparent/CHANGELOG.md @@ -13,6 +13,11 @@ and this library adheres to Rust's notion of ### Added - `zcash_transparent::pczt::Bip32Derivation::extract_bip_44_fields` +- `zcash_transparent::keys::NonHardenedChildIndex::saturating_sub` +- `zcash_transparent::keys::NonHardenedChildIndex::saturating_add` +- `zcash_transparent::keys::NonHardenedChildIndex::MAX` +- `impl From for zip32::DiversifierIndex` +- `impl TryFrom for NonHardenedChildIndex` ## [0.1.0] - 2024-12-16 diff --git a/zcash_transparent/src/keys.rs b/zcash_transparent/src/keys.rs index 74c6dccb39..4f56264b00 100644 --- a/zcash_transparent/src/keys.rs +++ b/zcash_transparent/src/keys.rs @@ -2,6 +2,7 @@ use bip32::ChildNumber; use subtle::{Choice, ConstantTimeEq}; +use zip32::DiversifierIndex; #[cfg(feature = "transparent-inputs")] use { @@ -67,7 +68,7 @@ impl From for ChildNumber { /// A child index for a derived transparent address. /// /// Only NON-hardened derivation is supported. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] pub struct NonHardenedChildIndex(u32); impl ConstantTimeEq for NonHardenedChildIndex { @@ -77,13 +78,17 @@ impl ConstantTimeEq for NonHardenedChildIndex { } impl NonHardenedChildIndex { + /// The minimum valid non-hardened child index. pub const ZERO: NonHardenedChildIndex = NonHardenedChildIndex(0); + /// The maximum valid non-hardened child index. + pub const MAX: NonHardenedChildIndex = NonHardenedChildIndex((1 << 31) - 1); + /// Parses the given ZIP 32 child index. /// /// Returns `None` if the hardened bit is set. - pub fn from_index(i: u32) -> Option { - if i < (1 << 31) { + pub const fn from_index(i: u32) -> Option { + if i <= Self::MAX.0 { Some(NonHardenedChildIndex(i)) } else { None @@ -91,15 +96,32 @@ impl NonHardenedChildIndex { } /// Returns the index as a 32-bit integer. - pub fn index(&self) -> u32 { + pub const fn index(&self) -> u32 { self.0 } - pub fn next(&self) -> Option { + /// Returns the successor to this index. + pub const fn next(&self) -> Option { // overflow cannot happen because self.0 is 31 bits, and the next index is at most 32 bits // which in that case would lead from_index to return None. Self::from_index(self.0 + 1) } + + /// Subtracts the given delta from this index. + pub const fn saturating_sub(&self, delta: u32) -> Self { + NonHardenedChildIndex(self.0.saturating_sub(delta)) + } + + /// Adds the given delta to this index, returning a maximum possible value of + /// [`NonHardenedChildIndex::MAX`]. + pub const fn saturating_add(&self, delta: u32) -> Self { + let idx = self.0 + delta; + if idx > Self::MAX.0 { + Self::MAX + } else { + NonHardenedChildIndex(idx) + } + } } impl TryFrom for NonHardenedChildIndex { @@ -120,6 +142,21 @@ impl From for ChildNumber { } } +impl TryFrom for NonHardenedChildIndex { + type Error = (); + + fn try_from(value: DiversifierIndex) -> Result { + let idx = u32::try_from(value).map_err(|_| ())?; + NonHardenedChildIndex::from_index(idx).ok_or(()) + } +} + +impl From for DiversifierIndex { + fn from(value: NonHardenedChildIndex) -> Self { + DiversifierIndex::from(value.0) + } +} + /// A [BIP44] private key at the account path level `m/44'/'/'`. /// /// [BIP44]: https://github.com/bitcoin/bips/blob/master/bip-0044.mediawiki