From f2dfa4702ad19c15c302f77c4000c3babd23d7eb Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Tue, 10 Dec 2024 16:03:38 +0000 Subject: [PATCH 1/2] zcash_client_sqlite: Fix accounts joins in add_account_uuids migration Some views had account ID fields that were potentially NULL. As part of migrating to account UUIDs, we now join on the accounts table to access the UUID. We need to use a `LEFT JOIN` specifically to ensure that we propagate a NULL account ID into a NULL account UUID, instead of dropping the row. --- zcash_client_sqlite/src/wallet/db.rs | 8 ++++---- .../src/wallet/init/migrations/add_account_uuids.rs | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/zcash_client_sqlite/src/wallet/db.rs b/zcash_client_sqlite/src/wallet/db.rs index 83b2a5fd5..78283a3e0 100644 --- a/zcash_client_sqlite/src/wallet/db.rs +++ b/zcash_client_sqlite/src/wallet/db.rs @@ -859,7 +859,7 @@ SELECT accounts.uuid AS account_uuid, AND MAX(COALESCE(sent_note_counts.sent_notes, 0)) = 0 ) AS is_shielding FROM notes -JOIN accounts ON accounts.id = notes.account_id +LEFT JOIN accounts ON accounts.id = notes.account_id LEFT JOIN transactions ON notes.txid = transactions.txid JOIN blocks_max_height @@ -900,8 +900,8 @@ JOIN transactions -- join to the sent_notes table to obtain `from_account_id` LEFT JOIN sent_notes ON sent_notes.id = ro.sent_note_id -- join on the accounts table to obtain account UUIDs -JOIN accounts from_account ON from_account.id = sent_notes.from_account_id -JOIN accounts to_account ON to_account.id = ro.account_id +LEFT JOIN accounts from_account ON from_account.id = sent_notes.from_account_id +LEFT JOIN accounts to_account ON to_account.id = ro.account_id UNION -- select all outputs sent from the wallet to external recipients SELECT transactions.txid AS txid, @@ -918,7 +918,7 @@ JOIN transactions ON transactions.id_tx = sent_notes.tx LEFT JOIN v_received_outputs ro ON ro.sent_note_id = sent_notes.id -- join on the accounts table to obtain account UUIDs -JOIN accounts from_account ON from_account.id = sent_notes.from_account_id +LEFT JOIN accounts from_account ON from_account.id = sent_notes.from_account_id -- exclude any sent notes for which a row exists in the v_received_outputs view WHERE ro.account_id IS NULL"; diff --git a/zcash_client_sqlite/src/wallet/init/migrations/add_account_uuids.rs b/zcash_client_sqlite/src/wallet/init/migrations/add_account_uuids.rs index a10149efb..1c7e8f5c5 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/add_account_uuids.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/add_account_uuids.rs @@ -260,7 +260,7 @@ impl RusqliteMigration for Migration { AND MAX(COALESCE(sent_note_counts.sent_notes, 0)) = 0 ) AS is_shielding FROM notes - JOIN accounts ON accounts.id = notes.account_id + LEFT JOIN accounts ON accounts.id = notes.account_id LEFT JOIN transactions ON notes.txid = transactions.txid JOIN blocks_max_height @@ -289,8 +289,8 @@ impl RusqliteMigration for Migration { -- join to the sent_notes table to obtain `from_account_id` LEFT JOIN sent_notes ON sent_notes.id = ro.sent_note_id -- join on the accounts table to obtain account UUIDs - JOIN accounts from_account ON from_account.id = sent_notes.from_account_id - JOIN accounts to_account ON to_account.id = ro.account_id + LEFT JOIN accounts from_account ON from_account.id = sent_notes.from_account_id + LEFT JOIN accounts to_account ON to_account.id = ro.account_id UNION -- select all outputs sent from the wallet to external recipients SELECT transactions.txid AS txid, @@ -307,7 +307,7 @@ impl RusqliteMigration for Migration { ON transactions.id_tx = sent_notes.tx LEFT JOIN v_received_outputs ro ON ro.sent_note_id = sent_notes.id -- join on the accounts table to obtain account UUIDs - JOIN accounts from_account ON from_account.id = sent_notes.from_account_id + LEFT JOIN accounts from_account ON from_account.id = sent_notes.from_account_id -- exclude any sent notes for which a row exists in the v_received_outputs view WHERE ro.account_id IS NULL", )?; From d9e74a5d0161e010643274f77d58df8c19db7618 Mon Sep 17 00:00:00 2001 From: Jack Grigg Date: Tue, 10 Dec 2024 16:41:38 +0000 Subject: [PATCH 2/2] zcash_client_sqlite: Merge partially overlapping v_tx_outputs rows For certain cross-account transactions, we can end up with some data being obtainable from the sent notes view, and other data being obtainable from the received notes view. Previously we dropped the sent notes view data to avoid duplicates; now we actively merge the two. --- zcash_client_sqlite/src/wallet/db.rs | 10 +++++++--- .../src/wallet/init/migrations/add_account_uuids.rs | 10 +++++++--- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/zcash_client_sqlite/src/wallet/db.rs b/zcash_client_sqlite/src/wallet/db.rs index 78283a3e0..804246b4f 100644 --- a/zcash_client_sqlite/src/wallet/db.rs +++ b/zcash_client_sqlite/src/wallet/db.rs @@ -884,6 +884,7 @@ GROUP BY notes.account_id, notes.txid"; /// that controls the output. pub(super) const VIEW_TX_OUTPUTS: &str = " CREATE VIEW v_tx_outputs AS +WITH unioned AS ( -- select all outputs received by the wallet SELECT transactions.txid AS txid, ro.pool AS output_pool, @@ -902,7 +903,7 @@ LEFT JOIN sent_notes ON sent_notes.id = ro.sent_note_id -- join on the accounts table to obtain account UUIDs LEFT JOIN accounts from_account ON from_account.id = sent_notes.from_account_id LEFT JOIN accounts to_account ON to_account.id = ro.account_id -UNION +UNION ALL -- select all outputs sent from the wallet to external recipients SELECT transactions.txid AS txid, sent_notes.output_pool AS output_pool, @@ -919,8 +920,11 @@ JOIN transactions LEFT JOIN v_received_outputs ro ON ro.sent_note_id = sent_notes.id -- join on the accounts table to obtain account UUIDs LEFT JOIN accounts from_account ON from_account.id = sent_notes.from_account_id --- exclude any sent notes for which a row exists in the v_received_outputs view -WHERE ro.account_id IS NULL"; +) +-- merge duplicate rows while retaining maximum information +SELECT txid, output_pool, output_index, max(from_account_uuid), max(to_account_uuid), max(to_address), max(value), max(is_change), max(memo) +FROM unioned +GROUP BY txid, output_pool, output_index"; pub(super) fn view_sapling_shard_scan_ranges(params: &P) -> String { format!( diff --git a/zcash_client_sqlite/src/wallet/init/migrations/add_account_uuids.rs b/zcash_client_sqlite/src/wallet/init/migrations/add_account_uuids.rs index 1c7e8f5c5..3a1743232 100644 --- a/zcash_client_sqlite/src/wallet/init/migrations/add_account_uuids.rs +++ b/zcash_client_sqlite/src/wallet/init/migrations/add_account_uuids.rs @@ -273,6 +273,7 @@ impl RusqliteMigration for Migration { -- Replace accounts.id with accounts.uuid in v_tx_outputs. DROP VIEW v_tx_outputs; CREATE VIEW v_tx_outputs AS + WITH unioned AS ( -- select all outputs received by the wallet SELECT transactions.txid AS txid, ro.pool AS output_pool, @@ -291,7 +292,7 @@ impl RusqliteMigration for Migration { -- join on the accounts table to obtain account UUIDs LEFT JOIN accounts from_account ON from_account.id = sent_notes.from_account_id LEFT JOIN accounts to_account ON to_account.id = ro.account_id - UNION + UNION ALL -- select all outputs sent from the wallet to external recipients SELECT transactions.txid AS txid, sent_notes.output_pool AS output_pool, @@ -308,8 +309,11 @@ impl RusqliteMigration for Migration { LEFT JOIN v_received_outputs ro ON ro.sent_note_id = sent_notes.id -- join on the accounts table to obtain account UUIDs LEFT JOIN accounts from_account ON from_account.id = sent_notes.from_account_id - -- exclude any sent notes for which a row exists in the v_received_outputs view - WHERE ro.account_id IS NULL", + ) + -- merge duplicate rows while retaining maximum information + SELECT txid, output_pool, output_index, max(from_account_uuid), max(to_account_uuid), max(to_address), max(value), max(is_change), max(memo) + FROM unioned + GROUP BY txid, output_pool, output_index", )?; Ok(())