From d06e7c7ef1ac4c12f47fad2d14846f30f4fb54c3 Mon Sep 17 00:00:00 2001 From: jeff-dude <102681548+jeff-dude@users.noreply.github.com> Date: Mon, 10 Jun 2024 07:57:56 -0400 Subject: [PATCH] add back changes for solana token 2022 (#6107) * add back changes for solana * moving * removing * removing * sources removed * Fix name * Update tokens_solana_transfers.sql * Try splitting up * Fxi typo * fix union * typeing * Try another split * Fix typo * Use correct path --------- Co-authored-by: Andrew <47720952+andrewhong5297@users.noreply.github.com> Co-authored-by: Alan Ghobadi --- models/tokens/solana/tokens_solana_schema.yml | 5 +- .../solana/tokens_solana_spl_transfers.sql | 131 ++++++++++++++++ ...ens_solana_spl_transfers_call_transfer.sql | 33 ++++ .../tokens_solana_token22_spl_transfers.sql | 145 ++++++++++++++++++ .../tokens/solana/tokens_solana_transfers.sql | 97 +++++++----- 5 files changed, 368 insertions(+), 43 deletions(-) create mode 100644 models/tokens/solana/tokens_solana_spl_transfers.sql create mode 100644 models/tokens/solana/tokens_solana_spl_transfers_call_transfer.sql create mode 100644 models/tokens/solana/tokens_solana_token22_spl_transfers.sql diff --git a/models/tokens/solana/tokens_solana_schema.yml b/models/tokens/solana/tokens_solana_schema.yml index 4a6bd1a599e..7242599a841 100644 --- a/models/tokens/solana/tokens_solana_schema.yml +++ b/models/tokens/solana/tokens_solana_schema.yml @@ -97,7 +97,7 @@ models: config: tags: ['solana','transfers','erc20','nft','spl'] description: > - get all spl token transfers (will add in token2022 later) + get all spl token transfers columns: - name: block_time - name: block_date @@ -105,10 +105,13 @@ models: - name: action - name: token_mint_address - name: amount + - name: fee + - name: token_version - name: from_owner - name: to_owner - name: from_token_account - name: to_token_account + - name: token_version - name: tx_signer - name: tx_id - name: outer_instruction_index diff --git a/models/tokens/solana/tokens_solana_spl_transfers.sql b/models/tokens/solana/tokens_solana_spl_transfers.sql new file mode 100644 index 00000000000..2f788049ac5 --- /dev/null +++ b/models/tokens/solana/tokens_solana_spl_transfers.sql @@ -0,0 +1,131 @@ + {{ + config( + schema = 'tokens_solana', + alias = 'spl_transfers', + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + partition_by = ['block_date'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')], + unique_key = ['tx_id','outer_instruction_index','inner_instruction_index', 'block_slot'] + ) +}} + + +WITH +base as ( + SELECT + account_source, account_destination + , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount + , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer + , 'transfer' as action + , call_outer_instruction_index, call_inner_instruction_index + , cast(null as double) as fee + , 'spl_token' as token_version + FROM {{ source('spl_token_solana','spl_token_call_transferChecked') }} + WHERE 1=1 + {% if is_incremental() %} + AND {{incremental_predicate('call_block_time')}} + {% endif %} + + UNION ALL + + SELECT + null as account_source, account_account as account_destination + , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount + , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer + , 'mint' as action + , call_outer_instruction_index, call_inner_instruction_index + , cast(null as double) as fee + , 'spl_token' as token_version + FROM {{ source('spl_token_solana','spl_token_call_mintTo') }} + WHERE 1=1 + {% if is_incremental() %} + AND {{incremental_predicate('call_block_time')}} + {% endif %} + + UNION ALL + + SELECT + null as account_source, account_account as account_destination + , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount + , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer + , 'mint' as action + , call_outer_instruction_index, call_inner_instruction_index + , cast(null as double) as fee + , 'spl_token' as token_version + FROM {{ source('spl_token_solana','spl_token_call_mintToChecked') }} + WHERE 1=1 + {% if is_incremental() %} + AND {{incremental_predicate('call_block_time')}} + {% endif %} + + UNION ALL + + SELECT + account_account as account_source, null as account_destination + , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount + , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer + , 'burn' as action + , call_outer_instruction_index, call_inner_instruction_index + , cast(null as double) as fee + , 'spl_token' as token_version + FROM {{ source('spl_token_solana','spl_token_call_burn') }} + WHERE 1=1 + {% if is_incremental() %} + AND {{incremental_predicate('call_block_time')}} + {% endif %} + + UNION ALL + + SELECT + account_account as account_source, null as account_destination + , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount + , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer + , 'burn' as action + , call_outer_instruction_index, call_inner_instruction_index + , cast(null as double) as fee + , 'spl_token' as token_version + FROM {{ source('spl_token_solana','spl_token_call_burnChecked') }} + WHERE 1=1 + {% if is_incremental() %} + AND {{incremental_predicate('call_block_time')}} + {% endif %} +) + +SELECT + call_block_time as block_time + , cast (date_trunc('day', call_block_time) as date) as block_date + , call_block_slot as block_slot + , action + , amount + , fee + , account_source as from_token_account + , account_destination as to_token_account + , token_version + , call_tx_signer as tx_signer + , call_tx_id as tx_id + , call_outer_instruction_index as outer_instruction_index + , COALESCE(call_inner_instruction_index,0) as inner_instruction_index + , call_outer_executing_account as outer_executing_account +FROM base +UNION ALL +SELECT + block_time + , block_date + , block_slot + , action + , amount + , fee + , from_token_account + , to_token_account + , token_version + , tx_signer + , tx_id + , outer_instruction_index + , inner_instruction_index + , outer_executing_account + FROM {{ref('tokens_solana_spl_transfers_call_transfer')}} +{% if is_incremental() %} + AND {{incremental_predicate('call_block_time')}} + {% endif %} \ No newline at end of file diff --git a/models/tokens/solana/tokens_solana_spl_transfers_call_transfer.sql b/models/tokens/solana/tokens_solana_spl_transfers_call_transfer.sql new file mode 100644 index 00000000000..cc319990015 --- /dev/null +++ b/models/tokens/solana/tokens_solana_spl_transfers_call_transfer.sql @@ -0,0 +1,33 @@ + {{ + config( + schema = 'tokens_solana', + alias = 'spl_transfers_call_transfer', + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + partition_by = ['block_date'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')], + unique_key = ['tx_id','outer_instruction_index','inner_instruction_index', 'block_slot'] + ) +}} + +SELECT + call_block_time as block_time + , cast (date_trunc('day', call_block_time) as date) as block_date + , call_block_slot as block_slot + , 'transfer' as action + , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount + , cast(null as double) as fee + , account_source as from_token_account + , account_destination as to_token_account + , 'spl_token' as token_version + , call_tx_signer as tx_signer + , call_tx_id as tx_id + , call_outer_instruction_index as outer_instruction_index + , COALESCE(call_inner_instruction_index,0) as inner_instruction_index + , call_outer_executing_account as outer_executing_account +FROM {{ source('spl_token_solana','spl_token_call_transfer') }} +WHERE 1=1 +{% if is_incremental() %} +AND {{incremental_predicate('call_block_time')}} +{% endif %} diff --git a/models/tokens/solana/tokens_solana_token22_spl_transfers.sql b/models/tokens/solana/tokens_solana_token22_spl_transfers.sql new file mode 100644 index 00000000000..9ebc9bff9cb --- /dev/null +++ b/models/tokens/solana/tokens_solana_token22_spl_transfers.sql @@ -0,0 +1,145 @@ + {{ + config( + schema = 'tokens_solana', + alias = 'token22_spl_transfers', + materialized = 'incremental', + file_format = 'delta', + incremental_strategy = 'merge', + partition_by = ['block_date'], + incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')], + unique_key = ['tx_id','outer_instruction_index','inner_instruction_index', 'block_slot'] + ) +}} + + +WITH +base as ( + --token2022. Most mint and account extensions still use the parent transferChecked instruction, hooks are excecuted after and interest-bearing is precalculated. + SELECT + account_source, account_destination, amount + , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer + , action + , call_outer_instruction_index, call_inner_instruction_index + , fee + , token_version + FROM ( + SELECT + account_source, account_destination + , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount --note that interestbearing mints have a different amount methodology, to add later + , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer + , 'transfer' as action + , call_outer_instruction_index, call_inner_instruction_index + , least( + cast(bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as double) + *cast(f.fee_basis as double)/10000 + ,f.fee_maximum) as fee --we want to take the percent fee on total amount, but not exceed the maximum fee + , 'token2022' as token_version + , f.fee_time + , row_number() over (partition by tr.call_tx_id, tr.call_outer_instruction_index, tr.call_inner_instruction_index order by f.fee_time desc) as latest_fee + FROM {{ source('spl_token_2022_solana','spl_token_2022_call_transferChecked') }} tr + LEFT JOIN {{ ref('tokens_solana_fees_history') }} f ON tr.account_tokenMint = f.account_mint AND tr.call_block_time >= f.fee_time + WHERE 1=1 + {% if is_incremental() %} + AND {{incremental_predicate('tr.call_block_time')}} + {% endif %} + ) WHERE latest_fee = 1 + + UNION ALL + + SELECT + null as account_source, account_mintTo as account_destination + , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount + , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer + , 'mint' as action + , call_outer_instruction_index, call_inner_instruction_index + , null as fee + , 'token2022' as token_version + FROM {{ source('spl_token_2022_solana','spl_token_2022_call_mintTo') }} + WHERE 1=1 + {% if is_incremental() %} + AND {{incremental_predicate('call_block_time')}} + {% endif %} + + UNION ALL + + SELECT + null as account_source, account_mintTo as account_destination + , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount + , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer + , 'mint' as action + , call_outer_instruction_index, call_inner_instruction_index + , null as fee + , 'token2022' as token_version + FROM {{ source('spl_token_2022_solana','spl_token_2022_call_mintToChecked') }} + WHERE 1=1 + {% if is_incremental() %} + AND {{incremental_predicate('call_block_time')}} + {% endif %} + + UNION ALL + + SELECT + account_burnAccount as account_source, null as account_destination + , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount + , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer + , 'burn' as action + , call_outer_instruction_index, call_inner_instruction_index + , null as fee + , 'token2022' as token_version + FROM {{ source('spl_token_2022_solana','spl_token_2022_call_burn') }} + WHERE 1=1 + {% if is_incremental() %} + AND {{incremental_predicate('call_block_time')}} + {% endif %} + + UNION ALL + + SELECT + account_burnAccount as account_source, null as account_destination + , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount + , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer + , 'burn' as action + , call_outer_instruction_index, call_inner_instruction_index + , null as fee + , 'token2022' as token_version + FROM {{ source('spl_token_2022_solana','spl_token_2022_call_burnChecked') }} + WHERE 1=1 + {% if is_incremental() %} + AND {{incremental_predicate('call_block_time')}} + {% endif %} + + --token2022 transferFeeExtension has some extra complications. It's the only extension with its own transferChecked wrapper (confidential transfers will have this too) + UNION ALL + + SELECT + call_account_arguments[1] as account_source, call_account_arguments[3] as account_destination + , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+2,8))) as amount + , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer + , 'transfer' as action + , call_outer_instruction_index, call_inner_instruction_index + , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data, 1+2+8+1,8))) as fee + , 'token2022' as token_version + FROM {{ source('spl_token_2022_solana','spl_token_2022_call_transferFeeExtension') }} + WHERE bytearray_substring(call_data,1,2) = 0x1a01 --https://github.com/solana-labs/solana-program-library/blob/8f50c6fabc6ec87ada229e923030381f573e0aed/token/program-2022/src/extension/transfer_fee/instruction.rs#L284 + {% if is_incremental() %} + AND {{incremental_predicate('call_block_time')}} + {% endif %} +) + +SELECT + call_block_time as block_time + , cast (date_trunc('day', call_block_time) as date) as block_date + , call_block_slot as block_slot + , action + , amount + , fee + , account_source as from_token_account + , account_destination as to_token_account + , token_version + , call_tx_signer as tx_signer + , call_tx_id as tx_id + , call_outer_instruction_index as outer_instruction_index + , COALESCE(call_inner_instruction_index,0) as inner_instruction_index + , call_outer_executing_account as outer_executing_account +FROM base tr +-- AND call_block_time > now() - interval '90' day --for faster CI testing \ No newline at end of file diff --git a/models/tokens/solana/tokens_solana_transfers.sql b/models/tokens/solana/tokens_solana_transfers.sql index a28903cf3d4..228ba1c332d 100644 --- a/models/tokens/solana/tokens_solana_transfers.sql +++ b/models/tokens/solana/tokens_solana_transfers.sql @@ -14,56 +14,69 @@ \'["ilemi"]\') }}') }} + +WITH +base as ( + SELECT + block_time, + , call_block_slot as block_slot + , action + , amount + , fee + , from_token_account + , to_token_account + , token_version + , tx_signer + , tx_id + , outer_instruction_index + , inner_instruction_index + , outer_executing_account +FROM {{ ref('tokens_solana_spl_transfers') }} +{% if is_incremental() %} +WHERE {{incremental_predicate('block_time')}} +{% endif %} +UNION ALL + SELECT + block_time, + , call_block_slot as block_slot + , action + , amount + , fee + , from_token_account + , to_token_account + , token_version + , tx_signer + , tx_id + , outer_instruction_index + , inner_instruction_index + , outer_executing_account +FROM {{ ref('tokens_solana_token22_spl_transfers') }} +{% if is_incremental() %} +WHERE {{incremental_predicate('block_time')}} +{% endif %} +) + SELECT call_block_time as block_time , cast (date_trunc('day', call_block_time) as date) as block_date , call_block_slot as block_slot , action , amount + , fee , COALESCE(tk_s.token_mint_address, tk_d.token_mint_address) as token_mint_address , tk_s.token_balance_owner as from_owner , tk_d.token_balance_owner as to_owner - , account_source as from_token_account - , account_destination as to_token_account - , call_tx_signer as tx_signer - , call_tx_id as tx_id - , call_outer_instruction_index as outer_instruction_index - , COALESCE(call_inner_instruction_index,0) as inner_instruction_index - , call_outer_executing_account as outer_executing_account -FROM ( - SELECT account_source, account_destination, amount, call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer, 'transfer' as action, call_outer_instruction_index, call_inner_instruction_index - FROM {{ source('spl_token_solana','spl_token_call_transfer') }} - - UNION ALL - - SELECT account_source, account_destination, amount, call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer, 'transfer' as action, call_outer_instruction_index, call_inner_instruction_index - FROM {{ source('spl_token_solana','spl_token_call_transferChecked') }} - - UNION ALL - - SELECT null, account_account as account_destination, amount, call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer, 'mint' as action, call_outer_instruction_index, call_inner_instruction_index - FROM {{ source('spl_token_solana','spl_token_call_mintTo') }} - - UNION ALL - - SELECT null, account_account as account_destination, amount, call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer, 'mint' as action, call_outer_instruction_index, call_inner_instruction_index - FROM {{ source('spl_token_solana','spl_token_call_mintToChecked') }} - - UNION ALL - - SELECT account_account as account_source, null, amount, call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer, 'burn' as action, call_outer_instruction_index, call_inner_instruction_index - FROM {{ source('spl_token_solana','spl_token_call_burn') }} - - UNION ALL - - SELECT account_account as account_source, null, amount, call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer, 'burn' as action, call_outer_instruction_index, call_inner_instruction_index - FROM {{ source('spl_token_solana','spl_token_call_burnChecked') }} -) tr + , from_token_account + , to_token_account + , token_version + , tx_signer + , tx_id + , outer_instruction_index + , inner_instruction_index + , outer_executing_account +FROM base tr --get token and accounts -LEFT JOIN {{ ref('solana_utils_token_accounts') }} tk_s ON tk_s.address = tr.account_source -LEFT JOIN {{ ref('solana_utils_token_accounts') }} tk_d ON tk_d.address = tr.account_destination +INNER JOIN {{ ref('solana_utils_token_accounts') }} tk_s ON tk_s.address = tr.account_source +INNER JOIN {{ ref('solana_utils_token_accounts') }} tk_d ON tk_d.address = tr.account_destination WHERE 1=1 -{% if is_incremental() %} -AND {{incremental_predicate('call_block_time')}} -{% endif %} --- AND call_block_time > now() - interval '600' day \ No newline at end of file +-- AND call_block_time > now() - interval '90' day --for faster CI testing \ No newline at end of file