add back changes for solana token 2022 (#6107)

* add back changes for solana * moving * removing * removing * sources removed * Fix name * Update tokens_solana_transfers.sql * Try splitting up * Fxi typo * fix union * typeing * Try another split * Fix typo * Use correct path --------- Co-authored-by: Andrew <[email protected]> Co-authored-by: Alan Ghobadi <[email protected]>
duneanalytics · Jun 10, 2024 · d06e7c7 · d06e7c7
1 parent 4e7b0c7
commit d06e7c7
Show file tree

Hide file tree

Showing 5 changed files with 368 additions and 43 deletions.
diff --git a/models/tokens/solana/tokens_solana_schema.yml b/models/tokens/solana/tokens_solana_schema.yml
@@ -97,18 +97,21 @@ models:
     config:
       tags: ['solana','transfers','erc20','nft','spl']
     description: >
-        get all spl token transfers (will add in token2022 later)
+        get all spl token transfers
     columns:
       - name: block_time
       - name: block_date
       - name: block_slot
       - name: action
       - name: token_mint_address
       - name: amount
+      - name: fee
+      - name: token_version
       - name: from_owner
       - name: to_owner
       - name: from_token_account
       - name: to_token_account
+      - name: token_version
       - name: tx_signer 
       - name: tx_id
       - name: outer_instruction_index

diff --git a/models/tokens/solana/tokens_solana_spl_transfers.sql b/models/tokens/solana/tokens_solana_spl_transfers.sql
@@ -0,0 +1,131 @@
+ {{
+  config(
+        schema = 'tokens_solana',
+        alias = 'spl_transfers',
+        materialized = 'incremental',
+        file_format = 'delta',
+        incremental_strategy = 'merge',
+        partition_by = ['block_date'],
+        incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')],
+        unique_key = ['tx_id','outer_instruction_index','inner_instruction_index', 'block_slot']
+  )
+}}
+
+
+WITH
+base as (
+      SELECT
+            account_source, account_destination
+            , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount
+            , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer
+            , 'transfer' as action
+            , call_outer_instruction_index, call_inner_instruction_index
+            , cast(null as double) as fee
+            , 'spl_token' as token_version
+      FROM {{ source('spl_token_solana','spl_token_call_transferChecked') }}
+      WHERE 1=1
+      {% if is_incremental() %}
+      AND {{incremental_predicate('call_block_time')}}
+      {% endif %}
+
+      UNION ALL
+
+      SELECT
+            null as account_source, account_account as account_destination
+            , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount
+            , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer
+            , 'mint' as action
+            , call_outer_instruction_index, call_inner_instruction_index
+            , cast(null as double) as fee
+            , 'spl_token' as token_version
+      FROM {{ source('spl_token_solana','spl_token_call_mintTo') }}
+      WHERE 1=1
+      {% if is_incremental() %}
+      AND {{incremental_predicate('call_block_time')}}
+      {% endif %}
+
+      UNION ALL
+
+      SELECT
+            null as account_source, account_account as account_destination
+            , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount
+            , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer
+            , 'mint' as action
+            , call_outer_instruction_index, call_inner_instruction_index
+            , cast(null as double) as fee
+            , 'spl_token' as token_version
+      FROM {{ source('spl_token_solana','spl_token_call_mintToChecked') }}
+      WHERE 1=1
+      {% if is_incremental() %}
+      AND {{incremental_predicate('call_block_time')}}
+      {% endif %}
+
+      UNION ALL
+
+      SELECT
+            account_account as account_source, null as account_destination
+            , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount
+            , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer
+            , 'burn' as action
+            , call_outer_instruction_index, call_inner_instruction_index
+            , cast(null as double) as fee
+            , 'spl_token' as token_version
+      FROM {{ source('spl_token_solana','spl_token_call_burn') }}
+      WHERE 1=1
+      {% if is_incremental() %}
+      AND {{incremental_predicate('call_block_time')}}
+      {% endif %}
+
+      UNION ALL
+
+      SELECT
+            account_account as account_source, null as account_destination
+            , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount
+            , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer
+            , 'burn' as action
+            , call_outer_instruction_index, call_inner_instruction_index
+            , cast(null as double) as fee
+            , 'spl_token' as token_version
+      FROM {{ source('spl_token_solana','spl_token_call_burnChecked') }}
+      WHERE 1=1
+      {% if is_incremental() %}
+      AND {{incremental_predicate('call_block_time')}}
+      {% endif %}
+)
+
+SELECT
+    call_block_time as block_time
+    , cast (date_trunc('day', call_block_time) as date) as block_date
+    , call_block_slot as block_slot
+    , action
+    , amount
+    , fee
+    , account_source as from_token_account
+    , account_destination as to_token_account
+    , token_version
+    , call_tx_signer as tx_signer
+    , call_tx_id as tx_id
+    , call_outer_instruction_index as outer_instruction_index
+    , COALESCE(call_inner_instruction_index,0) as inner_instruction_index
+    , call_outer_executing_account as outer_executing_account
+FROM base
+UNION ALL
+SELECT
+    block_time
+    , block_date
+    , block_slot
+    , action
+    , amount
+    , fee
+    , from_token_account
+    , to_token_account
+    , token_version
+    , tx_signer
+    , tx_id
+    , outer_instruction_index
+    , inner_instruction_index
+    , outer_executing_account
+ FROM {{ref('tokens_solana_spl_transfers_call_transfer')}}
+{% if is_incremental() %}
+      AND {{incremental_predicate('call_block_time')}}
+      {% endif %}
diff --git a/models/tokens/solana/tokens_solana_spl_transfers_call_transfer.sql b/models/tokens/solana/tokens_solana_spl_transfers_call_transfer.sql
@@ -0,0 +1,33 @@
+ {{
+  config(
+        schema = 'tokens_solana',
+        alias = 'spl_transfers_call_transfer',
+        materialized = 'incremental',
+        file_format = 'delta',
+        incremental_strategy = 'merge',
+        partition_by = ['block_date'],
+        incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')],
+        unique_key = ['tx_id','outer_instruction_index','inner_instruction_index', 'block_slot']
+  )
+}}
+
+SELECT
+      call_block_time as block_time
+    , cast (date_trunc('day', call_block_time) as date) as block_date
+    , call_block_slot as block_slot
+    , 'transfer' as action
+    , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount
+    , cast(null as double) as fee
+    , account_source as from_token_account
+    , account_destination as to_token_account
+    , 'spl_token' as token_version
+    , call_tx_signer as tx_signer
+    , call_tx_id as tx_id
+    , call_outer_instruction_index as outer_instruction_index
+    , COALESCE(call_inner_instruction_index,0) as inner_instruction_index
+    , call_outer_executing_account as outer_executing_account
+FROM {{ source('spl_token_solana','spl_token_call_transfer') }}
+WHERE 1=1
+{% if is_incremental() %}
+AND {{incremental_predicate('call_block_time')}}
+{% endif %}
diff --git a/models/tokens/solana/tokens_solana_token22_spl_transfers.sql b/models/tokens/solana/tokens_solana_token22_spl_transfers.sql
@@ -0,0 +1,145 @@
+ {{
+  config(
+        schema = 'tokens_solana',
+        alias = 'token22_spl_transfers',
+        materialized = 'incremental',
+        file_format = 'delta',
+        incremental_strategy = 'merge',
+        partition_by = ['block_date'],
+        incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.block_time')],
+        unique_key = ['tx_id','outer_instruction_index','inner_instruction_index', 'block_slot']
+  )
+}}
+
+
+WITH
+base as (
+      --token2022. Most mint and account extensions still use the parent transferChecked instruction, hooks are excecuted after and interest-bearing is precalculated.
+      SELECT
+            account_source, account_destination, amount
+            , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer
+            , action
+            , call_outer_instruction_index, call_inner_instruction_index
+            , fee
+            , token_version
+      FROM (
+            SELECT
+                  account_source, account_destination
+                  , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount --note that interestbearing mints have a different amount methodology, to add later
+                  , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer
+                  , 'transfer' as action
+                  , call_outer_instruction_index, call_inner_instruction_index
+                  , least(
+                        cast(bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as double)
+                              *cast(f.fee_basis as double)/10000
+                        ,f.fee_maximum) as fee --we want to take the percent fee on total amount, but not exceed the maximum fee
+                  , 'token2022' as token_version
+                  , f.fee_time
+                  , row_number() over (partition by tr.call_tx_id,  tr.call_outer_instruction_index,  tr.call_inner_instruction_index order by f.fee_time desc) as latest_fee
+            FROM {{ source('spl_token_2022_solana','spl_token_2022_call_transferChecked') }} tr
+            LEFT JOIN {{ ref('tokens_solana_fees_history') }} f ON tr.account_tokenMint = f.account_mint AND tr.call_block_time >= f.fee_time
+            WHERE 1=1
+            {% if is_incremental() %}
+            AND {{incremental_predicate('tr.call_block_time')}}
+            {% endif %}
+      ) WHERE latest_fee = 1
+
+      UNION ALL
+
+      SELECT
+            null as account_source, account_mintTo as account_destination
+            , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount
+            , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer
+            , 'mint' as action
+            , call_outer_instruction_index, call_inner_instruction_index
+            , null as fee
+            , 'token2022' as token_version
+      FROM {{ source('spl_token_2022_solana','spl_token_2022_call_mintTo') }}
+      WHERE 1=1
+      {% if is_incremental() %}
+      AND {{incremental_predicate('call_block_time')}}
+      {% endif %}
+
+      UNION ALL
+
+      SELECT
+            null as account_source, account_mintTo as account_destination
+            , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount
+            , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer
+            , 'mint' as action
+            , call_outer_instruction_index, call_inner_instruction_index
+            , null as fee
+            , 'token2022' as token_version
+      FROM {{ source('spl_token_2022_solana','spl_token_2022_call_mintToChecked') }}
+      WHERE 1=1
+      {% if is_incremental() %}
+      AND {{incremental_predicate('call_block_time')}}
+      {% endif %}
+
+      UNION ALL
+
+      SELECT
+            account_burnAccount as account_source, null as account_destination
+            , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount
+            , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer
+            , 'burn' as action
+            , call_outer_instruction_index, call_inner_instruction_index
+            , null as fee
+            , 'token2022' as token_version
+      FROM {{ source('spl_token_2022_solana','spl_token_2022_call_burn') }}
+      WHERE 1=1
+      {% if is_incremental() %}
+      AND {{incremental_predicate('call_block_time')}}
+      {% endif %}
+
+      UNION ALL
+
+      SELECT
+            account_burnAccount as account_source, null as account_destination
+            , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+1,8))) as amount
+            , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer
+            , 'burn' as action
+            , call_outer_instruction_index, call_inner_instruction_index
+            , null as fee
+            , 'token2022' as token_version
+      FROM {{ source('spl_token_2022_solana','spl_token_2022_call_burnChecked') }}
+      WHERE 1=1
+      {% if is_incremental() %}
+      AND {{incremental_predicate('call_block_time')}}
+      {% endif %}
+
+      --token2022 transferFeeExtension has some extra complications. It's the only extension with its own transferChecked wrapper (confidential transfers will have this too)
+      UNION ALL
+
+      SELECT
+            call_account_arguments[1] as account_source, call_account_arguments[3] as account_destination
+            , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,1+2,8))) as amount
+            , call_tx_id, call_block_time, call_block_slot, call_outer_executing_account, call_tx_signer
+            , 'transfer' as action
+            , call_outer_instruction_index, call_inner_instruction_index
+            , bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data, 1+2+8+1,8))) as fee
+            , 'token2022' as token_version
+      FROM {{ source('spl_token_2022_solana','spl_token_2022_call_transferFeeExtension') }}
+      WHERE bytearray_substring(call_data,1,2) = 0x1a01 --https://github.com/solana-labs/solana-program-library/blob/8f50c6fabc6ec87ada229e923030381f573e0aed/token/program-2022/src/extension/transfer_fee/instruction.rs#L284
+      {% if is_incremental() %}
+      AND {{incremental_predicate('call_block_time')}}
+      {% endif %}
+)
+
+SELECT
+    call_block_time as block_time
+    , cast (date_trunc('day', call_block_time) as date) as block_date
+    , call_block_slot as block_slot
+    , action
+    , amount
+    , fee
+    , account_source as from_token_account
+    , account_destination as to_token_account
+    , token_version
+    , call_tx_signer as tx_signer
+    , call_tx_id as tx_id
+    , call_outer_instruction_index as outer_instruction_index
+    , COALESCE(call_inner_instruction_index,0) as inner_instruction_index
+    , call_outer_executing_account as outer_executing_account
+FROM base tr
+-- AND call_block_time > now() - interval '90' day --for faster CI testing