Token 2022 fungible (add metadata model first

models/tokens/solana/tokens_solana.sources.yml

@@ -24,6 +24,29 @@ sources:
       - name: spl_token_call_burnChecked
         loaded_at_field: call_block_time
 
+  - name: spl_token_2022_solana
+    description: "spl_token 2022 decoded tables"
+    freshness: # default freshness
+      warn_after: { count: 12, period: hour }
+      error_after: { count: 24, period: hour }
+    tables:
+      - name: spl_token_2022_call_initializeMint
+        loaded_at_field: call_block_time
+      - name: spl_token_2022_call_initializeMint2
+        loaded_at_field: call_block_time
+      - name: spl_token_2022_call_transferFeeExtension
+        loaded_at_field: call_block_time
+      - name: spl_token_2022_call_transferChecked
+        loaded_at_field: call_block_time
+      - name: spl_token_2022_call_mintTo
+        loaded_at_field: call_block_time
+      - name: spl_token_2022_call_mintToChecked
+        loaded_at_field: call_block_time
+      - name: spl_token_2022_call_burn
+        loaded_at_field: call_block_time
+      - name: spl_token_2022_call_burnChecked
+        loaded_at_field: call_block_time
+
   - name: mpl_token_metadata_solana
     description: "mpl metadata decoded tables"
     freshness: # default freshness

models/tokens/solana/tokens_solana_fees_history.sql

@@ -0,0 +1,61 @@
+ {{
+  config(
+        schema = 'tokens_solana',
+        alias = 'fees_history',
+        materialized = 'incremental',
+        file_format = 'delta',
+        incremental_strategy = 'merge',
+        incremental_predicates = [incremental_predicate('DBT_INTERNAL_DEST.fee_time')],
+        unique_key = ['account_mint','fee_time'],
+        post_hook='{{ expose_spells(\'["solana"]\',
+                                    "sector",
+                                    "tokens",
+                                    \'["ilemi"]\') }}')
+}}
+
+--we need the fee basis points and maximum fee for token2022 transfers because the fee amount is not emitted in transferChecked
+SELECT
+call_account_arguments[1] as account_mint
+, try(bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,
+            1+1+1+1+1+case when bytearray_substring(call_data,1+1+1,1) = 0x01 and bytearray_substring(call_data,1+1+1+32+1,1) = 0x01
+                        then 64
+                        when bytearray_substring(call_data,1+1+1,1) = 0x01 and bytearray_substring(call_data,1+1+1+32+1,1) = 0x00
+                        then 32
+                        when bytearray_substring(call_data,1+1+1,1) = 0x00 and bytearray_substring(call_data,1+1+1+1,1) = 0x01
+                        then 32
+                        when bytearray_substring(call_data,1+1+1,1) = 0x00 and bytearray_substring(call_data,1+1+1+1,1) = 0x00
+                        then 0
+                        end --variations of COPTION enums for first two arguments
+            ,2)))) as fee_basis
+, try(bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,
+            1+1+1+1+1+case when bytearray_substring(call_data,1+1+1,1) = 0x01 and bytearray_substring(call_data,1+1+1+32+1,1) = 0x01
+                        then 64
+                        when bytearray_substring(call_data,1+1+1,1) = 0x01 and bytearray_substring(call_data,1+1+1+32+1,1) = 0x00
+                        then 32
+                        when bytearray_substring(call_data,1+1+1,1) = 0x00 and bytearray_substring(call_data,1+1+1+1,1) = 0x01
+                        then 32
+                        when bytearray_substring(call_data,1+1+1,1) = 0x00 and bytearray_substring(call_data,1+1+1+1,1) = 0x00
+                        then 0
+                        end
+                +2
+            ,16)))) as fee_maximum
+, call_block_time as fee_time
+FROM {{ source('spl_token_2022_solana','spl_token_2022_call_transferFeeExtension') }}
+WHERE bytearray_substring(call_data,1+1,1) = 0x00 --https://github.com/solana-labs/solana-program-library/blob/8f50c6fabc6ec87ada229e923030381f573e0aed/token/program-2022/src/extension/transfer_fee/instruction.rs#L38
+{% if is_incremental() %}
+AND {{incremental_predicate('call_block_time')}}
+{% endif %}
+
+UNION ALL
+SELECT
+call_account_arguments[1] as account_mint
+, try(bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,
+            1+1+1,2)))) as fee_basis
+, try(bytearray_to_uint256(bytearray_reverse(bytearray_substring(call_data,
+            1+1+1+2,16)))) as fee_maximum
+, call_block_time as fee_time
+FROM {{ source('spl_token_2022_solana','spl_token_2022_call_transferFeeExtension') }}
+WHERE bytearray_substring(call_data,1+1,1) = 0x05 --https://github.com/solana-labs/solana-program-library/blob/8f50c6fabc6ec87ada229e923030381f573e0aed/token/program-2022/src/extension/transfer_fee/instruction.rs#L147
+{% if is_incremental() %}
+AND {{incremental_predicate('call_block_time')}}
+{% endif %}

models/tokens/solana/tokens_solana_fungible.sql

@@ -1,7 +1,7 @@
 {{ config
 (
   alias = 'fungible',
-  
+
   post_hook='{{ expose_spells(\'["solana"]\',
                                   "sector",
                                   "tokens",
@@ -10,26 +10,32 @@
 }}
 
 
-with 
+with
     tokens as (
         SELECT
         bytearray_to_bigint(bytearray_reverse(bytearray_substring(call_data, 2, 1))) as decimals
         , call_data
         , account_mint
+        , token_version
         , call_tx_id
         , call_block_time
+        , row_number() over (partition by account_mint order by call_block_time desc) as latest
         FROM (
-            SELECT call_data, account_mint, call_tx_id, call_block_time FROM {{ source('spl_token_solana', 'spl_token_call_initializeMint') }}
-            UNION ALL 
-            SELECT call_data, account_mint, call_tx_id, call_block_time FROM {{ source('spl_token_solana', 'spl_token_call_initializeMint2') }}
+            SELECT call_data, account_mint, call_tx_id, call_block_time, 'spl_token' as token_version FROM {{ source('spl_token_solana', 'spl_token_call_initializeMint') }}
+            UNION ALL
+            SELECT call_data, account_mint, call_tx_id, call_block_time, 'spl_token' as token_version FROM {{ source('spl_token_solana', 'spl_token_call_initializeMint2') }}
+            UNION ALL
+            SELECT call_data, account_mint, call_tx_id, call_block_time, 'token2022' as token_version FROM {{ source('spl_token_2022_solana', 'spl_token_2022_call_initializeMint') }}
+            UNION ALL
+            SELECT call_data, account_mint, call_tx_id, call_block_time, 'token2022' as token_version FROM {{ source('spl_token_2022_solana', 'spl_token_2022_call_initializeMint2') }}
         )
         {% if is_incremental() %}
-        where call_block_time >= date_trunc('day', now() - interval '7' day)
+        where {{ incremental_predicate('call_block_time') }}
         {% endif %}
     )
-    
+
     , metadata as (
-        SELECT 
+        SELECT
             meta.call_tx_id
             , meta.call_block_slot
             , meta.call_block_time
@@ -38,8 +44,10 @@ with
             , meta.account_mint
             , meta.call_block_time
             , master.account_edition as master_edition
+            , metadata_program
+            , row_number() over (partition by meta.account_mint order by meta.call_block_time desc) as latest
         FROM (
-            SELECT 
+            SELECT
                 call_tx_id
                 , call_outer_instruction_index
                 , call_inner_instruction_index
@@ -48,9 +56,10 @@ with
                 , json_query(createMetadataAccountArgs, 'lax $.CreateMetadataAccountArgs.data.Data') as args
                 , account_metadata
                 , account_mint
+                , call_executing_account as metadata_program
             FROM  {{ source('mpl_token_metadata_solana', 'mpl_token_metadata_call_CreateMetadataAccount') }}
-            UNION ALL 
-            SELECT 
+            UNION ALL
+            SELECT
                 call_tx_id
                 , call_outer_instruction_index
                 , call_inner_instruction_index
@@ -59,51 +68,143 @@ with
                 , json_query(createMetadataAccountArgsV2, 'lax $.CreateMetadataAccountArgsV2.data.DataV2') as args
                 , account_metadata
                 , account_mint
+                , call_executing_account as metadata_program
             FROM {{ source('mpl_token_metadata_solana', 'mpl_token_metadata_call_CreateMetadataAccountV2') }}
-            UNION ALL 
-            SELECT  
+            UNION ALL
+            SELECT
                 call_tx_id
                 , call_outer_instruction_index
-                , call_inner_instruction_index                    
+                , call_inner_instruction_index
                 , call_block_slot
                 , call_block_time
                 , json_query(createMetadataAccountArgsV3, 'lax $.CreateMetadataAccountArgsV3.data.DataV2') as args
                 , account_metadata
                 , account_mint
-            FROM {{ source('mpl_token_metadata_solana', 'mpl_token_metadata_call_CreateMetadataAccountV3') }} 
-        ) meta 
+                , call_executing_account as metadata_program
+            FROM {{ source('mpl_token_metadata_solana', 'mpl_token_metadata_call_CreateMetadataAccountV3') }}
+        ) meta
         LEFT JOIN (
-            SELECT account_mintAuthority, account_edition, account_metadata FROM {{ source('mpl_token_metadata_solana', 'mpl_token_metadata_call_CreateMasterEdition') }} 
+            SELECT account_mintAuthority, account_edition, account_metadata FROM {{ source('mpl_token_metadata_solana', 'mpl_token_metadata_call_CreateMasterEdition') }}
             UNION ALL
             SELECT account_mintAuthority, account_edition, account_metadata FROM {{ source('mpl_token_metadata_solana', 'mpl_token_metadata_call_CreateMasterEditionV3') }}
             ) master ON master.account_metadata = meta.account_metadata
         {% if is_incremental() %}
-        WHERE meta.call_block_time >= date_trunc('day', now() - interval '7' day)
+        WHERE {{ incremental_predicate('meta.call_block_time') }}
+        {% endif %}
+    )
+
+    , token2022_metadata as (
+        --token2022 direct metadata extension
+        SELECT
+            from_utf8(bytearray_substring(data,1+8+4,bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+8,4))))) as name
+            , from_utf8(bytearray_substring(data,1+8+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+8,4))) + 4 --start from end of name and end of length of symbol
+                , bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+8+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+8,4))),4))) --get length of symbol from end of name
+                )) as symbol
+            , from_utf8(bytearray_substring(data,1+8+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+8,4))) + 4 --end of name and end of length of symbol
+                    + bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+8+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+8,4))),4))) + 4 --start from end of symbol and end of length of uri
+                , bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+8+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+8,4))) + 4
+                    + bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+8+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+8,4))),4))),4))) --get length of uri from end of symbol
+                )) as uri
+            , tx_id as metadata_tx
+            , account_arguments[3] as account_mint
+            , block_time
+            , executing_account as metadata_program
+            , row_number() over (partition by account_arguments[3] order by block_time desc) as latest
+        FROM {{ source('solana','instruction_calls') }}
+        WHERE executing_account = 'TokenzQdBNbLqP5VEhdkAS6EPFLC1PHnBqCXEpPxuEb'
+        AND bytearray_substring(data,1,1) = 0xd2 --deal with updateField later 0xdd
+        AND tx_success
+        {% if is_incremental() %}
+        AND {{ incremental_predicate('block_time') }}
+        {% endif %}
+    )
+
+    , token_metadata_other as (
+        --some other metadata program (idk the owner)
+        SELECT
+            from_utf8(bytearray_substring(data,1+1+4,bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+1,4))))) as name
+            , from_utf8(bytearray_substring(data,1+1+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+1,4))) + 4 --start from end of name and end of length of symbol
+                , bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+1+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+1,4))),4))) --get length of symbol from end of name
+                )) as symbol
+            , from_utf8(bytearray_substring(data,1+1+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+1,4))) + 4 --end of name and end of length of symbol
+                    + bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+1+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+1,4))),4))) + 4 --start from end of symbol and end of length of uri
+                , bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+1+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+1,4))) + 4
+                    + bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+1+4+bytearray_to_bigint(bytearray_reverse(bytearray_substring(data,1+1,4))),4))),4))) --get length of uri from end of symbol
+                )) as uri
+            , tx_id as metadata_tx
+            , account_arguments[2] as account_mint
+            , block_time
+            , executing_account as metadata_program
+            , row_number() over (partition by account_arguments[2] order by block_time desc) as latest
+        FROM {{ source('solana','instruction_calls') }}
+        WHERE executing_account = 'META4s4fSmpkTbZoUsgC1oBnWB31vQcmnN8giPw51Zu'
+        AND bytearray_substring(data,1,1) = 0x21
+        AND tx_success
+        {% if is_incremental() %}
+        AND {{ incremental_predicate('block_time') }}
         {% endif %}
     )
 
 SELECT
     tk.account_mint as token_mint_address
     , tk.decimals
-    , trim(json_value(args, 'strict $.name'))as name 
-    , trim(json_value(args, 'strict $.symbol')) as symbol 
-    , trim(json_value(args, 'strict $.uri')) as token_uri
+    , coalesce(m22.name,mo.name,trim(json_value(args, 'strict $.name'))) as name
+    , coalesce(m22.symbol,mo.symbol,trim(json_value(args, 'strict $.symbol'))) as symbol
+    , coalesce(m22.uri,mo.uri,trim(json_value(args, 'strict $.uri'))) as token_uri
     , tk.call_block_time as created_at
+    , coalesce(m22.metadata_program,mo.metadata_program,m.metadata_program) as metadata_program
+    , tk.token_version
+    , tk.call_tx_id as init_tx
 FROM tokens tk
-LEFT JOIN metadata m ON tk.account_mint = m.account_mint
+LEFT JOIN token2022_metadata m22 ON tk.account_mint = m22.account_mint AND m22.latest = 1
+LEFT JOIN token_metadata_other mo ON tk.account_mint = mo.account_mint AND mo.latest = 1
+LEFT JOIN metadata m ON tk.account_mint = m.account_mint AND m.latest = 1
 WHERE m.master_edition is null
+AND tk.latest = 1
+
+UNION ALL
+
+--token2022 wrapped sol https://solscan.io/tx/2L1o7sDMCMJ6PYqfNrnY6ozJC1DEx61pRYiLdfCCggxw81naQXsmHKDLn6EhJXmDmDSQ2eCKjUMjZAQuUsyNnYUv
+SELECT
+  trim(token_mint_address) as token_mint_address
+  , decimals
+  , trim(name) as name
+  , trim(symbol) as symbol
+  , token_uri
+  , cast(created_at as timestamp) created_at
+  , metadata_program
+  , token_version
+  , init_tx
+FROM
+(
+  VALUES
+(
+  '9pan9bMn5HatX4EJdBwg9VgCa7Uz5HL8N1m5D3NdXejP',
+  9,
+  'wrapped SOL',
+  'SOL',
+  null,
+  '2023-08-02 00:00:00',
+  null,
+  'token2022',
+  '2L1o7sDMCMJ6PYqfNrnY6ozJC1DEx61pRYiLdfCCggxw81naQXsmHKDLn6EhJXmDmDSQ2eCKjUMjZAQuUsyNnYUv'
+)
+) AS temp_table (token_mint_address, decimals, name, symbol, token_uri, created_at, metadata_program, token_version, init_tx)
 
 UNION ALL
 
---wrapped sol is special and doesn't have a init tx (that I can find)
-SELECT 
+--old wrapped sol is special and doesn't have a init tx (that I can find)
+SELECT
   trim(token_mint_address) as token_mint_address
   , decimals
   , trim(name) as name
   , trim(symbol) as symbol
   , token_uri
   , cast(created_at as timestamp) created_at
-FROM 
+  , metadata_program
+  , token_version
+  , init_tx
+FROM
 (
   VALUES
 (
@@ -112,6 +213,9 @@ FROM
   'wrapped SOL',
   'SOL',
   null,
-  '2021-01-31 00:00:00'
+  '2021-01-31 00:00:00',
+  null,
+  'spl_token',
+  null
 )
-) AS temp_table (token_mint_address, decimals, name, symbol, token_uri, created_at)
+) AS temp_table (token_mint_address, decimals, name, symbol, token_uri, created_at, metadata_program, token_version, init_tx)

models/tokens/solana/tokens_solana_schema.yml

@@ -19,8 +19,33 @@ models:
         description: "token symbol"
       - name: decimals
         description: "Number of decimals, refers to how divisible a token can be"
+      - name: metadata_program
+        description: program used for creating token metadata
+      - name: token_version
+        description: version of the token program used (spl_token, token2022)
       - name: created_at
         description: token mint created at
+      - name: init_tx
+        description: "transaction that initialized the mint"
+
+  - name: tokens_solana_fees_history
+    meta:
+      blockchain: solana
+      sector: tokens
+      project: fees
+      contributors: ilemi
+    config:
+      tags: ['table', 'metadata', 'fess', 'solana']
+    description: "fee updates on token2022 tokens"
+    columns:
+      - name: account_mint
+        description: "fungible token mint address on Solana"
+      - name: fee_basis
+        description: basis points fee on transfers
+      - name: fee_maximum
+        description: maximum fee amount on a given transfer
+      - name: fee_time
+        description: fee update time
 
   - name: tokens_solana_nft
     meta: