Merge branch 'main' into pr/6179

.github/workflows/commit_manifest.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: [ self-hosted, linux, spellbook-trino-ci ]
     strategy:
       matrix:
-        project: [ 'tokens', 'spellbook', 'daily_spellbook', 'nft']
+        project: [ 'tokens', 'spellbook', 'daily_spellbook', 'nft', 'dex']
       max-parallel: 1
 
     steps:

.github/workflows/dex.yml

@@ -0,0 +1,19 @@
+name: DBT DEX sector run
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - dex/**
+      - .github/workflows/dex.yml
+      - .github/workflows/dbt_run.yml
+
+concurrency:
+  group:  ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  dbt-run:
+    uses: ./.github/workflows/dbt_run.yml
+    with:
+      project: 'dex'

dbt_project.yml

@@ -86,23 +86,6 @@ models:
       bitcoin:
         +schema: cex_bitcoin
 
-    uniswap:
-      +schema: uniswap
-      arbitrum:
-        +schema: uniswap_arbitrum
-      ethereum:
-        +schema: uniswap_ethereum
-      optimism:
-        +schema: uniswap_optimism
-      polygon:
-        +schema: uniswap_polygon
-      bnb:
-        +schema: uniswap_bnb
-      base:
-        +schema: uniswap_base
-      celo:
-        +schema: uniswap_celo
-
     balances:
       +schema: balances
       ethereum:
@@ -114,21 +97,6 @@ models:
       polygon:
         +schema: balances_polygon
 
-    dodo:
-      +schema: dodo
-      ethereum:
-        +schema: dodo_ethereum
-      bnb:
-        +schema: dodo_bnb
-      polygon:
-        +schema: dodo_polygon
-      arbitrum:
-        +schema: dodo_arbitrum
-      optimism:
-        +schema: dodo_optimism
-      base:
-        +schema: dodo_base
-
     labels:
       +schema: labels
       +materialized: table
@@ -150,11 +118,6 @@ models:
     solana_utils:
       +schema: solana_utils
 
-    odos:
-      +schema: odos
-      avalanche_c:
-        +schema: odos_avalanche_c
-
     transfers:
       +schema: transfers
       ethereum:
@@ -200,13 +163,6 @@ models:
       base:
         +schema: safe_base
 
-    lifi:
-      +schema: lifi
-      fantom:
-        +schema: lifi_fantom
-      optimism:
-        +schema: lifi_optimism
-
     aragon:
       +schema: aragon
       ethereum:
@@ -220,11 +176,6 @@ models:
       arbitrum:
         +schema: aragon_arbitrum
 
-    spiritswap:
-      +schema: spiritswap
-      fantom:
-        +schema: spiritswap_fantom
-
     jarvis_network:
       +schema: jarvis_network
       polygon:
@@ -240,11 +191,6 @@ models:
       bnb:
         +schema: spaceid_bnb
 
-    spartacus_exchange:
-      +schema: spartacus_exchange
-      fantom:
-        +schema: spartacus_exchange_fantom
-
     tornado_cash:
       +schema: tornado_cash
       arbitrum:
@@ -302,15 +248,6 @@ models:
       ethereum:
         +schema: fiat_dao_ethereum
 
-    curvefi:
-      +schema: curvefi
-      ethereum:
-        +schema: curvefi_ethereum
-      optimism:
-        +schema: curvefi_optimism
-      fantom:
-        +schema: curvefi_fantom
-
     cow_protocol:
       +schema: cow_protocol
       ethereum:
@@ -340,21 +277,6 @@ models:
       avalanche_c:
         +schema: yield_yak_avalanche_c
 
-    kyberswap:
-      +schema: kyberswap
-      avalanche_c:
-        +schema: kyberswap_avalanche_c
-      optimism:
-        +schema: kyberswap_optimism
-      ethereum:
-        +schema: kyberswap_ethereum
-      arbitrum:
-        +schema: kyberswap_arbitrum
-      bnb:
-        +schema: kyberswap_bnb
-      polygon:
-        +schema: kyberswap_polygon
-
     hashflow:
       +schema: hashflow
       ethereum:
@@ -369,25 +291,6 @@ models:
       ethereum:
         +schema: nexusmutual_ethereum
 
-    zeroex:
-      +schema: zeroex
-      ethereum:
-        +schema: zeroex_ethereum
-      optimism:
-        +schema: zeroex_optimism
-      polygon:
-        +schema: zeroex_polygon
-      avalanche_c:
-        +schema: zeroex_avalanche_c
-      fantom:
-        +schema: zeroex_fantom
-      arbitrum:
-        +schema: zeroex_arbitrum
-      bnb:
-        +schema: zeroex_bnb
-      celo:
-        +schema: zeroex_celo
-
 
     ampleforth:
       +schema: ampleforth
@@ -471,11 +374,6 @@ models:
       ethereum:
         +schema: tokenfy_ethereum
 
-    tokenlon:
-      +schema: tokenlon
-      ethereum:
-        +schema: tokenlon_ethereum
-
     thales:
       +schema: thales
       ethereum:
@@ -569,11 +467,6 @@ models:
       polygon:
         +schema: rocifi_polygon
 
-    oneinch:
-      +schema: oneinch
-      ethereum:
-        +schema: oneinch_ethereum
-
     pooltogether:
       +schema: pooltogether
       ethereum:
@@ -628,15 +521,6 @@ models:
       base:
         +schema: layerzero_base
 
-    bebop:
-      +schema: bebop
-      ethereum:
-        +schema: bebop_ethereum
-      polygon:
-        +schema: bebop_polygon
-      arbitrum:
-        +schema: bebop_arbitrum
-
     tokemak:
       +schema: tokemak
       ethereum:

dex/README.md

@@ -0,0 +1,11 @@
+## DEX subproject
+
+This is a DBT subproject for the main lineages of the DEX sector. Included in this subproject, but not limited to over time:
+- `dex.trades`
+- `dex_aggreagtor.trades`
+- `dex.prices`
+- `dex.sandwiches`
+- `dex.sandwiched`
+- ...and more!
+
+This subproject will be dedicated to building the above spells (and others in the future related to DEX) on an hourly cadence in production. All other spells not included within this subproject will treat these spells as sources. For example, labels spells which read from `dex.trades` will now treat the spell as a source, rather than reference within dbt.

dex/dbt_project.yml

@@ -0,0 +1,67 @@
+#Welcome to your Spellbook!
+
+name: "dex"
+version: "1.0.0"
+config-version: 2
+
+quoting:
+  database: false
+  schema: false
+  identifier: false
+
+# This setting configures which "profile" dbt uses for this project.
+profile: "spellbook-local"
+
+vars:
+  DBT_ENV_CUSTOM_ENV_S3_BUCKET: "{{ env_var('DBT_ENV_CUSTOM_ENV_S3_BUCKET', 'local') }}"
+  DBT_ENV_INCREMENTAL_TIME: "{{ env_var('DBT_ENV_INCREMENTAL_TIME', '1') }}"
+  DBT_ENV_INCREMENTAL_TIME_UNIT: "{{ env_var('DBT_ENV_INCREMENTAL_TIME_UNIT', 'day') }}"
+
+# These configurations specify where dbt should look for different types of files.
+# You don't need to change these!
+model-paths: ["models", "../sources"]
+analysis-paths: ["analyses"]
+test-paths: ["tests"]
+seed-paths: ["seeds"]
+macro-paths: ["macros", "../macros"]
+snapshot-paths: ["snapshots"]
+
+target-path: "target" # directory which will store compiled SQL files
+clean-targets: # directories to be removed by `dbt clean`
+  - "target"
+  - "dbt_packages"
+
+# Configuring tests
+# Full documentation: https://docs.getdbt.com/reference/test-configs
+tests:
+  dex:
+    +store_failures: true  # store failures for all tests
+
+# Configuring seeds
+# Full documentation: https://docs.getdbt.com/reference/seed-configs
+# For configuring individual seeds (e.g. overriding column types) we recommend a yml file nested in the same folder as the seed
+seeds:
+  dex:
+    +schema: test_data
+
+# defining search order of macro invokation
+dispatch:
+  - macro_namespace: dbt_utils
+    search_order: ['trino_utils', 'dbt_utils']
+
+# Configuring models
+# Full documentation: https://docs.getdbt.com/docs/configuring-models
+models:
+  dex:
+    +post-hook:
+      - sql: "{{ set_trino_session_property(is_materialized(model), 'writer_scaling_min_data_processed', model.config.get('writer_min_size', '500MB')) }}"
+        transaction: true
+      - sql: "{{ set_trino_session_property(is_materialized(model), 'task_scale_writers_enabled', false) }}"
+        transaction: true
+      - sql: "{{ optimize_spell(this, model.config.materialized) }}"
+        transaction: true
+      - sql: "{{ mark_as_spell(this, model.config.materialized) }}"
+        transaction: true
+    +materialized: view
+    +schema: no_schema    # this should be overriden in model specific configs
+    +view_security: invoker