Adjust queues to prioritize account syncs (#1682)

maybe-finance · Jan 24, 2025 · 3140835 · 3140835
1 parent 7d04ea1
commit 3140835
Show file tree

Hide file tree

Showing 12 changed files with 37 additions and 12 deletions.
diff --git a/app/jobs/auto_upgrade_job.rb b/app/jobs/auto_upgrade_job.rb
@@ -1,5 +1,5 @@
 class AutoUpgradeJob < ApplicationJob
-  queue_as :default
+  queue_as :latency_low
 
   def perform(*args)
     raise_if_disabled

diff --git a/app/jobs/destroy_job.rb b/app/jobs/destroy_job.rb
@@ -1,5 +1,5 @@
 class DestroyJob < ApplicationJob
-  queue_as :default
+  queue_as :latency_low
 
   def perform(model)
     model.destroy

diff --git a/app/jobs/enrich_data_job.rb b/app/jobs/enrich_data_job.rb
@@ -1,5 +1,5 @@
 class EnrichDataJob < ApplicationJob
-  queue_as :default
+  queue_as :latency_high
 
   def perform(account)
     account.enrich_data

diff --git a/app/jobs/fetch_security_info_job.rb b/app/jobs/fetch_security_info_job.rb
@@ -1,5 +1,5 @@
 class FetchSecurityInfoJob < ApplicationJob
-  queue_as :default
+  queue_as :latency_low
 
   def perform(security_id)
     return unless Security.security_info_provider.present?

diff --git a/app/jobs/import_job.rb b/app/jobs/import_job.rb
@@ -1,5 +1,5 @@
 class ImportJob < ApplicationJob
-  queue_as :default
+  queue_as :latency_medium
 
   def perform(import)
     import.publish

diff --git a/app/jobs/sync_job.rb b/app/jobs/sync_job.rb
@@ -1,5 +1,5 @@
 class SyncJob < ApplicationJob
-  queue_as :default
+  queue_as :latency_medium
 
   def perform(sync)
     sync.perform

diff --git a/app/jobs/user_purge_job.rb b/app/jobs/user_purge_job.rb
@@ -1,5 +1,5 @@
 class UserPurgeJob < ApplicationJob
-  queue_as :default
+  queue_as :latency_low
 
   def perform(user)
     user.purge

diff --git a/app/models/time_series.rb b/app/models/time_series.rb
@@ -37,6 +37,14 @@ def trend
       series: self
   end
 
+  def empty?
+    values.empty?
+  end
+
+  def has_current_day_value?
+    values.any? { |v| v.date == Date.current }
+  end
+
   # `as_json` returns the data shape used by D3 charts
   def as_json
     {

diff --git a/app/views/accounts/chart.html.erb b/app/views/accounts/chart.html.erb
@@ -17,15 +17,19 @@
   </div>
 
   <div class="h-64">
-    <% if series %>
+    <% if series.has_current_day_value? %>
       <div
         id="lineChart"
         class="w-full h-full"
         data-controller="time-series-chart"
         data-time-series-chart-data-value="<%= series.to_json %>"></div>
+    <% elsif series.empty? %>
+      <div class="w-full h-full flex items-center justify-center">
+        <p class="text-gray-500 text-sm">No data available for the selected period.</p>
+      </div>
     <% else %>
       <div class="w-full h-full flex items-center justify-center">
-        <p class="text-gray-500">No data available for the selected period.</p>
+        <p class="text-gray-500 text-sm animate-pulse">Calculating latest balance data...</p>
       </div>
     <% end %>
   </div>

diff --git a/app/views/pages/dashboard/_net_worth_chart.html.erb b/app/views/pages/dashboard/_net_worth_chart.html.erb
@@ -1,12 +1,16 @@
 <%# locals: (series:) %>
-<% if series %>
+<% if series.has_current_day_value? %>
   <div
     id="netWorthChart"
     class="w-full flex-1 min-h-52"
     data-controller="time-series-chart"
     data-time-series-chart-data-value="<%= series.to_json %>"></div>
+<% elsif series.empty? %>
+  <div class="w-full h-full flex items-center justify-center">
+    <p class="text-gray-500 text-sm">No data available for the selected period.</p>
+  </div>
 <% else %>
   <div class="w-full h-full flex items-center justify-center">
-    <p class="text-gray-500">No data available for the selected period.</p>
+    <p class="text-gray-500 text-sm animate-pulse">Calculating latest balance data...</p>
   </div>
 <% end %>
diff --git a/config/database.yml b/config/database.yml
@@ -1,7 +1,8 @@
 default: &default
   adapter: postgresql
   encoding: unicode
-  pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %>
+  # 3 connections for Puma, 8 for GoodJob (in async mode, the default for self-hosters) = 11 connections
+  pool: <%= ENV.fetch("DB_POOL_SIZE") { 11 } %>
   host: <%= ENV.fetch("DB_HOST") { "127.0.0.1" } %>
   port: <%= ENV.fetch("DB_PORT") { "5432" } %>
   user: <%= ENV.fetch("POSTGRES_USER") { nil } %>

diff --git a/config/initializers/good_job.rb b/config/initializers/good_job.rb
@@ -11,6 +11,14 @@
     }
   end
 
+  # 5 queue threads + 3 for job listener, cron, executor = 8 threads allocated
+  config.queues = {
+    "latency_low" => { max_threads: 1, priority: 10 }, # ~30s jobs
+    "latency_low,latency_medium" => { max_threads: 2, priority: 5 }, # ~1-2 min jobs
+    "latency_low,latency_medium,latency_high" => { max_threads: 1, priority: 1 }, # ~5+ min jobs
+    "*" => { max_threads: 1, priority: 0 } # fallback queue
+  }
+
   # Auth for jobs admin dashboard
   ActiveSupport.on_load(:good_job_application_controller) do
     before_action do