misc(worker): Replace Sidekiq with SolidQueue

Dockerfile

@@ -15,6 +15,11 @@ RUN gem install bundler --no-document -v '2.5.5'
 RUN bundle config build.nokogiri --use-system-libraries &&\
   bundle install --jobs=3 --retry=3 --without development test
 
+# Temporary fix for mission_control-jobs
+# See https://github.com/rails/mission_control-jobs/issues/60#issuecomment-2486452545
+COPY . /app
+RUN RAILS_ENV=production SECRET_KEY_BASE_DUMMY=1 ./bin/rails assets:precompile
+
 FROM ruby:3.3.4-slim
 
 WORKDIR /app
@@ -32,5 +37,6 @@ ENV GOCARDLESS_CLIENT_ID=$GOCARDLESS_CLIENT_ID
 ENV GOCARDLESS_CLIENT_SECRET=$GOCARDLESS_CLIENT_SECRET
 
 COPY --from=build /usr/local/bundle/ /usr/local/bundle
+COPY --from=build /app/public/assets /app/public/assets
 
 CMD ["./scripts/start.sh"]

Gemfile

@@ -15,6 +15,7 @@ gem "puma", "~> 6.4"
 gem "rails", "~> 7.1.3.4"
 gem "redis"
 gem "sidekiq"
+gem 'solid_queue', "~> 1.0.2"
 gem "dry-validation"
 
 # Security
@@ -36,6 +37,10 @@ gem "scenic"
 gem "with_advisory_lock"
 gem "strong_migrations"
 
+# Job monitoring
+gem "mission_control-jobs", '~> 0.6.0'
+gem "propshaft"
+
 # Currencies, Countries, Timezones...
 gem "bigdecimal"
 gem "countries"

Gemfile.lock

@@ -232,6 +232,8 @@ GEM
       dry-schema (>= 1.12, < 2)
       zeitwerk (~> 2.6)
     erubi (1.13.0)
+    et-orbi (1.2.11)
+      tzinfo
     execjs (2.9.1)
     factory_bot (6.4.6)
       activesupport (>= 5.0.0)
@@ -251,6 +253,9 @@ GEM
     ffi (1.17.0-x86_64-linux-gnu)
     fiber-storage (1.0.0)
     formatador (1.1.0)
+    fugit (1.11.1)
+      et-orbi (~> 1, >= 1.2.11)
+      raabro (~> 1.4)
     globalid (1.2.1)
       activesupport (>= 6.1)
     gocardless_pro (2.57.0)
@@ -320,6 +325,10 @@ GEM
     httpclient (2.8.3)
     i18n (1.14.6)
       concurrent-ruby (~> 1.0)
+    importmap-rails (2.0.3)
+      actionpack (>= 6.0.0)
+      activesupport (>= 6.0.0)
+      railties (>= 6.0.0)
     io-console (0.7.2)
     irb (1.14.0)
       rdoc (>= 4.0.0)
@@ -376,6 +385,16 @@ GEM
     mini_mime (1.1.5)
     mini_portile2 (2.8.7)
     minitest (5.25.1)
+    mission_control-jobs (0.6.0)
+      actioncable (>= 7.1)
+      actionpack (>= 7.1)
+      activejob (>= 7.1)
+      activerecord (>= 7.1)
+      importmap-rails (>= 1.2.1)
+      irb (~> 1.13)
+      railties (>= 7.1)
+      stimulus-rails
+      turbo-rails
     monetize (1.13.0)
       money (~> 6.12)
     money (6.19.0)
@@ -631,6 +650,11 @@ GEM
       racc
     pg (1.5.7)
     prism (1.2.0)
+    propshaft (1.1.0)
+      actionpack (>= 7.0.0)
+      activesupport (>= 7.0.0)
+      rack
+      railties (>= 7.0.0)
     pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
@@ -639,6 +663,7 @@ GEM
     public_suffix (5.1.1)
     puma (6.4.3)
       nio4r (~> 2.0)
+    raabro (1.4.0)
     racc (1.8.1)
     rack (2.2.9)
     rack-cors (1.1.1)
@@ -837,6 +862,13 @@ GEM
     snaky_hash (2.0.1)
       hashie
       version_gem (~> 1.1, >= 1.1.1)
+    solid_queue (1.0.2)
+      activejob (>= 7.1)
+      activerecord (>= 7.1)
+      concurrent-ruby (>= 1.3.1)
+      fugit (~> 1.11.0)
+      railties (>= 7.1)
+      thor (~> 1.3.1)
     sorbet-runtime (0.5.11602)
     sprockets (4.2.1)
       concurrent-ruby (~> 1.0)
@@ -857,6 +889,8 @@ GEM
     standard-performance (1.4.0)
       lint_roller (~> 1.1)
       rubocop-performance (~> 1.21.0)
+    stimulus-rails (1.3.4)
+      railties (>= 6.0.0)
     stringio (3.1.1)
     stripe (6.5.0)
     strong_migrations (2.0.0)
@@ -869,6 +903,9 @@ GEM
     timecop (0.9.10)
     timeout (0.4.2)
     trailblazer-option (0.1.2)
+    turbo-rails (2.0.11)
+      actionpack (>= 6.0.0)
+      railties (>= 6.0.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     uber (0.1.0)
@@ -950,6 +987,7 @@ DEPENDENCIES
   lago-expression!
   lograge
   logstash-event
+  mission_control-jobs (~> 0.6.0)
   money-rails
   multipart-post
   mutex_m
@@ -961,6 +999,7 @@ DEPENDENCIES
   paper_trail (~> 15.1)
   parallel
   pg
+  propshaft
   puma (~> 6.4)
   rack-cors
   rails (~> 7.1.3.4)
@@ -984,6 +1023,7 @@ DEPENDENCIES
   simplecov
   slim
   slim-rails
+  solid_queue (~> 1.0.2)
   standard
   stripe
   strong_migrations

app/controllers/concerns/api_responses.rb

@@ -21,6 +21,8 @@ def not_found_error(resource:)
   protected
 
   def set_json_format
+    return if request.path.start_with?('/jobs')
+
     request.format = :json
   end
 end

app/jobs/application_job.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
 class ApplicationJob < ActiveJob::Base
+  # TODO: enable this with Rails 8
+  # self.enqueue_after_transaction_commit = true
+
   sidekiq_options retry: 0
 end

app/jobs/clock/activate_subscriptions_job.rb

@@ -5,8 +5,7 @@ class ActivateSubscriptionsJob < ApplicationJob
     include SentryCronConcern
 
     queue_as 'clock'
-
-    unique :until_executed, on_conflict: :log
+    limits_concurrency to: 1, key: 'activate_subscription', duration: 1.hour
 
     def perform
       Subscriptions::ActivateService.new(timestamp: Time.current.to_i).activate_all_pending

app/jobs/clock/events_validation_job.rb

@@ -5,10 +5,11 @@ class EventsValidationJob < ApplicationJob
     include SentryCronConcern
 
     queue_as 'clock'
-
-    unique :until_executed
+    limits_concurrency to: 1, key: 'post_validate_events', duration: 1.hour
 
     def perform
+      return if ActiveModel::Type::Boolean.new.cast(ENV['LAGO_DISABLE_EVENTS_VALIDATION'])
+
       # NOTE: refresh the last hour events materialized view
       Scenic.database.refresh_materialized_view(
         Events::LastHourMv.table_name,

app/jobs/clock/process_dunning_campaigns_job.rb

@@ -5,8 +5,7 @@ class ProcessDunningCampaignsJob < ApplicationJob
     include SentryCronConcern
 
     queue_as 'clock'
-
-    unique :until_executed, on_conflict: :log
+    limits_concurrency to: 1, key: 'process_dunning_campaign', duration: 1.hour
 
     def perform
       return unless License.premium?

app/jobs/clock/refresh_draft_invoices_job.rb

@@ -5,8 +5,7 @@ class RefreshDraftInvoicesJob < ApplicationJob
     include SentryCronConcern
 
     queue_as 'clock'
-
-    unique :until_executed, on_conflict: :log
+    limits_concurrency to: 1, key: 'refresh_draft_invoices', duration: 5.minutes
 
     def perform
       Invoice.ready_to_be_refreshed.with_active_subscriptions.find_each do |invoice|

app/jobs/clock/refresh_lifetime_usages_job.rb

@@ -6,6 +6,8 @@ class RefreshLifetimeUsagesJob < ApplicationJob
 
     queue_as 'clock'
     unique :until_executed, on_conflict: :log
+    limits_concurrency to: 1, key: 'refresh_lifetime_usage',
+      duration: (ENV["LAGO_LIFETIME_USAGE_REFRESH_INTERVAL_SECONDS"].presence || 5.minutes).to_i.seconds
 
     def perform
       return unless License.premium?

app/jobs/clock/refresh_wallets_ongoing_balance_job.rb

@@ -5,11 +5,12 @@ class RefreshWalletsOngoingBalanceJob < ApplicationJob
     include SentryCronConcern
 
     queue_as 'clock'
-
-    unique :until_executed, on_conflict: :log
+    limits_concurrency to: 1, key: 'refresh_wallets_ongoing_balance', duration: 5.minutes
 
     def perform
       return unless License.premium?
+      return if ActiveModel::Type::Boolean.new.cast(ENV['LAGO_DISABLE_WALLET_REFRESH'])
+      return unless ENV['LAGO_MEMCACHE_SERVERS'].present? || ENV['LAGO_REDIS_CACHE_URL'].present?
 
       Wallet.active.ready_to_be_refreshed.find_each do |wallet|
         Wallets::RefreshOngoingBalanceJob.perform_later(wallet)

app/jobs/events/pay_in_advance_job.rb

@@ -3,7 +3,7 @@
 module Events
   class PayInAdvanceJob < ApplicationJob
     queue_as do
-      if ActiveModel::Type::Boolean.new.cast(ENV['SIDEKIQ_EVENTS'])
+      if ActiveModel::Type::Boolean.new.cast(ENV['LAGO_WORKER_EVENTS'])
         :events
       else
         :default

app/jobs/events/pay_in_advance_kafka_job.rb

@@ -3,7 +3,7 @@
 module Events
   class PayInAdvanceKafkaJob < ApplicationJob
     queue_as do
-      if ActiveModel::Type::Boolean.new.cast(ENV['SIDEKIQ_EVENTS'])
+      if ActiveModel::Type::Boolean.new.cast(ENV['LAGO_WORKER_EVENTS'])
         :events
       else
         :default

app/jobs/events/post_process_job.rb

@@ -3,7 +3,7 @@
 module Events
   class PostProcessJob < ApplicationJob
     queue_as do
-      if ActiveModel::Type::Boolean.new.cast(ENV['SIDEKIQ_EVENTS'])
+      if ActiveModel::Type::Boolean.new.cast(ENV['LAGO_WORKER_EVENTS'])
         :events
       else
         :default

app/jobs/events/post_validation_job.rb

@@ -3,7 +3,7 @@
 module Events
   class PostValidationJob < ApplicationJob
     queue_as do
-      if ActiveModel::Type::Boolean.new.cast(ENV['SIDEKIQ_EVENTS'])
+      if ActiveModel::Type::Boolean.new.cast(ENV['LAGO_WORKER_EVENTS'])
         :events
       else
         :default

app/jobs/invoices/generate_pdf_and_notify_job.rb

@@ -3,7 +3,7 @@
 module Invoices
   class GeneratePdfAndNotifyJob < ApplicationJob
     queue_as do
-      if ActiveModel::Type::Boolean.new.cast(ENV['SIDEKIQ_PDFS'])
+      if ActiveModel::Type::Boolean.new.cast(ENV['LAGO_WORKER_PDFS'])
         :pdfs
       else
         :invoices

app/jobs/invoices/generate_pdf_job.rb

@@ -3,7 +3,7 @@
 module Invoices
   class GeneratePdfJob < ApplicationJob
     queue_as do
-      if ActiveModel::Type::Boolean.new.cast(ENV['SIDEKIQ_PDFS'])
+      if ActiveModel::Type::Boolean.new.cast(ENV['LAGO_WORKER_PDFS'])
         :pdfs
       else
         :invoices

bin/jobs

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+require_relative "../config/environment"
+require "solid_queue/cli"
+
+SolidQueue::Cli.start(ARGV)