From 7c6b0d38019450f33464423986f378b68854b360 Mon Sep 17 00:00:00 2001 From: Jonah George Date: Sun, 24 Dec 2023 11:38:08 -0800 Subject: [PATCH 1/2] WIP: Add MSSQL support --- .github/workflows/main.yml | 11 +- Gemfile | 3 + Gemfile.lock | 173 +++++++++++++++++-------------- app/models/solid_queue/record.rb | 6 +- docker-compose.yml | 12 ++- solid_queue.gemspec | 2 +- test/dummy/config/database.yml | 9 ++ 7 files changed, 131 insertions(+), 85 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9b7c9834..60d9bb71 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -9,7 +9,7 @@ jobs: fail-fast: false matrix: ruby-version: [3.2.2] - database: [mysql, postgres, sqlite] + database: [mysql, postgres, sqlite, mssql] services: mysql: image: mysql:8.0.31 @@ -24,11 +24,20 @@ jobs: POSTGRES_HOST_AUTH_METHOD: "trust" ports: - 55432:5432 + mssql: + image: mcr.microsoft.com/azure-sql-edge:1.0.4 + env: + SA_PASSWORD: yourStrongPassword123 + ACCEPT_EULA: Y + ports: + - 11433:1433 env: TARGET_DB: ${{ matrix.database }} steps: - name: Checkout code uses: actions/checkout@v3 + - name: Install native dependencies + run: sudo apt install freetds-dev freetds-bin - name: Setup Ruby and install gems uses: ruby/setup-ruby@v1 with: diff --git a/Gemfile b/Gemfile index 0461dab1..07a30a9b 100644 --- a/Gemfile +++ b/Gemfile @@ -7,3 +7,6 @@ gemspec gem "mysql2" gem "pg" gem "sqlite3" + +gem "activerecord-sqlserver-adapter", "~> 7.1.0" +gem 'tiny_tds' diff --git a/Gemfile.lock b/Gemfile.lock index 6e1eabeb..29aab81a 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -2,75 +2,79 @@ PATH remote: . specs: solid_queue (0.1.2) - rails (>= 7.0.3.1) + rails (>= 7.1.0) GEM remote: https://rubygems.org/ specs: - actioncable (7.1.0) - actionpack (= 7.1.0) - activesupport (= 7.1.0) + actioncable (7.1.2) + actionpack (= 7.1.2) + activesupport (= 7.1.2) nio4r (~> 2.0) websocket-driver (>= 0.6.1) zeitwerk (~> 2.6) - actionmailbox (7.1.0) - actionpack (= 7.1.0) - activejob (= 7.1.0) - activerecord (= 7.1.0) - activestorage (= 7.1.0) - activesupport (= 7.1.0) + actionmailbox (7.1.2) + actionpack (= 7.1.2) + activejob (= 7.1.2) + activerecord (= 7.1.2) + activestorage (= 7.1.2) + activesupport (= 7.1.2) mail (>= 2.7.1) net-imap net-pop net-smtp - actionmailer (7.1.0) - actionpack (= 7.1.0) - actionview (= 7.1.0) - activejob (= 7.1.0) - activesupport (= 7.1.0) + actionmailer (7.1.2) + actionpack (= 7.1.2) + actionview (= 7.1.2) + activejob (= 7.1.2) + activesupport (= 7.1.2) mail (~> 2.5, >= 2.5.4) net-imap net-pop net-smtp rails-dom-testing (~> 2.2) - actionpack (7.1.0) - actionview (= 7.1.0) - activesupport (= 7.1.0) + actionpack (7.1.2) + actionview (= 7.1.2) + activesupport (= 7.1.2) nokogiri (>= 1.8.5) + racc rack (>= 2.2.4) rack-session (>= 1.0.1) rack-test (>= 0.6.3) rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) - actiontext (7.1.0) - actionpack (= 7.1.0) - activerecord (= 7.1.0) - activestorage (= 7.1.0) - activesupport (= 7.1.0) + actiontext (7.1.2) + actionpack (= 7.1.2) + activerecord (= 7.1.2) + activestorage (= 7.1.2) + activesupport (= 7.1.2) globalid (>= 0.6.0) nokogiri (>= 1.8.5) - actionview (7.1.0) - activesupport (= 7.1.0) + actionview (7.1.2) + activesupport (= 7.1.2) builder (~> 3.1) erubi (~> 1.11) rails-dom-testing (~> 2.2) rails-html-sanitizer (~> 1.6) - activejob (7.1.0) - activesupport (= 7.1.0) + activejob (7.1.2) + activesupport (= 7.1.2) globalid (>= 0.3.6) - activemodel (7.1.0) - activesupport (= 7.1.0) - activerecord (7.1.0) - activemodel (= 7.1.0) - activesupport (= 7.1.0) + activemodel (7.1.2) + activesupport (= 7.1.2) + activerecord (7.1.2) + activemodel (= 7.1.2) + activesupport (= 7.1.2) timeout (>= 0.4.0) - activestorage (7.1.0) - actionpack (= 7.1.0) - activejob (= 7.1.0) - activerecord (= 7.1.0) - activesupport (= 7.1.0) + activerecord-sqlserver-adapter (7.1.0) + activerecord (~> 7.1.1) + tiny_tds + activestorage (7.1.2) + actionpack (= 7.1.2) + activejob (= 7.1.2) + activerecord (= 7.1.2) + activesupport (= 7.1.2) marcel (~> 1.0) - activesupport (7.1.0) + activesupport (7.1.2) base64 bigdecimal concurrent-ruby (~> 1.0, >= 1.0.2) @@ -80,27 +84,28 @@ GEM minitest (>= 5.1) mutex_m tzinfo (~> 2.0) - base64 (0.1.1) - bigdecimal (3.1.4) + base64 (0.2.0) + bigdecimal (3.1.5) builder (3.2.4) concurrent-ruby (1.2.2) connection_pool (2.4.1) crass (1.0.6) - date (3.3.3) - debug (1.7.1) - irb (>= 1.5.0) - reline (>= 0.3.1) - drb (2.1.1) + date (3.3.4) + debug (1.9.1) + irb (~> 1.10) + reline (>= 0.3.8) + drb (2.2.0) ruby2_keywords erubi (1.12.0) globalid (1.2.1) activesupport (>= 6.1) i18n (1.14.1) concurrent-ruby (~> 1.0) - io-console (0.6.0) - irb (1.6.2) - reline (>= 0.3.0) - loofah (2.21.4) + io-console (0.7.1) + irb (1.11.0) + rdoc + reline (>= 0.3.8) + loofah (2.22.0) crass (~> 1.0.2) nokogiri (>= 1.12.0) mail (2.8.1) @@ -110,28 +115,29 @@ GEM net-smtp marcel (1.0.2) mini_mime (1.1.5) - mini_portile2 (2.8.1) minitest (5.20.0) mocha (2.1.0) ruby2_keywords (>= 0.0.5) - mutex_m (0.1.2) - mysql2 (0.5.4) - net-imap (0.4.1) + mutex_m (0.2.0) + mysql2 (0.5.5) + net-imap (0.4.9) date net-protocol net-pop (0.1.2) net-protocol - net-protocol (0.2.1) + net-protocol (0.2.2) timeout net-smtp (0.4.0) net-protocol - nio4r (2.5.9) - nokogiri (1.15.4-arm64-darwin) + nio4r (2.7.0) + nokogiri (1.15.5-arm64-darwin) racc (~> 1.4) - nokogiri (1.15.4-x86_64-linux) + nokogiri (1.15.5-x86_64-linux) racc (~> 1.4) pg (1.5.4) - racc (1.7.1) + psych (5.1.2) + stringio + racc (1.7.3) rack (3.0.8) rack-session (2.0.0) rack (>= 3.0.0) @@ -140,20 +146,20 @@ GEM rackup (2.1.0) rack (>= 3) webrick (~> 1.8) - rails (7.1.0) - actioncable (= 7.1.0) - actionmailbox (= 7.1.0) - actionmailer (= 7.1.0) - actionpack (= 7.1.0) - actiontext (= 7.1.0) - actionview (= 7.1.0) - activejob (= 7.1.0) - activemodel (= 7.1.0) - activerecord (= 7.1.0) - activestorage (= 7.1.0) - activesupport (= 7.1.0) + rails (7.1.2) + actioncable (= 7.1.2) + actionmailbox (= 7.1.2) + actionmailer (= 7.1.2) + actionpack (= 7.1.2) + actiontext (= 7.1.2) + actionview (= 7.1.2) + activejob (= 7.1.2) + activemodel (= 7.1.2) + activerecord (= 7.1.2) + activestorage (= 7.1.2) + activesupport (= 7.1.2) bundler (>= 1.15.0) - railties (= 7.1.0) + railties (= 7.1.2) rails-dom-testing (2.2.0) activesupport (>= 5.0.0) minitest @@ -161,22 +167,26 @@ GEM rails-html-sanitizer (1.6.0) loofah (~> 2.21) nokogiri (~> 1.14) - railties (7.1.0) - actionpack (= 7.1.0) - activesupport (= 7.1.0) + railties (7.1.2) + actionpack (= 7.1.2) + activesupport (= 7.1.2) irb rackup (>= 1.0.0) rake (>= 12.2) thor (~> 1.0, >= 1.2.2) zeitwerk (~> 2.6) - rake (13.0.6) - reline (0.3.2) + rake (13.1.0) + rdoc (6.6.2) + psych (>= 4.0.0) + reline (0.4.1) io-console (~> 0.5) ruby2_keywords (0.0.5) - sqlite3 (1.5.4) - mini_portile2 (~> 2.8.0) - thor (1.2.2) - timeout (0.4.0) + sqlite3 (1.6.9-arm64-darwin) + sqlite3 (1.6.9-x86_64-linux) + stringio (3.1.0) + thor (1.3.0) + timeout (0.4.1) + tiny_tds (2.1.6) tzinfo (2.0.6) concurrent-ruby (~> 1.0) webrick (1.8.1) @@ -186,16 +196,19 @@ GEM zeitwerk (2.6.12) PLATFORMS + arm64-darwin-21 arm64-darwin-22 x86_64-linux DEPENDENCIES + activerecord-sqlserver-adapter (~> 7.1.0) debug mocha mysql2 pg solid_queue! sqlite3 + tiny_tds BUNDLED WITH 2.4.2 diff --git a/app/models/solid_queue/record.rb b/app/models/solid_queue/record.rb index a779ea3b..737fad09 100644 --- a/app/models/solid_queue/record.rb +++ b/app/models/solid_queue/record.rb @@ -8,7 +8,11 @@ class Record < ActiveRecord::Base def self.non_blocking_lock if SolidQueue.use_skip_locked - lock(Arel.sql("FOR UPDATE SKIP LOCKED")) + if self.connection.adapter_name == 'SQLServer' + lock(Arel.sql("WITH(READPAST)")) + else + lock(Arel.sql("FOR UPDATE SKIP LOCKED")) + end else lock end diff --git a/docker-compose.yml b/docker-compose.yml index 9777d762..cca7f562 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,11 +10,19 @@ services: MYSQL_ALLOW_EMPTY_PASSWORD: "yes" volumes: - db:/var/lib/mysql - ports: [ "127.0.0.1:33060:3306" ] + ports: ["127.0.0.1:33060:3306"] postgres: image: postgres:15.1 environment: POSTGRES_HOST_AUTH_METHOD: "trust" volumes: - db:/var/lib/postgres - ports: [ "127.0.0.1:55432:5432" ] + ports: ["127.0.0.1:55432:5432"] + mssql: + image: mcr.microsoft.com/azure-sql-edge:1.0.4 + environment: + SA_PASSWORD: yourStrongPassword123 + ACCEPT_EULA: Y + volumes: + - db:/var/lib/mssql + ports: ["127.0.0.1:11433:1433"] diff --git a/solid_queue.gemspec b/solid_queue.gemspec index 51effb59..eb716569 100644 --- a/solid_queue.gemspec +++ b/solid_queue.gemspec @@ -17,7 +17,7 @@ Gem::Specification.new do |spec| Dir["{app,config,db,lib}/**/*", "MIT-LICENSE", "Rakefile", "README.md"] end - spec.add_dependency "rails", ">= 7.0.3.1" + spec.add_dependency "rails", ">= 7.1.0" spec.add_development_dependency "debug" spec.add_development_dependency "mocha" end diff --git a/test/dummy/config/database.yml b/test/dummy/config/database.yml index a05e6c3f..662f270c 100644 --- a/test/dummy/config/database.yml +++ b/test/dummy/config/database.yml @@ -23,6 +23,15 @@ default: &default port: 55432 gssencmode: disable # https://github.com/ged/ruby-pg/issues/311 +<% elsif ENV["TARGET_DB"] == "mssql" %> +default: &default + adapter: sqlserver + username: sa + password: "yourStrongPassword123" + pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %> + host: localhost + port: 11433 + <% else %> default: &default adapter: mysql2 From 47c64edfa2c07948509af976c2919e522e0622ee Mon Sep 17 00:00:00 2001 From: Jonah George Date: Sun, 24 Dec 2023 13:24:45 -0800 Subject: [PATCH 2/2] Attempt to patch MSSQL missing features --- app/models/solid_queue/blocked_execution.rb | 6 +++--- app/models/solid_queue/claimed_execution.rb | 7 ++++++- app/models/solid_queue/scheduled_execution.rb | 12 +++++++++++- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/app/models/solid_queue/blocked_execution.rb b/app/models/solid_queue/blocked_execution.rb index 8e0cf74b..bfc676c2 100644 --- a/app/models/solid_queue/blocked_execution.rb +++ b/app/models/solid_queue/blocked_execution.rb @@ -11,9 +11,9 @@ class BlockedExecution < Execution class << self def unblock(count) - expired.distinct.limit(count).pluck(:concurrency_key).then do |concurrency_keys| - release_many releasable(concurrency_keys) - end + # This is probably wrong + rows = expired.distinct.limit(count) + release_many releasable(rows.map(&:concurrency_key)) end def release_many(concurrency_keys) diff --git a/app/models/solid_queue/claimed_execution.rb b/app/models/solid_queue/claimed_execution.rb index 9af5fe38..d4795575 100644 --- a/app/models/solid_queue/claimed_execution.rb +++ b/app/models/solid_queue/claimed_execution.rb @@ -13,7 +13,12 @@ class << self def claiming(job_ids, process_id, &block) job_data = Array(job_ids).collect { |job_id| { job_id: job_id, process_id: process_id } } - insert_all!(job_data) + if self.connection.supports_insert_on_duplicate_skip? + insert_all!(job_data) + else + job_data.each { |jd| create_with(process_id: jd[:process_id]).find_or_create_by!(job_id: jd[:job_id]) } + end + where(job_id: job_ids, process_id: process_id).load.tap do |claimed| block.call(claimed) SolidQueue.logger.info("[SolidQueue] Claimed #{claimed.size} jobs") diff --git a/app/models/solid_queue/scheduled_execution.rb b/app/models/solid_queue/scheduled_execution.rb index 0f8c07e1..f2e62144 100644 --- a/app/models/solid_queue/scheduled_execution.rb +++ b/app/models/solid_queue/scheduled_execution.rb @@ -34,7 +34,17 @@ def dispatch_batch(job_ids) end def dispatch_at_once(jobs) - ReadyExecution.insert_all ready_rows_from_batch(jobs) + if self.connection.supports_insert_on_duplicate_skip? + ReadyExecution.insert_all ready_rows_from_batch(jobs) + else + ready_rows_from_batch(jobs).each do |job| + ReadyExecution.create_with( + queue_name: job[:queue_name], + priority: job[:priority], + created_at: job[:created_at] + ).find_or_create_by(job_id: job[:job_id]) + end + end end def dispatch_one_by_one(jobs)