From 82986a1d48334345cc109811680616bc9a5e453e Mon Sep 17 00:00:00 2001 From: Nicholas Jakobsen Date: Tue, 21 Dec 2021 15:42:43 -0800 Subject: [PATCH 1/3] Override global option with group option Local options override globally set options, but options set on the `every` group do not. It makes sense for the options defined closer to the actual job to override those set further away, so we change the merge order of options to allow options passed to `every` to override those set globally with `set`. We still allow local options set on each job to override those set on the group. --- lib/whenever/job_list.rb | 2 +- test/functional/output_defined_job_test.rb | 28 ++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/lib/whenever/job_list.rb b/lib/whenever/job_list.rb index 7df39c68..da88cce8 100644 --- a/lib/whenever/job_list.rb +++ b/lib/whenever/job_list.rb @@ -66,7 +66,7 @@ def job_type(name, template) @jobs[options.fetch(:mailto)] ||= {} @jobs[options.fetch(:mailto)][@current_time_scope] ||= [] - @jobs[options.fetch(:mailto)][@current_time_scope] << Whenever::Job.new(@options.merge(@set_variables).merge(options)) + @jobs[options.fetch(:mailto)][@current_time_scope] << Whenever::Job.new(@set_variables.merge(@options).merge(options)) end end end diff --git a/test/functional/output_defined_job_test.rb b/test/functional/output_defined_job_test.rb index 9f163f66..6fcacf4b 100644 --- a/test/functional/output_defined_job_test.rb +++ b/test/functional/output_defined_job_test.rb @@ -55,6 +55,34 @@ class OutputDefinedJobTest < Whenever::TestCase assert_match(/^.+ .+ .+ .+ before during after local$/, output) end + test "defined job with a :task and an option where the option is set globally and on the group" do + output = Whenever.cron \ + <<-file + set :job_template, nil + job_type :some_job, "before :task after :option1" + set :option1, 'global' + every 2.hours, :option1 => 'group' do + some_job "during" + end + file + + assert_match(/^.+ .+ .+ .+ before during after group$/, output) + end + + test "defined job with a :task and an option where the option is set globally, on the group, and locally" do + output = Whenever.cron \ + <<-file + set :job_template, nil + job_type :some_job, "before :task after :option1" + set :option1, 'global' + every 2.hours, :option1 => 'group' do + some_job "during", :option1 => 'local' + end + file + + assert_match(/^.+ .+ .+ .+ before during after local$/, output) + end + test "defined job with a :task and an option that is not set" do output = Whenever.cron \ <<-file From 11361045889119a8d1cdabdc611ab61d36cde248 Mon Sep 17 00:00:00 2001 From: Nicholas Jakobsen Date: Wed, 31 Jul 2024 12:59:24 -0700 Subject: [PATCH 2/3] Add support for sequential job definitions Jobs now support a `sequence` option that will cause jobs with the same `sequence` value to run sequentially when scheduled for the same time. `sequential: true` can be passed to the `every` block to automatically set a common `sequence` value for each contained job. Closes https://github.com/javan/whenever/issues/19 Closes https://github.com/javan/whenever/issues/696 Closes https://github.com/javan/whenever/issues/835 --- README.md | 8 + lib/whenever.rb | 1 + lib/whenever/job.rb | 3 +- lib/whenever/job_list.rb | 19 ++- lib/whenever/job_sequence.rb | 35 +++++ .../output_jobs_with_sequence_test.rb | 138 ++++++++++++++++++ test/unit/job_test.rb | 1 - 7 files changed, 200 insertions(+), 5 deletions(-) create mode 100644 lib/whenever/job_sequence.rb create mode 100644 test/functional/output_jobs_with_sequence_test.rb diff --git a/README.md b/README.md index ed210700..983ccd3f 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,14 @@ every 3.hours do # 1.minute 1.day 1.week 1.month 1.year is also supported command "/usr/bin/my_great_command" end +every 3.hours, sequential: true do + # Jobs in this block that don't set their own sequence will default to run in the same sequence (not in parallel) + runner "MyModel.some_process" + rake "my:rake:task" + command "/usr/bin/my_great_command", sequence: nil # This job runs in parallel with other jobs that are not part of a sequence + runner "MyModel.task_to_run_at_four_thirty_in_the_morning", sequence: 'other' # This job runs sequentially with other jobs in the sequence called 'other' +end + every 1.day, at: '4:30 am' do runner "MyModel.task_to_run_at_four_thirty_in_the_morning" end diff --git a/lib/whenever.rb b/lib/whenever.rb index d2ef6dd7..4206f42d 100644 --- a/lib/whenever.rb +++ b/lib/whenever.rb @@ -1,6 +1,7 @@ require 'whenever/numeric' require 'whenever/numeric_seconds' require 'whenever/job_list' +require 'whenever/job_sequence' require 'whenever/job' require 'whenever/command_line' require 'whenever/cron' diff --git a/lib/whenever/job.rb b/lib/whenever/job.rb index 2dad8329..7802de2e 100644 --- a/lib/whenever/job.rb +++ b/lib/whenever/job.rb @@ -2,7 +2,7 @@ module Whenever class Job - attr_reader :at, :roles, :mailto + attr_reader :at, :roles, :mailto, :sequence def initialize(options = {}) @options = options @@ -10,6 +10,7 @@ def initialize(options = {}) @template = options.delete(:template) @mailto = options.fetch(:mailto, :default_mailto) @job_template = options.delete(:job_template) || ":job" + @sequence = options.delete(:sequence) @roles = Array(options.delete(:roles)) @options[:output] = options.has_key?(:output) ? Whenever::Output::Redirection.new(options[:output]).to_s : '' @options[:environment_variable] ||= "RAILS_ENV" diff --git a/lib/whenever/job_list.rb b/lib/whenever/job_list.rb index da88cce8..25e497f1 100644 --- a/lib/whenever/job_list.rb +++ b/lib/whenever/job_list.rb @@ -48,6 +48,12 @@ def env(variable, value) def every(frequency, options = {}) @current_time_scope = frequency @options = options + + if @options[:sequential] + @default_sequence_id ||= 0 + @options[:sequence] ||= "default_sequence_#{@default_sequence_id += 1}" + end + yield end @@ -138,10 +144,17 @@ def combine(entries) def cron_jobs_of_time(time, jobs) shortcut_jobs, regular_jobs = [], [] + filtered_jobs = jobs.select do |job| + roles.empty? || roles.any? { |r| job.has_role?(r) } + end + + grouped_jobs = filtered_jobs.group_by(&:sequence) + grouped_jobs.each do |sequence, jobs| + grouped_jobs[sequence] = JobSequence.new(jobs) if sequence + end + jobs = grouped_jobs.values.flatten + jobs.each do |job| - next unless roles.empty? || roles.any? do |r| - job.has_role?(r) - end Whenever::Output::Cron.output(time, job, :chronic_options => @chronic_options) do |cron| cron << "\n\n" diff --git a/lib/whenever/job_sequence.rb b/lib/whenever/job_sequence.rb new file mode 100644 index 00000000..d58c4f8c --- /dev/null +++ b/lib/whenever/job_sequence.rb @@ -0,0 +1,35 @@ +require 'shellwords' + +module Whenever + class JobSequence + attr_reader :at, :roles, :mailto + + def initialize(jobs, options = {}) + validate!(jobs) + + @jobs = jobs + @options = options + @at = options.fetch(:at, primary_job.at) + @mailto = options.fetch(:mailto, primary_job.mailto || :default_mailto) + @roles = Array(options.delete(:roles), *primary_job.roles) + end + + def output + @jobs.map(&:output).join(' && ') + end + + def has_role?(role) + roles.empty? || roles.include?(role) + end + + private + + def primary_job + @jobs.first + end + + def validate!(jobs) + raise ArgumentError, "Jobs in a sequence don't support different `at` values" if jobs.map(&:at).uniq.count > 1 + end + end +end diff --git a/test/functional/output_jobs_with_sequence_test.rb b/test/functional/output_jobs_with_sequence_test.rb new file mode 100644 index 00000000..3118a0df --- /dev/null +++ b/test/functional/output_jobs_with_sequence_test.rb @@ -0,0 +1,138 @@ +require 'test_helper' + +class OutputJobsWithSequenceTest < Whenever::TestCase + test "defined jobs with a sequence argument specified per-job" do + output = Whenever.cron \ + <<-file + every 2.hours do + command "blahblah", sequence: 'backups' + command "foofoo", sequence: 'backups' + command "barbar" + end + file + + output_without_empty_line = lines_without_empty_line(output.lines) + assert_equal two_hours + " /bin/bash -l -c 'blahblah' && /bin/bash -l -c 'foofoo'", output_without_empty_line.shift + assert_equal two_hours + " /bin/bash -l -c 'barbar'", output_without_empty_line.shift + end + + test "defined jobs with a sequence argument specified on the group" do + output = Whenever.cron \ + <<-file + every 2.hours, sequence: 'backups' do + command "blahblah" + command "foofoo" + end + file + + output_without_empty_line = lines_without_empty_line(output.lines) + assert_equal two_hours + " /bin/bash -l -c 'blahblah' && /bin/bash -l -c 'foofoo'", output_without_empty_line.shift + end + + test "defined jobs with a sequences specified on the group and jobs" do + output = Whenever.cron \ + <<-file + every 2.hours, sequence: 'backups' do + command "blahblah" + command "barbar", sequence: nil + command "foofoo" + command "bazbaz", sequence: 'bees' + command "buzzbuzz", sequence: 'bees' + end + file + + output_without_empty_line = lines_without_empty_line(output.lines) + assert_equal two_hours + " /bin/bash -l -c 'blahblah' && /bin/bash -l -c 'foofoo'", output_without_empty_line.shift + assert_equal two_hours + " /bin/bash -l -c 'barbar'", output_without_empty_line.shift + assert_equal two_hours + " /bin/bash -l -c 'bazbaz' && /bin/bash -l -c 'buzzbuzz'", output_without_empty_line.shift + end + + test "defined jobs with a multiple groups with sequences specified on the group and jobs" do + output = Whenever.cron \ + <<-file + every 2.hours, sequence: 'backups' do + command "blahblah" + command "barbar", sequence: nil + command "bazbaz", sequence: 'bees' + end + + every 2.hours, sequence: 'backups' do + command "foofoo" + command "buzzbuzz", sequence: 'bees' + end + file + + output_without_empty_line = lines_without_empty_line(output.lines) + assert_equal two_hours + " /bin/bash -l -c 'blahblah' && /bin/bash -l -c 'foofoo'", output_without_empty_line.shift + assert_equal two_hours + " /bin/bash -l -c 'barbar'", output_without_empty_line.shift + assert_equal two_hours + " /bin/bash -l -c 'bazbaz' && /bin/bash -l -c 'buzzbuzz'", output_without_empty_line.shift + end + + test "defined jobs with a multiple groups with sequences specified on the group and jobs" do + output = Whenever.cron \ + <<-file + every 2.hours, sequence: 'backups' do + command "blahblah" + command "barbar", sequence: nil + command "bazbaz", sequence: 'bees' + end + + every 3.hours, sequence: 'backups' do + command "foofoo" + command "buzzbuzz", sequence: 'bees' + end + file + + output_without_empty_line = lines_without_empty_line(output.lines) + assert_equal two_hours + " /bin/bash -l -c 'blahblah'", output_without_empty_line.shift + assert_equal two_hours + " /bin/bash -l -c 'barbar'", output_without_empty_line.shift + assert_equal two_hours + " /bin/bash -l -c 'bazbaz'", output_without_empty_line.shift + assert_equal three_hours + " /bin/bash -l -c 'foofoo'", output_without_empty_line.shift + assert_equal three_hours + " /bin/bash -l -c 'buzzbuzz'", output_without_empty_line.shift + end + + test "defined jobs with a multiple groups with sequences specified on the group and jobs" do + assert_raises ArgumentError do + Whenever.cron \ + <<-file + every 2.hours, sequence: 'backups' do + command "blahblah", at: 1 + command "barbar", at: 2 + end + file + end + end + + def three_hours + "0 0,3,6,9,12,15,18,21 * * *" + end +end + +class OutputJobsWithSequentialTest < Whenever::TestCase + test "defined jobs with a sequential argument" do + output = Whenever.cron \ + <<-file + every 2.hours, sequential: true do + command "blahblah" + command "foofoo" + end + file + + output_without_empty_line = lines_without_empty_line(output.lines) + assert_equal two_hours + " /bin/bash -l -c 'blahblah' && /bin/bash -l -c 'foofoo'", output_without_empty_line.shift + end + + test "defined jobs with a sequential argument" do + output = Whenever.cron \ + <<-file + every 2.hours, sequential: true do + command "blahblah" + command "foofoo", sequence: false + end + file + + output_without_empty_line = lines_without_empty_line(output.lines) + assert_equal two_hours + " /bin/bash -l -c 'blahblah'", output_without_empty_line.shift + assert_equal two_hours + " /bin/bash -l -c 'foofoo'", output_without_empty_line.shift + end +end diff --git a/test/unit/job_test.rb b/test/unit/job_test.rb index a9e34ce5..fae54f13 100644 --- a/test/unit/job_test.rb +++ b/test/unit/job_test.rb @@ -62,7 +62,6 @@ class JobTest < Whenever::TestCase end end - class JobWithQuotesTest < Whenever::TestCase should "output the :task if it's in single quotes" do job = new_job(:template => "':task'", :task => 'abc123') From 6a4c7484661bbb52787a6642073ab494069b0a4e Mon Sep 17 00:00:00 2001 From: Nicholas Jakobsen Date: Wed, 31 Jul 2024 15:44:19 -0700 Subject: [PATCH 3/3] Allow sequential jobs to continue by default It's likely that since jobs ran in parallel before, there were no dependencies between jobs. Therefore, now that sequential jobs are possible, we should continue execution of the job sequence even if one job fails. To override this and halt the sequence on failure, individual jobs can set `halt_sequence_on_failure: true` to suffix their command with an "&&" instead of a ";". --- README.md | 2 +- lib/whenever/job.rb | 3 ++- lib/whenever/job_sequence.rb | 2 +- .../output_jobs_with_sequence_test.rb | 25 ++++++++++++++----- 4 files changed, 23 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 983ccd3f..f4d6e7b8 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,7 @@ end every 3.hours, sequential: true do # Jobs in this block that don't set their own sequence will default to run in the same sequence (not in parallel) - runner "MyModel.some_process" + runner "MyModel.some_process", halt_sequence_on_failure: true # If this job fails, subsequent jobs will not run (defaults to false) rake "my:rake:task" command "/usr/bin/my_great_command", sequence: nil # This job runs in parallel with other jobs that are not part of a sequence runner "MyModel.task_to_run_at_four_thirty_in_the_morning", sequence: 'other' # This job runs sequentially with other jobs in the sequence called 'other' diff --git a/lib/whenever/job.rb b/lib/whenever/job.rb index 7802de2e..a56063cd 100644 --- a/lib/whenever/job.rb +++ b/lib/whenever/job.rb @@ -2,7 +2,7 @@ module Whenever class Job - attr_reader :at, :roles, :mailto, :sequence + attr_reader :at, :roles, :mailto, :sequence, :halt_sequence_on_failure def initialize(options = {}) @options = options @@ -11,6 +11,7 @@ def initialize(options = {}) @mailto = options.fetch(:mailto, :default_mailto) @job_template = options.delete(:job_template) || ":job" @sequence = options.delete(:sequence) + @halt_sequence_on_failure = options.delete(:halt_sequence_on_failure) @roles = Array(options.delete(:roles)) @options[:output] = options.has_key?(:output) ? Whenever::Output::Redirection.new(options[:output]).to_s : '' @options[:environment_variable] ||= "RAILS_ENV" diff --git a/lib/whenever/job_sequence.rb b/lib/whenever/job_sequence.rb index d58c4f8c..b63ddad0 100644 --- a/lib/whenever/job_sequence.rb +++ b/lib/whenever/job_sequence.rb @@ -15,7 +15,7 @@ def initialize(jobs, options = {}) end def output - @jobs.map(&:output).join(' && ') + @jobs.map { |job| [job.output, job.halt_sequence_on_failure ? ' && ' : ' ; '] }.flatten[0..-2].join end def has_role?(role) diff --git a/test/functional/output_jobs_with_sequence_test.rb b/test/functional/output_jobs_with_sequence_test.rb index 3118a0df..15d693b8 100644 --- a/test/functional/output_jobs_with_sequence_test.rb +++ b/test/functional/output_jobs_with_sequence_test.rb @@ -12,7 +12,7 @@ class OutputJobsWithSequenceTest < Whenever::TestCase file output_without_empty_line = lines_without_empty_line(output.lines) - assert_equal two_hours + " /bin/bash -l -c 'blahblah' && /bin/bash -l -c 'foofoo'", output_without_empty_line.shift + assert_equal two_hours + " /bin/bash -l -c 'blahblah' ; /bin/bash -l -c 'foofoo'", output_without_empty_line.shift assert_equal two_hours + " /bin/bash -l -c 'barbar'", output_without_empty_line.shift end @@ -25,6 +25,19 @@ class OutputJobsWithSequenceTest < Whenever::TestCase end file + output_without_empty_line = lines_without_empty_line(output.lines) + assert_equal two_hours + " /bin/bash -l -c 'blahblah' ; /bin/bash -l -c 'foofoo'", output_without_empty_line.shift + end + + test "defined jobs with a sequence argument and a job that halts the sequence on failure" do + output = Whenever.cron \ + <<-file + every 2.hours, sequence: 'backups' do + command "blahblah", halt_sequence_on_failure: true + command "foofoo" + end + file + output_without_empty_line = lines_without_empty_line(output.lines) assert_equal two_hours + " /bin/bash -l -c 'blahblah' && /bin/bash -l -c 'foofoo'", output_without_empty_line.shift end @@ -42,9 +55,9 @@ class OutputJobsWithSequenceTest < Whenever::TestCase file output_without_empty_line = lines_without_empty_line(output.lines) - assert_equal two_hours + " /bin/bash -l -c 'blahblah' && /bin/bash -l -c 'foofoo'", output_without_empty_line.shift + assert_equal two_hours + " /bin/bash -l -c 'blahblah' ; /bin/bash -l -c 'foofoo'", output_without_empty_line.shift assert_equal two_hours + " /bin/bash -l -c 'barbar'", output_without_empty_line.shift - assert_equal two_hours + " /bin/bash -l -c 'bazbaz' && /bin/bash -l -c 'buzzbuzz'", output_without_empty_line.shift + assert_equal two_hours + " /bin/bash -l -c 'bazbaz' ; /bin/bash -l -c 'buzzbuzz'", output_without_empty_line.shift end test "defined jobs with a multiple groups with sequences specified on the group and jobs" do @@ -63,9 +76,9 @@ class OutputJobsWithSequenceTest < Whenever::TestCase file output_without_empty_line = lines_without_empty_line(output.lines) - assert_equal two_hours + " /bin/bash -l -c 'blahblah' && /bin/bash -l -c 'foofoo'", output_without_empty_line.shift + assert_equal two_hours + " /bin/bash -l -c 'blahblah' ; /bin/bash -l -c 'foofoo'", output_without_empty_line.shift assert_equal two_hours + " /bin/bash -l -c 'barbar'", output_without_empty_line.shift - assert_equal two_hours + " /bin/bash -l -c 'bazbaz' && /bin/bash -l -c 'buzzbuzz'", output_without_empty_line.shift + assert_equal two_hours + " /bin/bash -l -c 'bazbaz' ; /bin/bash -l -c 'buzzbuzz'", output_without_empty_line.shift end test "defined jobs with a multiple groups with sequences specified on the group and jobs" do @@ -119,7 +132,7 @@ class OutputJobsWithSequentialTest < Whenever::TestCase file output_without_empty_line = lines_without_empty_line(output.lines) - assert_equal two_hours + " /bin/bash -l -c 'blahblah' && /bin/bash -l -c 'foofoo'", output_without_empty_line.shift + assert_equal two_hours + " /bin/bash -l -c 'blahblah' ; /bin/bash -l -c 'foofoo'", output_without_empty_line.shift end test "defined jobs with a sequential argument" do