From e3882504e452efc4ada18b9e3f1a685df16e0ec1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Rasmusson?= Date: Tue, 20 Jan 2026 21:29:13 +0100 Subject: [PATCH 1/3] Change to use worst Test Step result as the Test Case result When calculating the Test Case result Cucumber-Ruby-Core should conform with the GetWorstTestCaseResult function used in the Messages module. --- lib/cucumber/core/test/result.rb | 2 +- lib/cucumber/core/test/runner.rb | 41 ++++---- spec/cucumber/core/test/runner_spec.rb | 99 +++++++++++++++++++ .../support/shared_context/test_step_types.rb | 2 + 4 files changed, 122 insertions(+), 22 deletions(-) diff --git a/lib/cucumber/core/test/result.rb b/lib/cucumber/core/test/result.rb index d8e20a00..c044370b 100644 --- a/lib/cucumber/core/test/result.rb +++ b/lib/cucumber/core/test/result.rb @@ -7,7 +7,7 @@ module Cucumber module Core module Test module Result - TYPES = %i[failed ambiguous flaky skipped undefined pending passed unknown].freeze + TYPES = %i[failed ambiguous flaky undefined pending skipped passed unknown].freeze STRICT_AFFECTED_TYPES = %i[flaky undefined pending].freeze def self.ok?(type, strict: StrictConfiguration.new) diff --git a/lib/cucumber/core/test/runner.rb b/lib/cucumber/core/test/runner.rb index 58c90deb..60632d3d 100644 --- a/lib/cucumber/core/test/runner.rb +++ b/lib/cucumber/core/test/runner.rb @@ -59,27 +59,27 @@ def result end def failed(step_result) - @status = Status::Failing.new(step_result) + not_passing(step_result) self end def ambiguous(step_result) - @status = Status::Ambiguous.new(step_result) + failed(step_result) self end def passed(step_result) - @status = Status::Passing.new(step_result) + @status = Status::Passing.new(step_result) if Result::TYPES.index(step_result.to_sym) < Result::TYPES.index(status.step_result_sym) self end def pending(_message, step_result) - @status = Status::Pending.new(step_result) + failed(step_result) self end def skipped(step_result) - @status = Status::Skipping.new(step_result) + failed(step_result) self end @@ -96,6 +96,13 @@ def duration(_step_duration, _step_result) self end + private + + def not_passing(step_result) + @status = Status::NotPassing.new(step_result) if Result::TYPES.index(step_result.to_sym) < Result::TYPES.index(status.step_result_sym) + self + end + attr_reader :status private :status @@ -118,6 +125,10 @@ def execute(test_step, monitor, &) def result raise NoMethodError, 'Override me' end + + def step_result_sym + step_result.to_sym + end end class Unknown < Base @@ -132,26 +143,14 @@ def result(duration) end end - class Failing < Base + class NotPassing < Base def execute(test_step, monitor) result = test_step.skip(monitor.result) - if result.undefined? - result = result.with_message(%(Undefined step: "#{test_step.text}")) - result = result.with_appended_backtrace(test_step) - end - result - end - - def result(duration) - step_result.with_duration(duration) + result = result.with_message(%(Undefined step: "#{test_step.text}")) if result.undefined? + result = result.with_appended_backtrace(test_step) unless test_step.hook? + result.describe_to(monitor, result) end - end - - Pending = Class.new(Failing) - - Ambiguous = Class.new(Failing) - class Skipping < Failing def result(duration) step_result.with_duration(duration) end diff --git a/spec/cucumber/core/test/runner_spec.rb b/spec/cucumber/core/test/runner_spec.rb index 65ed7669..16d980b7 100644 --- a/spec/cucumber/core/test/runner_spec.rb +++ b/spec/cucumber/core/test/runner_spec.rb @@ -250,6 +250,105 @@ test_case.describe_to(runner) end end + + context 'with an initial undefined step' do + let(:test_steps) { [undefined_step, passing_step] } + + it 'emits a test_step_finished event with an undefined result' do + expect(event_bus).to receive(:test_step_finished).with(undefined_step, anything) do |_reported_test_case, result| + expect(result).to be_undefined + end + test_case.describe_to(runner) + end + + it 'emits a test_step_finished event with a skipped result' do + expect(event_bus).to receive(:test_step_finished).with(passing_step, anything) do |_reported_test_case, result| + expect(result).to be_skipped + end + test_case.describe_to(runner) + end + + it 'emits a test_case_finished event with an undefined result' do + allow(event_bus).to receive(:test_case_finished) do |_reported_test_case, result| + expect(result).to be_undefined + expect(result.exception).to be_a StandardError + end + test_case.describe_to(runner) + end + + it 'skips, rather than executing the second step' do + expect(passing_step).not_to receive(:execute) + + allow(passing_step).to receive(:skip).and_return(Cucumber::Core::Test::Result::Skipped.new) + test_case.describe_to(runner) + end + + context 'with a following ambiguous step' do + let(:test_steps) { [undefined_step, ambiguous_step] } + + it 'emits a test_step_finished event with an undefined result' do + expect(event_bus).to receive(:test_step_finished).with(undefined_step, anything) do |_reported_test_case, result| + expect(result).to be_undefined + end + test_case.describe_to(runner) + end + + it 'emits a test_step_finished event with a ambiguous result' do + expect(event_bus).to receive(:test_step_finished).with(ambiguous_step, anything) do |_reported_test_case, result| + expect(result).to be_ambiguous + end + test_case.describe_to(runner) + end + + it 'emits a test_case_finished event with an ambiguous result' do + allow(event_bus).to receive(:test_case_finished) do |_reported_test_case, result| + expect(result).to be_ambiguous + expect(result.exception).to be_a StandardError + end + test_case.describe_to(runner) + end + + it 'skips, rather than executing the second step' do + expect(ambiguous_step).not_to receive(:execute) + + allow(ambiguous_step).to receive(:skip).and_return(Cucumber::Core::Test::Result::Ambiguous.new) + test_case.describe_to(runner) + end + end + + context 'with a failing after hook' do + let(:test_steps) { [undefined_step, failing_hook] } + + it 'emits a test_step_finished event with an undefined result' do + expect(event_bus).to receive(:test_step_finished).with(undefined_step, anything) do |_reported_test_case, result| + expect(result).to be_undefined + end + test_case.describe_to(runner) + end + + it 'emits a test_step_finished event with a failing result' do + expect(event_bus).to receive(:test_step_finished).with(failing_hook, anything) do |_reported_test_case, result| + expect(result).to be_failed + end + test_case.describe_to(runner) + end + + it 'emits a test_case_finished event with an failing result' do + allow(event_bus).to receive(:test_case_finished) do |_reported_test_case, result| + expect(result).to be_failed + expect(result.exception).to be_a StandardError + end + test_case.describe_to(runner) + end + + it 'call skip, rather than execute on test step of the hook' do + expect(failing_hook).not_to receive(:execute) + + allow(failing_hook).to receive(:skip).and_return(Cucumber::Core::Test::Result::Failed.new(anything, instance_double(StandardError, backtrace: []))) + test_case.describe_to(runner) + end + end + end end context 'with multiple test cases' do diff --git a/spec/support/shared_context/test_step_types.rb b/spec/support/shared_context/test_step_types.rb index 51cd040e..551956a4 100644 --- a/spec/support/shared_context/test_step_types.rb +++ b/spec/support/shared_context/test_step_types.rb @@ -8,4 +8,6 @@ let(:pending_step) { Cucumber::Core::Test::Step.new(3, 'Pending Step', double).with_action { raise Cucumber::Core::Test::Result::Pending, 'TODO' } } let(:skipping_step) { Cucumber::Core::Test::Step.new(4, 'Skipped Step', double).with_action { raise Cucumber::Core::Test::Result::Skipped } } let(:undefined_step) { Cucumber::Core::Test::Step.new(5, 'Undefined Step', double) } + let(:ambiguous_step) { Cucumber::Core::Test::Step.new(6, 'Ambiguous Step', double).with_unskippable_action { raise Cucumber::Core::Test::Result::Ambiguous } } + let(:failing_hook) { Cucumber::Core::Test::Step.new(7, 'Failing Step', double).with_unskippable_action { raise StandardError, 'Error' } } end From 36ac5199785d688d01f227052044bf80270ce2f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Rasmusson?= Date: Thu, 19 Mar 2026 12:30:10 +0100 Subject: [PATCH 2/3] Update ChangeLog.md --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6992ce2c..e2af3ea2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,10 @@ Please visit [cucumber/CONTRIBUTING.md](https://github.com/cucumber/cucumber/blo ## [Unreleased] +### Changed +- Change to use worst Test Step result as the Test Case result +([#317](https://github.com/cucumber/cucumber-ruby-core/pull/317)) + ## [16.2.0] - 2026-02-06 ### Changed - Added the test result type 'ambiguous' From da401623dd5fd7d1e4ba05edd2f549d9b5beae65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Rasmusson?= Date: Sun, 29 Mar 2026 18:55:33 +0200 Subject: [PATCH 3/3] Use test_step_result_rankings from messages in the Runner Use test_step_result_rankings from Cucumber::Messages::Helpers::TestStepResultComparator to compare test results to find the worst one. --- lib/cucumber/core/test/runner.rb | 11 +++++++---- spec/cucumber/core/test/runner_spec.rb | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/lib/cucumber/core/test/runner.rb b/lib/cucumber/core/test/runner.rb index 60632d3d..236170c3 100644 --- a/lib/cucumber/core/test/runner.rb +++ b/lib/cucumber/core/test/runner.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require 'cucumber/core/test/timer' +require 'cucumber/messages/helpers/test_step_result_comparator' module Cucumber module Core @@ -45,6 +46,8 @@ def done end class RunningTestCase + include Cucumber::Messages::Helpers::TestStepResultComparator + def initialize @timer = Timer.new.start @status = Status::Unknown.new(Result::Unknown.new) @@ -69,7 +72,7 @@ def ambiguous(step_result) end def passed(step_result) - @status = Status::Passing.new(step_result) if Result::TYPES.index(step_result.to_sym) < Result::TYPES.index(status.step_result_sym) + @status = Status::Passing.new(step_result) if test_step_result_rankings[step_result.to_message.status] > test_step_result_rankings[status.step_result_message.status] self end @@ -99,7 +102,7 @@ def duration(_step_duration, _step_result) private def not_passing(step_result) - @status = Status::NotPassing.new(step_result) if Result::TYPES.index(step_result.to_sym) < Result::TYPES.index(status.step_result_sym) + @status = Status::NotPassing.new(step_result) if test_step_result_rankings[step_result.to_message.status] > test_step_result_rankings[status.step_result_message.status] self end @@ -126,8 +129,8 @@ def result raise NoMethodError, 'Override me' end - def step_result_sym - step_result.to_sym + def step_result_message + step_result.to_message end end diff --git a/spec/cucumber/core/test/runner_spec.rb b/spec/cucumber/core/test/runner_spec.rb index 16d980b7..780bc51b 100644 --- a/spec/cucumber/core/test/runner_spec.rb +++ b/spec/cucumber/core/test/runner_spec.rb @@ -344,7 +344,7 @@ it 'call skip, rather than execute on test step of the hook' do expect(failing_hook).not_to receive(:execute) - allow(failing_hook).to receive(:skip).and_return(Cucumber::Core::Test::Result::Failed.new(anything, instance_double(StandardError, backtrace: []))) + allow(failing_hook).to receive(:skip).and_return(Cucumber::Core::Test::Result::Failed.new(Cucumber::Core::Test::Result::UnknownDuration.new, instance_double(StandardError, backtrace: []))) test_case.describe_to(runner) end end