diff --git a/CHANGELOG.md b/CHANGELOG.md index 6992ce2c..e2af3ea2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,10 @@ Please visit [cucumber/CONTRIBUTING.md](https://github.com/cucumber/cucumber/blo ## [Unreleased] +### Changed +- Change to use worst Test Step result as the Test Case result +([#317](https://github.com/cucumber/cucumber-ruby-core/pull/317)) + ## [16.2.0] - 2026-02-06 ### Changed - Added the test result type 'ambiguous' diff --git a/lib/cucumber/core/test/result.rb b/lib/cucumber/core/test/result.rb index d8e20a00..c044370b 100644 --- a/lib/cucumber/core/test/result.rb +++ b/lib/cucumber/core/test/result.rb @@ -7,7 +7,7 @@ module Cucumber module Core module Test module Result - TYPES = %i[failed ambiguous flaky skipped undefined pending passed unknown].freeze + TYPES = %i[failed ambiguous flaky undefined pending skipped passed unknown].freeze STRICT_AFFECTED_TYPES = %i[flaky undefined pending].freeze def self.ok?(type, strict: StrictConfiguration.new) diff --git a/lib/cucumber/core/test/runner.rb b/lib/cucumber/core/test/runner.rb index 58c90deb..236170c3 100644 --- a/lib/cucumber/core/test/runner.rb +++ b/lib/cucumber/core/test/runner.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require 'cucumber/core/test/timer' +require 'cucumber/messages/helpers/test_step_result_comparator' module Cucumber module Core @@ -45,6 +46,8 @@ def done end class RunningTestCase + include Cucumber::Messages::Helpers::TestStepResultComparator + def initialize @timer = Timer.new.start @status = Status::Unknown.new(Result::Unknown.new) @@ -59,27 +62,27 @@ def result end def failed(step_result) - @status = Status::Failing.new(step_result) + not_passing(step_result) self end def ambiguous(step_result) - @status = Status::Ambiguous.new(step_result) + failed(step_result) self end def passed(step_result) - @status = Status::Passing.new(step_result) + @status = Status::Passing.new(step_result) if test_step_result_rankings[step_result.to_message.status] > test_step_result_rankings[status.step_result_message.status] self end def pending(_message, step_result) - @status = Status::Pending.new(step_result) + failed(step_result) self end def skipped(step_result) - @status = Status::Skipping.new(step_result) + failed(step_result) self end @@ -96,6 +99,13 @@ def duration(_step_duration, _step_result) self end + private + + def not_passing(step_result) + @status = Status::NotPassing.new(step_result) if test_step_result_rankings[step_result.to_message.status] > test_step_result_rankings[status.step_result_message.status] + self + end + attr_reader :status private :status @@ -118,6 +128,10 @@ def execute(test_step, monitor, &) def result raise NoMethodError, 'Override me' end + + def step_result_message + step_result.to_message + end end class Unknown < Base @@ -132,26 +146,14 @@ def result(duration) end end - class Failing < Base + class NotPassing < Base def execute(test_step, monitor) result = test_step.skip(monitor.result) - if result.undefined? - result = result.with_message(%(Undefined step: "#{test_step.text}")) - result = result.with_appended_backtrace(test_step) - end - result - end - - def result(duration) - step_result.with_duration(duration) + result = result.with_message(%(Undefined step: "#{test_step.text}")) if result.undefined? + result = result.with_appended_backtrace(test_step) unless test_step.hook? + result.describe_to(monitor, result) end - end - - Pending = Class.new(Failing) - - Ambiguous = Class.new(Failing) - class Skipping < Failing def result(duration) step_result.with_duration(duration) end diff --git a/spec/cucumber/core/test/runner_spec.rb b/spec/cucumber/core/test/runner_spec.rb index 65ed7669..780bc51b 100644 --- a/spec/cucumber/core/test/runner_spec.rb +++ b/spec/cucumber/core/test/runner_spec.rb @@ -250,6 +250,105 @@ test_case.describe_to(runner) end end + + context 'with an initial undefined step' do + let(:test_steps) { [undefined_step, passing_step] } + + it 'emits a test_step_finished event with an undefined result' do + expect(event_bus).to receive(:test_step_finished).with(undefined_step, anything) do |_reported_test_case, result| + expect(result).to be_undefined + end + test_case.describe_to(runner) + end + + it 'emits a test_step_finished event with a skipped result' do + expect(event_bus).to receive(:test_step_finished).with(passing_step, anything) do |_reported_test_case, result| + expect(result).to be_skipped + end + test_case.describe_to(runner) + end + + it 'emits a test_case_finished event with an undefined result' do + allow(event_bus).to receive(:test_case_finished) do |_reported_test_case, result| + expect(result).to be_undefined + expect(result.exception).to be_a StandardError + end + test_case.describe_to(runner) + end + + it 'skips, rather than executing the second step' do + expect(passing_step).not_to receive(:execute) + + allow(passing_step).to receive(:skip).and_return(Cucumber::Core::Test::Result::Skipped.new) + test_case.describe_to(runner) + end + + context 'with a following ambiguous step' do + let(:test_steps) { [undefined_step, ambiguous_step] } + + it 'emits a test_step_finished event with an undefined result' do + expect(event_bus).to receive(:test_step_finished).with(undefined_step, anything) do |_reported_test_case, result| + expect(result).to be_undefined + end + test_case.describe_to(runner) + end + + it 'emits a test_step_finished event with a ambiguous result' do + expect(event_bus).to receive(:test_step_finished).with(ambiguous_step, anything) do |_reported_test_case, result| + expect(result).to be_ambiguous + end + test_case.describe_to(runner) + end + + it 'emits a test_case_finished event with an ambiguous result' do + allow(event_bus).to receive(:test_case_finished) do |_reported_test_case, result| + expect(result).to be_ambiguous + expect(result.exception).to be_a StandardError + end + test_case.describe_to(runner) + end + + it 'skips, rather than executing the second step' do + expect(ambiguous_step).not_to receive(:execute) + + allow(ambiguous_step).to receive(:skip).and_return(Cucumber::Core::Test::Result::Ambiguous.new) + test_case.describe_to(runner) + end + end + + context 'with a failing after hook' do + let(:test_steps) { [undefined_step, failing_hook] } + + it 'emits a test_step_finished event with an undefined result' do + expect(event_bus).to receive(:test_step_finished).with(undefined_step, anything) do |_reported_test_case, result| + expect(result).to be_undefined + end + test_case.describe_to(runner) + end + + it 'emits a test_step_finished event with a failing result' do + expect(event_bus).to receive(:test_step_finished).with(failing_hook, anything) do |_reported_test_case, result| + expect(result).to be_failed + end + test_case.describe_to(runner) + end + + it 'emits a test_case_finished event with an failing result' do + allow(event_bus).to receive(:test_case_finished) do |_reported_test_case, result| + expect(result).to be_failed + expect(result.exception).to be_a StandardError + end + test_case.describe_to(runner) + end + + it 'call skip, rather than execute on test step of the hook' do + expect(failing_hook).not_to receive(:execute) + + allow(failing_hook).to receive(:skip).and_return(Cucumber::Core::Test::Result::Failed.new(Cucumber::Core::Test::Result::UnknownDuration.new, instance_double(StandardError, backtrace: []))) + test_case.describe_to(runner) + end + end + end end context 'with multiple test cases' do diff --git a/spec/support/shared_context/test_step_types.rb b/spec/support/shared_context/test_step_types.rb index 51cd040e..551956a4 100644 --- a/spec/support/shared_context/test_step_types.rb +++ b/spec/support/shared_context/test_step_types.rb @@ -8,4 +8,6 @@ let(:pending_step) { Cucumber::Core::Test::Step.new(3, 'Pending Step', double).with_action { raise Cucumber::Core::Test::Result::Pending, 'TODO' } } let(:skipping_step) { Cucumber::Core::Test::Step.new(4, 'Skipped Step', double).with_action { raise Cucumber::Core::Test::Result::Skipped } } let(:undefined_step) { Cucumber::Core::Test::Step.new(5, 'Undefined Step', double) } + let(:ambiguous_step) { Cucumber::Core::Test::Step.new(6, 'Ambiguous Step', double).with_unskippable_action { raise Cucumber::Core::Test::Result::Ambiguous } } + let(:failing_hook) { Cucumber::Core::Test::Step.new(7, 'Failing Step', double).with_unskippable_action { raise StandardError, 'Error' } } end