Skip to content

Commit 5220e53

Browse files
New option '--retry-total' as a circuit breaker for option '--retry' (#1669)
* fix steps which can only be used once per scenario * add --retry-total option Co-authored-by: Matt Wynne <[email protected]>
1 parent 778ae9b commit 5220e53

File tree

8 files changed

+162
-19
lines changed

8 files changed

+162
-19
lines changed

CHANGELOG.md

+1
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ Please visit [cucumber/CONTRIBUTING.md](https://github.com/cucumber/cucumber/blo
1010

1111
## [Unreleased]
1212
### Added
13+
* Add option `--retry-total` ([PR#](https://github.com/cucumber/cucumber-ruby/pull/1669))
1314

1415
### Changed
1516

features/docs/cli/retry_failing_tests.feature

+28
Original file line numberDiff line numberDiff line change
@@ -95,3 +95,31 @@ Feature: Retry failing tests
9595
9696
3 scenarios (2 flaky, 1 passed)
9797
"""
98+
99+
Scenario: Retry each test but suspend retrying after 2 failing tests, so later tests are not retried
100+
Given a scenario "Fails-forever-1" that fails
101+
And a scenario "Fails-forever-2" that fails
102+
When I run `cucumber -q --retry 1 --retry-total 2 --format summary`
103+
Then it should fail with:
104+
"""
105+
5 scenarios (4 failed, 1 passed)
106+
"""
107+
And it should fail with:
108+
"""
109+
Fails-forever-1
110+
Fails-forever-1 ✗
111+
Fails-forever-1 ✗
112+
113+
Fails-forever-2
114+
Fails-forever-2 ✗
115+
Fails-forever-2 ✗
116+
117+
Fails-once feature
118+
Fails-once ✗
119+
120+
Fails-twice feature
121+
Fails-twice ✗
122+
123+
Solid
124+
Solid ✓
125+
"""

features/lib/step_definitions/scenarios_and_steps.rb

+8-8
Original file line numberDiff line numberDiff line change
@@ -30,26 +30,26 @@ def snake_case(name)
3030
Given('a scenario {string} that passes') do |name|
3131
create_feature(name) do
3232
create_scenario(name) do
33-
' Given it passes'
33+
" Given it passes in #{name}"
3434
end
3535
end
3636

3737
write_file(
3838
"features/step_definitions/#{name}_steps.rb",
39-
step_definition('/^it passes$/', 'expect(true).to be true')
39+
step_definition("/^it passes in #{name}$/", 'expect(true).to be true')
4040
)
4141
end
4242

4343
Given('a scenario {string} that fails') do |name|
4444
create_feature(name) do
4545
create_scenario(name) do
46-
' Given it fails'
46+
" Given it fails in #{name}"
4747
end
4848
end
4949

5050
write_file(
5151
"features/step_definitions/#{name}_steps.rb",
52-
step_definition('/^it fails$/', 'expect(false).to be true')
52+
step_definition("/^it fails in #{name}$/", 'expect(false).to be true')
5353
)
5454
end
5555

@@ -58,14 +58,14 @@ def snake_case(name)
5858

5959
create_feature("#{full_name} feature") do
6060
create_scenario(full_name) do
61-
' Given it fails once, then passes'
61+
" Given it fails once, then passes in #{full_name}"
6262
end
6363
end
6464

6565
write_file(
6666
"features/step_definitions/#{name}_steps.rb",
6767
step_definition(
68-
'/^it fails once, then passes$/',
68+
"/^it fails once, then passes in #{full_name}$/",
6969
[
7070
"$#{name} += 1",
7171
"expect($#{name}).to be > 1"
@@ -84,14 +84,14 @@ def snake_case(name)
8484

8585
create_feature("#{full_name} feature") do
8686
create_scenario(full_name) do
87-
' Given it fails twice, then passes'
87+
" Given it fails twice, then passes in #{full_name}"
8888
end
8989
end
9090

9191
write_file(
9292
"features/step_definitions/#{name}_steps.rb",
9393
step_definition(
94-
'/^it fails twice, then passes$/',
94+
"/^it fails twice, then passes in #{full_name}$/",
9595
[
9696
"$#{name} ||= 0",
9797
"$#{name} += 1",

lib/cucumber/cli/options.rb

+14-3
Original file line numberDiff line numberDiff line change
@@ -56,11 +56,12 @@ class Options
5656
NO_PROFILE_LONG_FLAG = '--no-profile'.freeze
5757
FAIL_FAST_FLAG = '--fail-fast'.freeze
5858
RETRY_FLAG = '--retry'.freeze
59+
RETRY_TOTAL_FLAG = '--retry-total'.freeze
5960
OPTIONS_WITH_ARGS = [
6061
'-r', '--require', '--i18n-keywords', '-f', '--format', '-o',
6162
'--out', '-t', '--tags', '-n', '--name', '-e', '--exclude',
62-
PROFILE_SHORT_FLAG, PROFILE_LONG_FLAG, RETRY_FLAG, '-l',
63-
'--lines', '--port', '-I', '--snippet-type'
63+
PROFILE_SHORT_FLAG, PROFILE_LONG_FLAG, RETRY_FLAG, RETRY_TOTAL_FLAG,
64+
'-l', '--lines', '--port', '-I', '--snippet-type'
6465
].freeze
6566
ORDER_TYPES = %w[defined random].freeze
6667
TAG_LIMIT_MATCHER = /(?<tag_name>@\w+):(?<limit>\d+)/x
@@ -108,6 +109,7 @@ def parse!(args) # rubocop:disable Metrics/AbcSize, Metrics/MethodLength
108109
opts.on('-j DIR', '--jars DIR', 'Load all the jars under DIR') { |jars| load_jars(jars) } if Cucumber::JRUBY
109110

110111
opts.on("#{RETRY_FLAG} ATTEMPTS", *retry_msg) { |v| set_option :retry, v.to_i }
112+
opts.on("#{RETRY_TOTAL_FLAG} TESTS", *retry_total_msg) { |v| set_option :retry_total, v.to_i }
111113
opts.on('--i18n-languages', *i18n_languages_msg) { list_languages_and_exit }
112114
opts.on('--i18n-keywords LANG', *i18n_keywords_msg) { |lang| language lang }
113115
opts.on(FAIL_FAST_FLAG, 'Exit immediately following the first failing scenario') { set_option :fail_fast }
@@ -271,6 +273,13 @@ def retry_msg
271273
['Specify the number of times to retry failing tests (default: 0)']
272274
end
273275

276+
def retry_total_msg
277+
[
278+
'The total number of failing test after which retrying of tests is suspended.',
279+
'Example: --retry-total 10 -> Will stop retrying tests after 10 failing tests.'
280+
]
281+
end
282+
274283
def name_msg
275284
[
276285
'Only execute the feature elements which match part of the given name.',
@@ -543,6 +552,7 @@ def reverse_merge(other_options) # rubocop:disable Metrics/AbcSize
543552
end
544553

545554
@options[:retry] = other_options[:retry] if @options[:retry].zero?
555+
@options[:retry_total] = other_options[:retry_total] if @options[:retry_total].infinite?
546556

547557
self
548558
end
@@ -616,7 +626,8 @@ def default_options
616626
snippets: true,
617627
source: true,
618628
duration: true,
619-
retry: 0
629+
retry: 0,
630+
retry_total: Float::INFINITY
620631
}
621632
end
622633
end

lib/cucumber/configuration.rb

+6-1
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,10 @@ def retry_attempts
7878
@options[:retry]
7979
end
8080

81+
def retry_total_tests
82+
@options[:retry_total]
83+
end
84+
8185
def guess?
8286
@options[:guess]
8387
end
@@ -273,7 +277,8 @@ def default_options
273277
snippets: true,
274278
source: true,
275279
duration: true,
276-
event_bus: Cucumber::Events.make_event_bus
280+
event_bus: Cucumber::Events.make_event_bus,
281+
retry_total: Float::INFINITY
277282
}
278283
end
279284

lib/cucumber/filters/retry.rb

+20-1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,11 @@
77
module Cucumber
88
module Filters
99
class Retry < Core::Filter.new(:configuration)
10+
def initialize(*_args)
11+
super
12+
@total_permanently_failed = 0
13+
end
14+
1015
def test_case(test_case)
1116
configuration.on_event(:test_case_finished) do |event|
1217
next unless retry_required?(test_case, event)
@@ -21,7 +26,21 @@ def test_case(test_case)
2126
private
2227

2328
def retry_required?(test_case, event)
24-
event.test_case == test_case && event.result.failed? && test_case_counts[test_case] < configuration.retry_attempts
29+
return false unless event.test_case == test_case
30+
31+
return false unless event.result.failed?
32+
33+
return false if @total_permanently_failed >= configuration.retry_total_tests
34+
35+
retry_required = test_case_counts[test_case] < configuration.retry_attempts
36+
if retry_required
37+
# retry test
38+
true
39+
else
40+
# test failed after max. attempts
41+
@total_permanently_failed += 1
42+
false
43+
end
2544
end
2645

2746
def test_case_counts

spec/cucumber/cli/options_spec.rb

+34
Original file line numberDiff line numberDiff line change
@@ -379,6 +379,26 @@ def with_env(name, value)
379379
end
380380
end
381381
end
382+
383+
context '--retry-total TESTS' do
384+
context '--retry-total option not defined on the command line' do
385+
it 'uses the --retry-total option from the profile' do
386+
given_cucumber_yml_defined_as('foo' => '--retry-total 2')
387+
options.parse!(%w[-p foo])
388+
389+
expect(options[:retry_total]).to be 2
390+
end
391+
end
392+
393+
context '--retry-total option defined on the command line' do
394+
it 'ignores the --retry-total option from the profile' do
395+
given_cucumber_yml_defined_as('foo' => '--retry-total 2')
396+
options.parse!(%w[--retry-total 1 -p foo])
397+
398+
expect(options[:retry_total]).to be 1
399+
end
400+
end
401+
end
382402
end
383403

384404
context '-P or --no-profile' do
@@ -441,6 +461,20 @@ def with_env(name, value)
441461
end
442462
end
443463

464+
context '--retry-total TESTS' do
465+
it 'is INFINITY by default' do
466+
after_parsing('') do
467+
expect(options[:retry_total]).to eql Float::INFINITY
468+
end
469+
end
470+
471+
it 'sets the options[:retry_total] value' do
472+
after_parsing('--retry 3 --retry-total 10') do
473+
expect(options[:retry_total]).to eql 10
474+
end
475+
end
476+
end
477+
444478
it 'assigns any extra arguments as paths to features' do
445479
after_parsing('-f pretty my_feature.feature my_other_features') do
446480
expect(options[:paths]).to eq ['my_feature.feature', 'my_other_features']

spec/cucumber/filters/retry_spec.rb

+51-6
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,8 @@
1313
include Cucumber::Core
1414
include Cucumber::Events
1515

16-
let(:configuration) { Cucumber::Configuration.new(retry: 2) }
16+
let(:configuration) { Cucumber::Configuration.new(retry: 2, retry_total: retry_total) }
17+
let(:retry_total) { Float::INFINITY }
1718
let(:id) { double }
1819
let(:name) { double }
1920
let(:location) { double }
@@ -55,12 +56,32 @@
5556
context 'consistently failing test case' do
5657
let(:result) { Cucumber::Core::Test::Result::Failed.new(0, StandardError.new) }
5758

58-
it 'describes the test case the specified number of times' do
59-
expect(receiver).to receive(:test_case) { |test_case|
60-
configuration.notify :test_case_finished, test_case, result
61-
}.exactly(3).times
59+
shared_examples 'retries the test case the specified number of times' do |expected_nr_of_times|
60+
it 'describes the test case the specified number of times' do
61+
expect(receiver).to receive(:test_case) { |test_case|
62+
configuration.notify :test_case_finished, test_case, result
63+
}.exactly(expected_nr_of_times).times
6264

63-
filter.test_case(test_case)
65+
filter.test_case(test_case)
66+
end
67+
end
68+
69+
context 'when retry_total infinit' do
70+
let(:retry_total) { Float::INFINITY }
71+
72+
include_examples 'retries the test case the specified number of times', 3
73+
end
74+
75+
context 'when retry_total 1' do
76+
let(:retry_total) { 1 }
77+
78+
include_examples 'retries the test case the specified number of times', 3
79+
end
80+
81+
context 'when retry_total 0' do
82+
let(:retry_total) { 0 }
83+
84+
include_examples 'retries the test case the specified number of times', 1
6485
end
6586
end
6687

@@ -100,4 +121,28 @@
100121
end
101122
end
102123
end
124+
125+
context 'too many failing tests' do
126+
let(:retry_total) { 1 }
127+
let(:always_failing_test_case1) do
128+
Cucumber::Core::Test::Case.new(id, name, [double('test steps')], 'test.rb:1', tags, language)
129+
end
130+
let(:always_failing_test_case2) do
131+
Cucumber::Core::Test::Case.new(id, name, [double('test steps')], 'test.rb:9', tags, language)
132+
end
133+
let(:fail_result) { Cucumber::Core::Test::Result::Failed.new(0, StandardError.new) }
134+
135+
it 'stops retrying tests' do
136+
expect(receiver).to receive(:test_case).with(always_failing_test_case1) { |test_case|
137+
configuration.notify :test_case_finished, test_case, fail_result
138+
}.ordered.exactly(3).times
139+
140+
expect(receiver).to receive(:test_case).with(always_failing_test_case2) { |test_case|
141+
configuration.notify :test_case_finished, test_case, fail_result
142+
}.ordered.exactly(1).times
143+
144+
filter.test_case(always_failing_test_case1)
145+
filter.test_case(always_failing_test_case2)
146+
end
147+
end
103148
end

0 commit comments

Comments
 (0)