Skip to content

Commit 3bc34be

Browse files
estolfoestolfo
committed
[XPACK] Blacklist individual tests, not entire files
1 parent ce6401f commit 3bc34be

File tree

2 files changed

+97
-46
lines changed

2 files changed

+97
-46
lines changed

‎elasticsearch-xpack/spec/spec_helper.rb‎

Lines changed: 62 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@
2222
end
2323

2424
TEST_HOST, TEST_PORT = split_hosts.first.split(':')
25+
else
26+
TEST_HOST, TEST_PORT = 'localhost', 9200
2527
end
2628

2729
raw_certificate = File.read(File.join(PROJECT_PATH, '/.ci/certs/testnode.crt'))
@@ -56,49 +58,83 @@
5658
end
5759

5860

59-
6061
YAML_FILES_DIRECTORY = "#{File.expand_path(File.dirname('..'), '..')}" +
6162
"/tmp/elasticsearch/x-pack/plugin/src/test/resources/rest-api-spec/test"
62-
skipped_files = []
63+
SINGLE_TEST = if ENV['SINGLE_TEST']
64+
["#{File.expand_path(File.dirname('..'), '..')}" +
65+
"/tmp/elasticsearch/x-pack/plugin/src/test/resources/rest-api-spec/test/#{ENV['SINGLE_TEST']}"]
66+
end
6367

64-
# Respone from Elasticsearch includes the ca.crt, so length doesn't match.
65-
skipped_files += Dir.glob("#{YAML_FILES_DIRECTORY}/ssl/10_basic.yml")
68+
SKIPPED_TESTS = []
6669

67-
# Current license is basic.
68-
skipped_files += Dir.glob("#{YAML_FILES_DIRECTORY}/license/20_put_license.yml")
70+
# Response from Elasticsearch includes the ca.crt, so length doesn't match.
71+
SKIPPED_TESTS << { file: 'ssl/10_basic.yml',
72+
description: 'Test get SSL certificates' }
6973

70-
# ArgumentError for empty body
71-
skipped_files += Dir.glob("#{YAML_FILES_DIRECTORY}/watcher/put_watch/10_basic.yml")
74+
# Searching the monitoring index returns no results.
75+
SKIPPED_TESTS << { file: 'monitoring/bulk/20_privileges.yml',
76+
description: 'Monitoring Bulk API' }
7277

73-
# The number of shards when a snapshot is successfully created is more than 1. Maybe because of the security index?
74-
skipped_files += Dir.glob("#{YAML_FILES_DIRECTORY}/snapshot/10_basic.yml")
78+
# Searching the monitoring index returns no results.
79+
SKIPPED_TESTS << { file: 'monitoring/bulk/10_basic.yml',
80+
description: 'Bulk indexing of monitoring data on closed indices should throw an export exception' }
7581

7682
# The test inserts an invalid license, which makes all subsequent tests fail.
77-
skipped_files += Dir.glob("#{YAML_FILES_DIRECTORY}/xpack/15_basic.yml")
83+
SKIPPED_TESTS << { file: 'xpack/15_basic.yml',
84+
description: '*' }
7885

79-
# 'invalidated_tokens' is returning 5 in 'Test invalidate user's tokens' test.
80-
skipped_files += Dir.glob("#{YAML_FILES_DIRECTORY}/token/10_basic.yml")
86+
# The test inserts an invalid license, which makes all subsequent tests fail.
87+
SKIPPED_TESTS << { file: 'license/20_put_license.yml',
88+
description: '*' }
8189

82-
# 'invalidated_tokens' is returning 5 in 'Test invalidate user's tokens' test.
83-
skipped_files += Dir.glob("#{YAML_FILES_DIRECTORY}/token/10_basic.yml")
90+
# The test inserts an invalid license, which makes all subsequent tests fail.
91+
SKIPPED_TESTS << { file: 'token/10_basic.yml',
92+
description: "Test invalidate user's tokens" }
8493

85-
# Searching the monitoring index returns no results.
86-
skipped_files += Dir.glob("#{YAML_FILES_DIRECTORY}/monitoring/bulk/10_basic.yml")
87-
skipped_files += Dir.glob("#{YAML_FILES_DIRECTORY}/monitoring/bulk/20_privileges.yml")
94+
# The test inserts an invalid license, which makes all subsequent tests fail.
95+
SKIPPED_TESTS << { file: 'token/10_basic.yml',
96+
description: "Test invalidate realm's tokens" }
97+
98+
# The number of shards when a snapshot is successfully created is more than 1. Maybe because of the security index?
99+
SKIPPED_TESTS << { file: 'snapshot/10_basic.yml',
100+
description: 'Create a source only snapshot and then restore it' }
101+
102+
# ArgumentError for empty body
103+
SKIPPED_TESTS << { file: 'watcher/put_watch/10_basic.yml',
104+
description: 'Test empty body is rejected by put watch' }
105+
106+
# Operation times out "failed_node_exception"
107+
SKIPPED_TESTS << { file: 'ml/set_upgrade_mode.yml',
108+
description: 'Setting upgrade_mode to enabled' }
109+
110+
# Operation times out "failed_node_exception"
111+
SKIPPED_TESTS << { file: 'ml/set_upgrade_mode.yml',
112+
description: 'Setting upgrade_mode to disabled' }
113+
114+
# Operation times out "failed_node_exception"
115+
SKIPPED_TESTS << { file: 'ml/set_upgrade_mode.yml',
116+
description: 'Setting upgrade mode to disabled from enabled' }
88117

89118
# Operation times out "failed_node_exception"
90-
skipped_files += Dir.glob("#{YAML_FILES_DIRECTORY}/ml/set_upgrade_mode.yml")
119+
SKIPPED_TESTS << { file: 'ml/set_upgrade_mode.yml',
120+
description: 'Attempt to open job when upgrade_mode is enabled' }
91121

92-
# 'Test Deprecations' has non-zero length node_settings field
93-
skipped_files += Dir.glob("#{YAML_FILES_DIRECTORY}/deprecation/10_basic.yml")
122+
# Non-zero length node_settings field
123+
SKIPPED_TESTS << { file: 'deprecation/10_basic.yml',
124+
description: 'Test Deprecations' }
94125

95-
SINGLE_TEST = nil
96-
# Uncomment the following line and set it to a file when a single test should be run.
97-
# SINGLE_TEST = ["#{File.expand_path(File.dirname('..'), '..')}" +
98-
# "/tmp/elasticsearch/x-pack/plugin/src/test/resources/rest-api-spec/test/ml/jobs_crud.yml"]
99126

127+
# TODO
128+
SKIPPED_TESTS << { file: 'ml/forecast.yml',
129+
description: 'Test forecast unknown job' }
130+
# TODO
131+
SKIPPED_TESTS << { file: 'ml/post_data.yml',
132+
description: 'Test POST data with invalid parameters' }
133+
# TODO
134+
SKIPPED_TESTS << { file: 'ml/post_data.yml',
135+
description: 'Test Flush data with invalid parameters' }
100136

101-
REST_API_YAML_FILES = SINGLE_TEST || Dir.glob("#{YAML_FILES_DIRECTORY}/**/*.yml") - skipped_files
137+
REST_API_YAML_FILES = SINGLE_TEST || Dir.glob("#{YAML_FILES_DIRECTORY}/**/*.yml")
102138
REST_API_YAML_SKIP_FEATURES = ['warnings'].freeze
103139

104140

‎elasticsearch-xpack/spec/support/test_file/test.rb‎

Lines changed: 35 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ class Test
1212
attr_reader :description
1313
attr_reader :test_file
1414
attr_reader :cached_values
15+
attr_reader :file_basename
1516

1617
# Actions that if followed by a 'do' action, indicate that they complete their task group.
1718
# For example, consider this sequence of actions:
@@ -53,11 +54,14 @@ class Test
5354
# @since 6.2.0
5455
def initialize(test_file, test_definition)
5556
@test_file = test_file
57+
@file_basename = test_file.name.gsub("#{YAML_FILES_DIRECTORY}/", '')
5658
@description = test_definition.keys.first
57-
@skip = test_definition[description].select { |doc| doc['skip'] }.compact
5859
@definition = test_definition[description].select { |doc| !doc.key?('skip') }
5960
@definition.delete_if { |doc| doc['skip'] }
6061
@cached_values = {}
62+
63+
skip_definitions = test_definition[description].select { |doc| doc['skip'] }.compact
64+
@skip = skip_definitions unless skip_definitions.empty?
6165
end
6266

6367
# Get the list of task groups in this test.
@@ -143,31 +147,42 @@ def run(client)
143147
#
144148
# @since 6.2.0
145149
def skip_test?(client, features_to_skip = test_file.features_to_skip)
146-
return false if @skip.empty?
147-
range_partition = /\s*-\s*/
148-
@skip.collect { |s| s['skip'] }.any? do |skip|
149-
if !(features_to_skip & ([skip['features']].flatten || [])).empty?
150-
true
151-
elsif skip['version'] == 'all'
152-
true
153-
elsif versions = skip['version'] && skip['version'].partition(range_partition)
154-
low = versions[0]
155-
high = versions[2] unless versions[2] == ''
156-
range = low..high
157-
begin
158-
client_version = client.info['version']['number']
159-
rescue
160-
warn('Could not determine Elasticsearch version when checking if test should be skipped.')
161-
end
162-
range.cover?(client_version)
150+
return true if pre_defined_skip?
151+
152+
if @skip
153+
@skip.collect { |s| s['skip'] }.any? do |skip|
154+
contains_features_to_skip?(features_to_skip, skip) ||
155+
!version_requirement_met?(client, skip)
163156
end
164157
end
165158
end
166159

167160
private
168161

169-
def meets_version_requirement?(version)
170-
version >= MIN_REQUIRED_VERSION && version <= MAX_REQUIRED_VERSION
162+
def contains_features_to_skip?(features_to_skip, skip_defintion)
163+
!(features_to_skip & ([skip_defintion['features']].flatten || [])).empty?
164+
end
165+
166+
def pre_defined_skip?
167+
SKIPPED_TESTS.find do |t|
168+
file_basename == t[:file] && (description == t[:description] || t[:description] == '*')
169+
end
170+
end
171+
172+
def version_requirement_met?(client, skip_definition)
173+
return false if skip_definition['version'] == 'all'
174+
range_partition = /\s*-\s*/
175+
if versions = skip_definition['version'] && skip_definition['version'].partition(range_partition)
176+
low = versions[0]
177+
high = versions[2] unless versions[2] == ''
178+
range = low..high
179+
begin
180+
client_version = client.info['version']['number']
181+
rescue
182+
warn('Could not determine Elasticsearch version when checking if test should be skipped.')
183+
end
184+
range.cover?(client_version)
185+
end || true
171186
end
172187

173188
def is_a_validation?(action)

0 commit comments

Comments
 (0)