Skip to content

Commit 5411303

Browse files
picandocodigopicandocodigo
committed
[XPACK] Test Runner: Refactor skipped tests into yml file
1 parent 9e81734 commit 5411303

File tree

2 files changed

+145
-106
lines changed

2 files changed

+145
-106
lines changed

‎elasticsearch-xpack/spec/rest_yaml_tests_helper.rb‎

Lines changed: 12 additions & 106 deletions
Original file line numberDiff line numberDiff line change
@@ -83,114 +83,20 @@
8383
end
8484
end
8585

86-
SKIPPED_TESTS = []
87-
88-
# Respone from Elasticsearch includes the ca.crt, so length doesn't match.
89-
SKIPPED_TESTS << { file: 'ssl/10_basic.yml',
90-
description: 'Test get SSL certificates' }
91-
92-
# ArgumentError for empty body
93-
SKIPPED_TESTS << { file: 'watcher/put_watch/10_basic.yml',
94-
description: 'Test empty body is rejected by put watch' }
95-
96-
# The number of shards when a snapshot is successfully created is more than 1. Maybe because of the security index?
97-
SKIPPED_TESTS << { file: 'snapshot/10_basic.yml',
98-
description: 'Create a source only snapshot and then restore it' }
99-
100-
# The test inserts an invalid license, which makes all subsequent tests fail.
101-
SKIPPED_TESTS << { file: 'xpack/15_basic.yml',
102-
description: '*' }
103-
104-
# 'invalidated_tokens' is returning 5 in 'Test invalidate user's tokens' test.
105-
SKIPPED_TESTS << { file: 'token/10_basic.yml',
106-
description: "Test invalidate user's tokens" }
107-
108-
SKIPPED_TESTS << { file: 'token/10_basic.yml',
109-
description: "Test invalidate realm's tokens" }
110-
111-
# Possible Docker issue. The IP from the response cannot be used to connect.HTTP input supports extracting of keys
112-
SKIPPED_TESTS << { file: 'watcher/execute_watch/60_http_input.yml',
113-
description: 'HTTP input supports extracting of keys' }
114-
115-
# Searching the monitoring index returns no results.
116-
SKIPPED_TESTS << { file: 'monitoring/bulk/10_basic.yml',
117-
description: 'Bulk indexing of monitoring data on closed indices should throw an export exception' }
118-
119-
# Searching the monitoring index returns no results.
120-
SKIPPED_TESTS << { file: 'monitoring/bulk/20_privileges.yml',
121-
description: 'Monitoring Bulk API' }
122-
123-
# Operation times out "failed_node_exception"
124-
SKIPPED_TESTS << { file: 'ml/set_upgrade_mode.yml',
125-
description: 'Setting upgrade_mode to enabled' }
126-
127-
# Operation times out "failed_node_exception"
128-
SKIPPED_TESTS << { file: 'ml/set_upgrade_mode.yml',
129-
description: 'Setting upgrade_mode to disabled' }
130-
131-
# Operation times out "failed_node_exception"
132-
SKIPPED_TESTS << { file: 'ml/set_upgrade_mode.yml',
133-
description: 'Setting upgrade mode to disabled from enabled' }
134-
135-
# Operation times out "failed_node_exception"
136-
SKIPPED_TESTS << { file: 'ml/set_upgrade_mode.yml',
137-
description: 'Attempt to open job when upgrade_mode is enabled' }
138-
139-
# 'calendar3' in the field instead of 'calendar2'
140-
SKIPPED_TESTS << { file: 'ml/calendar_crud.yml',
141-
description: 'Test PageParams' }
142-
143-
# Error about creating a job that already exists.
144-
SKIPPED_TESTS << { file: 'ml/jobs_crud.yml',
145-
description: 'Test close job with body params' }
146-
147-
# Feature is currently experimental.
148-
SKIPPED_TESTS << { file: 'ml/evaluate_data_frame.yml',
149-
description: '*' }
150-
151-
# Feature is currently experimental.
152-
SKIPPED_TESTS << { file: 'ml/start_data_frame_analytics.yml',
153-
description: '*' }
154-
155-
# Feature is currently experimental.
156-
SKIPPED_TESTS << { file: 'ml/stop_data_frame_analytics.yml',
157-
description: '*' }
158-
159-
# Feature is currently experimental.
160-
SKIPPED_TESTS << { file: 'ml/data_frame_analytics_crud.yml',
161-
description: '*' }
162-
163-
# https://github.com/elastic/clients-team/issues/142
164-
SKIPPED_TESTS << { file: 'ml/forecast.yml',
165-
description: 'Test forecast unknown job' }
166-
SKIPPED_TESTS << { file: 'ml/post_data.yml',
167-
description: 'Test POST data with invalid parameters' }
168-
SKIPPED_TESTS << { file: 'ml/post_data.yml',
169-
description: 'Test Flush data with invalid parameters' }
170-
171-
# TODO: To be fixed with a release patch
172-
SKIPPED_TESTS << { file: 'api_key/10_basic.yml',
173-
description: 'Test get api key' }
174-
175-
# TODO: Failing due to processing of regexp in test
176-
SKIPPED_TESTS << { file: 'ml/explain_data_frame_analytics.yml',
177-
description: 'Test non-empty data frame given body'}
178-
179-
# Stats is not working in versions earlier than 8.0.0
180-
SKIPPED_TESTS << { file: 'analytics/usage.yml',
181-
description: 'Usage stats on analytics indices'}
182-
183-
# TODO https://github.com/elastic/elasticsearch-ruby/issues/853
184-
SKIPPED_TESTS << { file: 'ml/jobs_crud.yml',
185-
description: 'Test put job with model_memory_limit as string and lazy open' }
186-
187-
SKIPPED_TESTS << { file: 'ml/inference_crud.yml',
188-
description: 'Test delete given unused trained model' }
189-
SKIPPED_TESTS << { file: 'ml/filter_crud.yml',
190-
description: '*' }
86+
# Skipped tests
87+
file = File.expand_path(__dir__ + '/skipped_tests.yml')
88+
skipped_tests = YAML.load_file(file)
19189

19290
# The directory of rest api YAML files.
193-
REST_API_YAML_FILES = SINGLE_TEST || Dir.glob("#{YAML_FILES_DIRECTORY}/**/*.yml")
91+
REST_API_YAML_FILES = if ENV['RUN_SKIPPED_TESTS'] # only run the skipped tests if true
92+
SKIPPED_TESTS = []
93+
skipped_tests.map { |test| "#{YAML_FILES_DIRECTORY}/#{test[:file]}" }
94+
else
95+
# If not, define the skipped tests constant and try the single test or all
96+
# the tests
97+
SKIPPED_TESTS = skipped_tests
98+
SINGLE_TEST || Dir.glob("#{YAML_FILES_DIRECTORY}/**/*.yml")
99+
end
194100

195101
# The features to skip
196102
REST_API_YAML_SKIP_FEATURES = ['warnings', 'node_selector'].freeze

‎elasticsearch-xpack/spec/skipped_tests.yml‎

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
# Respone from Elasticsearch includes the ca.crt, so length doesn't match.
2+
-
3+
:file: 'ssl/10_basic.yml'
4+
:description: 'Test get SSL certificates'
5+
6+
# ArgumentError for empty body
7+
-
8+
:file: 'watcher/put_watch/10_basic.yml'
9+
:description: 'Test empty body is rejected by put watch'
10+
11+
# The number of shards when a snapshot is successfully created is more than 1. Maybe because of the security index?
12+
-
13+
:file: 'snapshot/10_basic.yml'
14+
:description: 'Create a source only snapshot and then restore it'
15+
16+
# The test inserts an invalid license, which makes all subsequent tests fail.
17+
-
18+
:file: 'xpack/15_basic.yml'
19+
:description: '*'
20+
21+
# 'invalidated_tokens' is returning 5 in 'Test invalidate user's tokens' test.
22+
-
23+
:file: 'token/10_basic.yml'
24+
:description: "Test invalidate user's tokens"
25+
26+
-
27+
:file: 'token/10_basic.yml'
28+
:description: "Test invalidate realm's tokens"
29+
30+
# Possible Docker issue. The IP from the response cannot be used to connect.HTTP input supports extracting of keys
31+
-
32+
:file: 'watcher/execute_watch/60_http_input.yml'
33+
:description: 'HTTP input supports extracting of keys'
34+
35+
# Searching the monitoring index returns no results.
36+
-
37+
:file: 'monitoring/bulk/10_basic.yml'
38+
:description: 'Bulk indexing of monitoring data on closed indices should throw an export exception'
39+
40+
# Searching the monitoring index returns no results.
41+
-
42+
:file: 'monitoring/bulk/20_privileges.yml'
43+
:description: 'Monitoring Bulk API'
44+
45+
# Operation times out "failed_node_exception"
46+
-
47+
:file: 'ml/set_upgrade_mode.yml'
48+
:description: 'Setting upgrade_mode to enabled'
49+
50+
# Operation times out "failed_node_exception"
51+
-
52+
:file: 'ml/set_upgrade_mode.yml'
53+
:description: 'Setting upgrade_mode to disabled'
54+
55+
# Operation times out "failed_node_exception"
56+
-
57+
:file: 'ml/set_upgrade_mode.yml'
58+
:description: 'Setting upgrade mode to disabled from enabled'
59+
60+
# Operation times out "failed_node_exception"
61+
-
62+
:file: 'ml/set_upgrade_mode.yml'
63+
:description: 'Attempt to open job when upgrade_mode is enabled'
64+
65+
# 'calendar3' in the field instead of 'calendar2'
66+
-
67+
:file: 'ml/calendar_crud.yml'
68+
:description: 'Test PageParams'
69+
70+
# Error about creating a job that already exists.
71+
-
72+
:file: 'ml/jobs_crud.yml'
73+
:description: 'Test close job with body params'
74+
75+
# Feature is currently experimental.
76+
-
77+
:file: 'ml/evaluate_data_frame.yml'
78+
:description: '*'
79+
80+
# # Feature is currently experimental.
81+
-
82+
:file: 'ml/start_data_frame_analytics.yml'
83+
:description: '*'
84+
85+
# # Feature is currently experimental.
86+
-
87+
:file: 'ml/stop_data_frame_analytics.yml'
88+
:description: '*'
89+
90+
# Feature is currently experimental.
91+
-
92+
:file: 'ml/data_frame_analytics_crud.yml'
93+
:description: '*'
94+
95+
# # https://github.com/elastic/clients-team/issues/142
96+
-
97+
:file: 'ml/forecast.yml'
98+
:description: 'Test forecast unknown job'
99+
-
100+
:file: 'ml/post_data.yml'
101+
:description: 'Test POST data with invalid parameters'
102+
-
103+
:file: 'ml/post_data.yml'
104+
:description: 'Test Flush data with invalid parameters'
105+
106+
-
107+
:file: 'ml/inference_crud.yml'
108+
:description: 'Test delete given unused trained model'
109+
110+
-
111+
:file: 'ml/filter_crud.yml'
112+
:description: '*'
113+
114+
# TODO: To be fixed with a release patch
115+
-
116+
:file: 'api_key/10_basic.yml'
117+
:description: 'Test get api key'
118+
119+
# TODO: Failing due to processing of regexp in test
120+
-
121+
:file: 'ml/explain_data_frame_analytics.yml'
122+
:description: 'Test non-empty data frame given body'
123+
124+
# TODO https://github.com/elastic/elasticsearch-ruby/issues/852
125+
-
126+
:file: 'analytics/usage.yml'
127+
:description: 'Usage stats on analytics indices'
128+
129+
# TODO https://github.com/elastic/elasticsearch-ruby/issues/853
130+
-
131+
:file: 'ml/jobs_crud.yml'
132+
:description: 'Test put job with model_memory_limit as string and lazy open'
133+

0 commit comments

Comments
 (0)