Skip to content

Commit db964fb

Browse files
pamfilospamfilos
committed
cms: adds datasets and triggers validation
Signed-off-by: Pamfilos Fokianos <pamfilosf@gmail.com>
1 parent a01c0e6 commit db964fb

File tree

4 files changed

+356
-8
lines changed

4 files changed

+356
-8
lines changed

‎cap/modules/deposit/validators.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,8 @@
3232

3333
deposit_validators = dict(Draft4Validator.VALIDATORS)
3434

35-
# deposit_validators['x-validate-cms-trigger'] = validate_cms_trigger
36-
# deposit_validators['x-validate-das-path'] = validate_das_path
35+
deposit_validators['x-validate-cms-trigger'] = validate_cms_trigger
36+
deposit_validators['x-validate-das-path'] = validate_das_path
3737
# deposit_validators['x-validate-cadi-id'] = validate_cadi_id
3838

3939
DepositValidator = extend(Draft4Validator, validators=deposit_validators)

‎cap/modules/experiments/validators.py‎

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -33,15 +33,19 @@ def validate_cms_trigger(validator, value, instance, schema):
3333
errors = []
3434
path = instance.get('path')
3535
year = instance.get('year')
36-
for trigger in instance.get('triggers', []):
36+
for index, trigger in enumerate(instance.get('triggers', [])):
3737
search = CMSTriggerSearch().exact_search(trigger['trigger'], path,
3838
year)
3939
if search.count() == 0:
40-
errors.append("{} is not a valid trigger for this dataset.".format(
41-
trigger['trigger']))
42-
43-
if errors:
44-
yield ValidationError(errors)
40+
errors.append({
41+
"message":
42+
"{} is not a valid trigger for this dataset.".format(
43+
trigger['trigger']),
44+
"path": ["triggers", index, "trigger"]
45+
}),
46+
47+
for error in errors:
48+
yield ValidationError(**error)
4549

4650

4751
def validate_das_path(validator, value, instance, schema):

‎tests/conftest.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -511,6 +511,11 @@ def cms_triggers_index(es):
511511
'year': 2012,
512512
'trigger': 'Trigger_2'
513513
},
514+
{
515+
'dataset': 'dataset2',
516+
'year': 2012,
517+
'trigger': 'Trigger_2'
518+
},
514519
{
515520
'dataset': 'Dataset1',
516521
'year': 2011,

‎tests/integration/deposits/test_deposit_validation.py‎

Lines changed: 339 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,339 @@
1+
# -*- coding: utf-8 -*-
2+
#
3+
# This file is part of CERN Analysis Preservation Framework.
4+
# Copyright (C) 2017 CERN.
5+
#
6+
# CERN Analysis Preservation Framework is free software; you can redistribute
7+
# it and/or modify it under the terms of the GNU General Public License as
8+
# published by the Free Software Foundation; either version 2 of the
9+
# License, or (at your option) any later version.
10+
#
11+
# CERN Analysis Preservation Framework is distributed in the hope that it will
12+
# be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
13+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14+
# General Public License for more details.
15+
#
16+
# You should have received a copy of the GNU General Public License
17+
# along with CERN Analysis Preservation Framework; if not, write to the
18+
# Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston,
19+
# MA 02111-1307, USA.
20+
#
21+
# In applying this license, CERN does not
22+
# waive the privileges and immunities granted to it by virtue of its status
23+
# as an Intergovernmental Organization or submit itself to any jurisdiction.
24+
# or submit itself to any jurisdiction.
25+
"""Integration tests for publishing deposits."""
26+
27+
import json
28+
29+
from invenio_access.models import ActionRoles, ActionUsers
30+
from pytest import mark
31+
32+
from cap.modules.experiments.permissions import exp_need_factory
33+
from conftest import _datastore
34+
35+
36+
@mark.skip
37+
def test_deposit_validation_when_schema_not_specified(client, users,
38+
auth_headers_for_user,
39+
json_headers,
40+
create_schema,
41+
create_deposit):
42+
owner = users['cms_user']
43+
headers = auth_headers_for_user(owner)
44+
create_schema('test-analysis',
45+
experiment='CMS',
46+
deposit_schema={
47+
'type': 'object',
48+
'properties': {
49+
'title': {
50+
'type': 'string'
51+
}
52+
},
53+
'additionalProperties': False
54+
},
55+
record_schema={
56+
'type': 'object',
57+
'required': ['title'],
58+
'properties': {
59+
'title': {
60+
'type': 'string'
61+
}
62+
},
63+
'additionalProperties': False
64+
},
65+
use_deposit_as_record=False)
66+
67+
deposit = create_deposit(owner,
68+
'test-analysis',
69+
experiment='CMS',
70+
metadata={"random_prop": "boom"})
71+
depid = deposit['_deposit']['id']
72+
resp = client.post(f'/deposits/{depid}/actions/publish', headers=headers)
73+
74+
assert resp.status_code == 422
75+
assert resp.json[
76+
'message'] == 'Validation error. Try again with valid data'
77+
assert resp.json['errors'][0][
78+
'message'] == "'title' is a required property"
79+
80+
81+
def test_deposit_validation_on_additional_properties(client, users,
82+
auth_headers_for_user,
83+
json_headers,
84+
create_schema,
85+
create_deposit):
86+
owner = users['cms_user']
87+
headers = auth_headers_for_user(owner)
88+
create_schema('test-analysis',
89+
experiment='CMS',
90+
deposit_schema={
91+
'type': 'object',
92+
'required': ['title'],
93+
'properties': {
94+
'title': {
95+
'type': 'string'
96+
},
97+
'obj': {
98+
'type': 'object',
99+
'properties': {
100+
'allowed_prop': {
101+
'type': 'string'
102+
}
103+
},
104+
'additionalProperties': False
105+
}
106+
},
107+
})
108+
109+
resp = client.post('/deposits',
110+
headers=headers + json_headers,
111+
data=json.dumps({"$ana_type": "test-analysis"}))
112+
113+
assert resp.status_code == 201
114+
depid = resp.json['id']
115+
116+
resp = client.put('/deposits/{}'.format(depid),
117+
headers=headers + json_headers,
118+
data=json.dumps({"obj": {
119+
"random_prop": "boom"
120+
}}))
121+
122+
# resp = client.post(f'/deposits/{depid}/actions/publish', headers=headers)
123+
124+
assert resp.status_code == 422
125+
assert resp.json[
126+
'message'] == 'Validation error. Try again with valid data'
127+
128+
129+
def test_deposit_validation_on_validate_das_path(client, users,
130+
auth_headers_for_user,
131+
json_headers, create_schema,
132+
create_deposit,
133+
das_datasets_index):
134+
owner = users['cms_user']
135+
headers = auth_headers_for_user(owner)
136+
create_schema('test-analysis-with-das-validation',
137+
experiment='CMS',
138+
deposit_schema={
139+
'type': 'object',
140+
'required': ['title'],
141+
'properties': {
142+
'title': {
143+
'type': 'string'
144+
},
145+
'obj': {
146+
'type': 'object',
147+
'properties': {
148+
'allowed_prop': {
149+
'type': 'string',
150+
'x-validate-das-path': {}
151+
}
152+
},
153+
'additionalProperties': False
154+
}
155+
},
156+
})
157+
158+
resp = client.post('/deposits',
159+
headers=headers + json_headers,
160+
data=json.dumps({"$ana_type": "test-analysis-with-das-validation"}))
161+
162+
assert resp.status_code == 201
163+
depid = resp.json['id']
164+
165+
resp = client.put('/deposits/{}'.format(depid),
166+
headers=headers + json_headers,
167+
data=json.dumps({"obj": {
168+
"allowed_prop": "boom"
169+
}}))
170+
171+
assert resp.status_code == 422
172+
assert resp.json[
173+
'message'] == 'Validation error. Try again with valid data'
174+
175+
resp = client.put('/deposits/{}'.format(depid),
176+
headers=headers + json_headers,
177+
data=json.dumps(
178+
{"obj": {
179+
"allowed_prop": "/dataset1/run1/AOD"
180+
}}))
181+
182+
assert resp.status_code == 200
183+
184+
185+
def test_deposit_validation_on_validate_das_path_and_triggers(
186+
client, users, auth_headers_for_user, json_headers, create_schema,
187+
create_deposit, das_datasets_index, cms_triggers_index):
188+
owner = users['cms_user']
189+
headers = auth_headers_for_user(owner)
190+
create_schema(
191+
'test-schema-with-das-and-trigger-validation',
192+
experiment='CMS',
193+
deposit_schema={
194+
'type': 'object',
195+
'required': ['title'],
196+
'properties': {
197+
'title': {
198+
'type': 'string'
199+
},
200+
'obj': {
201+
'type': 'object',
202+
'properties': {
203+
'allowed_prop': {
204+
'type': 'string',
205+
'x-validate-das-path': {}
206+
},
207+
"list": {
208+
"items": {
209+
"type": "object",
210+
"x-validate-cms-trigger": {},
211+
"properties": {
212+
"path": {
213+
"type": "string",
214+
"x-validate-das-path": {},
215+
"title": "Path"
216+
},
217+
"year": {
218+
"type": "number",
219+
"title": "Year"
220+
},
221+
"triggers": {
222+
"items": {
223+
"type": "object",
224+
"properties": {
225+
"trigger": {
226+
"type": "string",
227+
"title": "Trigger"
228+
},
229+
"prescale": {
230+
"type": "number",
231+
"title": "Prescale"
232+
}
233+
}
234+
},
235+
"title": "Triggers",
236+
"type": "array",
237+
"description":
238+
"Add selection triggers here",
239+
"id": "triggers"
240+
}
241+
},
242+
"required": ["path"]
243+
}
244+
}
245+
},
246+
'additionalProperties': False
247+
}
248+
},
249+
})
250+
251+
resp = client.post('/deposits',
252+
headers=headers + json_headers,
253+
data=json.dumps({"$ana_type": "test-schema-with-das-and-trigger-validation"}))
254+
255+
assert resp.status_code == 201
256+
depid = resp.json['id']
257+
258+
resp = client.put('/deposits/{}'.format(depid),
259+
headers=headers + json_headers,
260+
data=json.dumps({"obj": {
261+
"allowed_prop": "boom"
262+
}}))
263+
264+
assert resp.status_code == 422
265+
assert resp.json[
266+
'message'] == 'Validation error. Try again with valid data'
267+
268+
resp = client.put('/deposits/{}'.format(depid),
269+
headers=headers + json_headers,
270+
data=json.dumps(
271+
{"obj": {
272+
"allowed_prop": "/dataset1/run1/AOD"
273+
}}))
274+
275+
assert resp.status_code == 200
276+
277+
resp = client.put('/deposits/{}'.format(depid),
278+
headers=headers + json_headers,
279+
data=json.dumps({
280+
"obj": {
281+
"allowed_prop":
282+
"/dataset1/run1/AOD",
283+
"list": [{
284+
"path": "boom"
285+
}, {
286+
"path": "/dataset1/run1/AOD"
287+
}, {
288+
"path": "bam"
289+
}]
290+
}
291+
}))
292+
293+
assert resp.status_code == 422
294+
error_field_path = resp.json.get('errors', [])[0].get('field')
295+
assert ['obj', 'list', 0, 'path'] == error_field_path
296+
297+
resp = client.put('/deposits/{}'.format(depid),
298+
headers=headers + json_headers,
299+
data=json.dumps({
300+
"obj": {
301+
"allowed_prop":
302+
"/dataset1/run1/AOD",
303+
"list": [{
304+
"path": "/dataset2/run1/RECO"
305+
}, {
306+
"path": "/dataset1/run1/AOD"
307+
}]
308+
}
309+
}))
310+
311+
assert resp.status_code == 200
312+
313+
resp = client.put('/deposits/{}'.format(depid),
314+
headers=headers + json_headers,
315+
data=json.dumps({
316+
"obj": {
317+
"allowed_prop":
318+
"/dataset1/run1/AOD",
319+
"list": [{
320+
"path": "/dataset2/run1/RECO",
321+
"year": 2012,
322+
"triggers": [
323+
{"trigger": "wrong_trigger"},
324+
{"trigger": "Trigger_2"},
325+
{"trigger": "wrong_trigger2"}
326+
]
327+
}, {
328+
"path": "/dataset1/run1/AOD"
329+
}]
330+
}
331+
}))
332+
333+
assert resp.status_code == 422
334+
errors = resp.json.get('errors', [])
335+
assert errors[0].get('field') == ['obj', 'list', 0,
336+
"triggers", 0, "trigger"]
337+
assert errors[1].get('field') == ['obj', 'list', 0,
338+
"triggers", 2, "trigger"]
339+
assert len(errors) == 2

0 commit comments

Comments
 (0)