@@ -40,14 +40,14 @@ def files(paths):
4040 Each test file in the provided paths, as an array of test cases.
4141 """
4242 for path in paths :
43- yield json .loads (path .read_text ())
43+ yield path , json .loads (path .read_text ())
4444
4545
4646def cases (paths ):
4747 """
4848 Each test case within each file in the provided paths.
4949 """
50- for test_file in files (paths ):
50+ for _ , test_file in files (paths ):
5151 yield from test_file
5252
5353
@@ -82,52 +82,115 @@ class SanityTests(unittest.TestCase):
8282 assert cls .remote_files , "Didn't find the remote files!"
8383 print (f"Found { len (cls .remote_files )} )
8484
85+ def assertUnique (self , iterable ):
86+ """
87+ Assert that the elements of an iterable are unique.
88+ """
89+
90+ seen , duplicated = set (), set ()
91+ for each in iterable :
92+ if each in seen :
93+ duplicated .add (each )
94+ seen .add (each )
95+ self .assertFalse (duplicated , "Elements are not unique." )
96+
8597 def test_all_test_files_are_valid_json (self ):
8698 """
8799 All test files contain valid JSON.
88100 """
89101 for path in self .test_files :
90- try :
91- json .loads (path .read_text ())
92- except ValueError as error :
93- self .fail (f"{ path } { error } )
102+ with self .subTest (path = path ):
103+ try :
104+ json .loads (path .read_text ())
105+ except ValueError as error :
106+ self .fail (f"{ path } { error } )
94107
95108 def test_all_remote_files_are_valid_json (self ):
96109 """
97110 All remote files contain valid JSON.
98111 """
99112 for path in self .remote_files :
100- try :
101- json .loads (path .read_text ())
102- except ValueError as error :
103- self .fail (f"{ path } { error } )
113+ with self .subTest (path = path ):
114+ try :
115+ json .loads (path .read_text ())
116+ except ValueError as error :
117+ self .fail (f"{ path } { error } )
104118
105- def test_all_descriptions_have_reasonable_length (self ):
119+ def test_all_case_descriptions_have_reasonable_length (self ):
120+ """
121+ All cases have reasonably long descriptions.
122+ """
123+ for case in cases (self .test_files ):
124+ with self .subTest (description = case ["description" ]):
125+ self .assertLess (
126+ len (case ["description" ]),
127+ 150 ,
128+ "Description is too long (keep it to less than 150 chars)."
129+ )
130+
131+ def test_all_test_descriptions_have_reasonable_length (self ):
106132 """
107133 All tests have reasonably long descriptions.
108134 """
109135 for count , test in enumerate (tests (self .test_files )):
110- description = test ["description" ]
111- self .assertLess (
112- len (description ),
113- 70 ,
114- f" { description !r } ! (keep it to less than 70 chars)"
115- )
136+ with self . subTest ( description = test ["description" ]):
137+ self .assertLess (
138+ len (test [ " description" ] ),
139+ 70 ,
140+ "Description is too long (keep it to less than 70 chars). "
141+ )
116142 print (f"Found { count } )
117143
118- def test_all_descriptions_are_unique (self ):
144+ def test_all_case_descriptions_are_unique (self ):
145+ """
146+ All cases have unique descriptions in their files.
147+ """
148+ for path , cases in files (self .test_files ):
149+ with self .subTest (path = path ):
150+ self .assertUnique (case ["description" ] for case in cases )
151+
152+ def test_all_test_descriptions_are_unique (self ):
119153 """
120154 All test cases have unique test descriptions in their tests.
121155 """
122156 for count , case in enumerate (cases (self .test_files )):
123- descriptions = set (test ["description" ] for test in case ["tests" ])
124- self .assertEqual (
125- len (descriptions ),
126- len (case ["tests" ]),
127- f"{ case !r} ,
128- )
157+ with self .subTest (description = case ["description" ]):
158+ self .assertUnique (
159+ test ["description" ] for test in case ["tests" ]
160+ )
129161 print (f"Found { count } )
130162
163+ def test_descriptions_do_not_use_modal_verbs (self ):
164+ """
165+ Instead of saying "test that X frobs" or "X should frob" use "X frobs".
166+
167+ See e.g. https://jml.io/pages/test-docstrings.html
168+
169+ This test isn't comprehensive (it doesn't catch all the extra
170+ verbiage there), but it's just to catch whatever it manages to
171+ cover.
172+ """
173+
174+ message = (
175+ "In descriptions, don't say 'Test that X frobs' or 'X should "
176+ "frob' or 'X should be valid'. Just say 'X frobs' or 'X is "
177+ "valid'. It's shorter, and the test suite is entirely about "
178+ "what *should* be already. "
179+ "See https://jml.io/pages/test-docstrings.html for help."
180+ )
181+ for test in tests (self .test_files ):
182+ with self .subTest (description = test ["description" ]):
183+ self .assertNotRegex (
184+ test ["description" ],
185+ r"\bshould\b" ,
186+ message ,
187+ )
188+ self .assertNotRegex (
189+ test ["description" ],
190+ r"(?i)\btest(s)? that\b" ,
191+ message ,
192+ )
193+
131194 @unittest .skipIf (jsonschema is None , "Validation library not present!" )
132195 def test_all_schemas_are_valid (self ):
133196 """
@@ -141,12 +204,14 @@ class SanityTests(unittest.TestCase):
141204 if Validator is not None :
142205 test_files = collect (version )
143206 for case in cases (test_files ):
144- try :
145- Validator .check_schema (case ["schema" ])
146- except jsonschema .SchemaError as error :
147- self .fail (
148- f"{ case } { error } ,
149- )
207+ with self .subTest (case = case ):
208+ try :
209+ Validator .check_schema (case ["schema" ])
210+ except jsonschema .SchemaError :
211+ self .fail (
212+ "Found an invalid schema."
213+ "See the traceback for details on why."
214+ )
150215 else :
151216 warnings .warn (f"No schema validator for { version .name } )
152217
@@ -157,11 +222,12 @@ class SanityTests(unittest.TestCase):
157222 """
158223 Validator = jsonschema .validators .validator_for (TESTSUITE_SCHEMA )
159224 validator = Validator (TESTSUITE_SCHEMA )
160- for tests in files (self .test_files ):
161- try :
162- validator .validate (tests )
163- except jsonschema .ValidationError as error :
164- self .fail (str (error ))
225+ for path , cases in files (self .test_files ):
226+ with self .subTest (path = path ):
227+ try :
228+ validator .validate (cases )
229+ except jsonschema .ValidationError as error :
230+ self .fail (str (error ))
165231
166232
167233def main (arguments ):
0 commit comments