11 import os |
11 import os |
12 import sys |
12 import sys |
13 import time |
13 import time |
14 import unittest |
14 import unittest |
15 |
15 |
16 sys.path.insert( |
16 sys.path.insert(2, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))) |
17 2, |
|
18 os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) |
|
19 ) |
|
20 |
17 |
21 |
18 |
22 class EricTestResult(unittest.TestResult): |
19 class EricTestResult(unittest.TestResult): |
23 """ |
20 """ |
24 Class implementing a TestResult derivative to send the data via a network |
21 Class implementing a TestResult derivative to send the data via a network |
25 connection. |
22 connection. |
26 """ |
23 """ |
|
24 |
27 def __init__(self, writer, failfast): |
25 def __init__(self, writer, failfast): |
28 """ |
26 """ |
29 Constructor |
27 Constructor |
30 |
28 |
31 @param writer reference to the object to write the results to |
29 @param writer reference to the object to write the results to |
32 @type EricJsonWriter |
30 @type EricJsonWriter |
33 @param failfast flag indicating to stop at the first error |
31 @param failfast flag indicating to stop at the first error |
34 @type bool |
32 @type bool |
35 """ |
33 """ |
36 super().__init__() |
34 super().__init__() |
37 self.__writer = writer |
35 self.__writer = writer |
38 self.failfast = failfast |
36 self.failfast = failfast |
39 self.__testsRun = 0 |
37 self.__testsRun = 0 |
40 |
38 |
41 self.__currentTestStatus = {} |
39 self.__currentTestStatus = {} |
42 |
40 |
43 def addFailure(self, test, err): |
41 def addFailure(self, test, err): |
44 """ |
42 """ |
45 Public method called if a test failed. |
43 Public method called if a test failed. |
46 |
44 |
47 @param test reference to the test object |
45 @param test reference to the test object |
48 @type TestCase |
46 @type TestCase |
49 @param err tuple containing the exception data like sys.exc_info |
47 @param err tuple containing the exception data like sys.exc_info |
50 (exception type, exception instance, traceback) |
48 (exception type, exception instance, traceback) |
51 @type tuple |
49 @type tuple |
52 """ |
50 """ |
53 super().addFailure(test, err) |
51 super().addFailure(test, err) |
54 tracebackLines = self._exc_info_to_string(err, test) |
52 tracebackLines = self._exc_info_to_string(err, test) |
55 |
53 |
56 self.__currentTestStatus.update({ |
54 self.__currentTestStatus.update( |
57 "status": "failure", |
55 { |
58 "traceback": tracebackLines, |
56 "status": "failure", |
59 }) |
57 "traceback": tracebackLines, |
60 |
58 } |
|
59 ) |
|
60 |
61 def addError(self, test, err): |
61 def addError(self, test, err): |
62 """ |
62 """ |
63 Public method called if a test errored. |
63 Public method called if a test errored. |
64 |
64 |
65 @param test reference to the test object |
65 @param test reference to the test object |
66 @type TestCase |
66 @type TestCase |
67 @param err tuple containing the exception data like sys.exc_info |
67 @param err tuple containing the exception data like sys.exc_info |
68 (exception type, exception instance, traceback) |
68 (exception type, exception instance, traceback) |
69 @type tuple |
69 @type tuple |
70 """ |
70 """ |
71 super().addError(test, err) |
71 super().addError(test, err) |
72 tracebackLines = self._exc_info_to_string(err, test) |
72 tracebackLines = self._exc_info_to_string(err, test) |
73 |
73 |
74 self.__currentTestStatus.update({ |
74 self.__currentTestStatus.update( |
75 "status": "error", |
75 { |
76 "traceback": tracebackLines, |
76 "status": "error", |
77 }) |
77 "traceback": tracebackLines, |
78 |
78 } |
|
79 ) |
|
80 |
79 def addSkip(self, test, reason): |
81 def addSkip(self, test, reason): |
80 """ |
82 """ |
81 Public method called if a test was skipped. |
83 Public method called if a test was skipped. |
82 |
84 |
83 @param test reference to the test object |
85 @param test reference to the test object |
84 @type TestCase |
86 @type TestCase |
85 @param reason reason for skipping the test |
87 @param reason reason for skipping the test |
86 @type str |
88 @type str |
87 """ |
89 """ |
88 super().addSkip(test, reason) |
90 super().addSkip(test, reason) |
89 |
91 |
90 self.__currentTestStatus.update({ |
92 self.__currentTestStatus.update( |
91 "status": "skipped", |
93 { |
92 "shortmsg": reason, |
94 "status": "skipped", |
93 }) |
95 "shortmsg": reason, |
94 |
96 } |
|
97 ) |
|
98 |
95 def addExpectedFailure(self, test, err): |
99 def addExpectedFailure(self, test, err): |
96 """ |
100 """ |
97 Public method called if a test failed expected. |
101 Public method called if a test failed expected. |
98 |
102 |
99 @param test reference to the test object |
103 @param test reference to the test object |
100 @type TestCase |
104 @type TestCase |
101 @param err tuple containing the exception data like sys.exc_info |
105 @param err tuple containing the exception data like sys.exc_info |
102 (exception type, exception instance, traceback) |
106 (exception type, exception instance, traceback) |
103 @type tuple |
107 @type tuple |
104 """ |
108 """ |
105 super().addExpectedFailure(test, err) |
109 super().addExpectedFailure(test, err) |
106 tracebackLines = self._exc_info_to_string(err, test) |
110 tracebackLines = self._exc_info_to_string(err, test) |
107 |
111 |
108 self.__currentTestStatus.update({ |
112 self.__currentTestStatus.update( |
109 "status": "expected failure", |
113 { |
110 "traceback": tracebackLines, |
114 "status": "expected failure", |
111 }) |
115 "traceback": tracebackLines, |
112 |
116 } |
|
117 ) |
|
118 |
113 def addUnexpectedSuccess(self, test): |
119 def addUnexpectedSuccess(self, test): |
114 """ |
120 """ |
115 Public method called if a test succeeded expectedly. |
121 Public method called if a test succeeded expectedly. |
116 |
122 |
117 @param test reference to the test object |
123 @param test reference to the test object |
118 @type TestCase |
124 @type TestCase |
119 """ |
125 """ |
120 super().addUnexpectedSuccess(test) |
126 super().addUnexpectedSuccess(test) |
121 |
127 |
122 self.__currentTestStatus["status"] = "unexpected success" |
128 self.__currentTestStatus["status"] = "unexpected success" |
123 |
129 |
124 def addSubTest(self, test, subtest, err): |
130 def addSubTest(self, test, subtest, err): |
125 """ |
131 """ |
126 Public method called for each subtest to record its result. |
132 Public method called for each subtest to record its result. |
127 |
133 |
128 @param test reference to the test object |
134 @param test reference to the test object |
129 @type TestCase |
135 @type TestCase |
130 @param subtest reference to the subtest object |
136 @param subtest reference to the subtest object |
131 @type TestCase |
137 @type TestCase |
132 @param err tuple containing the exception data like sys.exc_info |
138 @param err tuple containing the exception data like sys.exc_info |
134 @type tuple |
140 @type tuple |
135 """ |
141 """ |
136 if err is not None: |
142 if err is not None: |
137 super().addSubTest(test, subtest, err) |
143 super().addSubTest(test, subtest, err) |
138 tracebackLines = self._exc_info_to_string(err, test) |
144 tracebackLines = self._exc_info_to_string(err, test) |
139 status = ( |
145 status = "failure" if issubclass(err[0], test.failureException) else "error" |
140 "failure" |
146 |
141 if issubclass(err[0], test.failureException) else |
|
142 "error" |
|
143 ) |
|
144 |
|
145 # record the last subtest fail status as the overall status |
147 # record the last subtest fail status as the overall status |
146 self.__currentTestStatus["status"] = status |
148 self.__currentTestStatus["status"] = status |
147 |
149 |
148 self.__writer.write({ |
150 self.__writer.write( |
149 "event": "result", |
151 { |
150 "status": status, |
152 "event": "result", |
151 "name": str(subtest), |
153 "status": status, |
152 "id": subtest.id(), |
154 "name": str(subtest), |
153 "description": subtest.shortDescription(), |
155 "id": subtest.id(), |
154 "traceback": tracebackLines, |
156 "description": subtest.shortDescription(), |
155 "subtest": True, |
157 "traceback": tracebackLines, |
156 }) |
158 "subtest": True, |
157 |
159 } |
|
160 ) |
|
161 |
158 if self.failfast: |
162 if self.failfast: |
159 self.stop() |
163 self.stop() |
160 else: |
164 else: |
161 self.__writer.write({ |
165 self.__writer.write( |
162 "event": "result", |
166 { |
163 "status": "success", |
167 "event": "result", |
164 "name": str(subtest), |
168 "status": "success", |
165 "id": subtest.id(), |
169 "name": str(subtest), |
166 "description": subtest.shortDescription(), |
170 "id": subtest.id(), |
167 "subtest": True, |
171 "description": subtest.shortDescription(), |
168 }) |
172 "subtest": True, |
169 |
173 } |
|
174 ) |
|
175 |
170 def startTest(self, test): |
176 def startTest(self, test): |
171 """ |
177 """ |
172 Public method called at the start of a test. |
178 Public method called at the start of a test. |
173 |
179 |
174 @param test reference to the test object |
180 @param test reference to the test object |
175 @type TestCase |
181 @type TestCase |
176 """ |
182 """ |
177 super().startTest(test) |
183 super().startTest(test) |
178 |
184 |
179 self.__testsRun += 1 |
185 self.__testsRun += 1 |
180 self.__currentTestStatus = { |
186 self.__currentTestStatus = { |
181 "event": "result", |
187 "event": "result", |
182 "status": "success", |
188 "status": "success", |
183 "name": str(test), |
189 "name": str(test), |
184 "id": test.id(), |
190 "id": test.id(), |
185 "description": test.shortDescription(), |
191 "description": test.shortDescription(), |
186 "subtest": False, |
192 "subtest": False, |
187 } |
193 } |
188 |
194 |
189 self.__writer.write({ |
195 self.__writer.write( |
190 "event": "started", |
196 { |
191 "name": str(test), |
197 "event": "started", |
192 "id": test.id(), |
198 "name": str(test), |
193 "description": test.shortDescription(), |
199 "id": test.id(), |
194 }) |
200 "description": test.shortDescription(), |
195 |
201 } |
|
202 ) |
|
203 |
196 self.__startTime = time.monotonic_ns() |
204 self.__startTime = time.monotonic_ns() |
197 |
205 |
198 def stopTest(self, test): |
206 def stopTest(self, test): |
199 """ |
207 """ |
200 Public method called at the end of a test. |
208 Public method called at the end of a test. |
201 |
209 |
202 @param test reference to the test object |
210 @param test reference to the test object |
203 @type TestCase |
211 @type TestCase |
204 """ |
212 """ |
205 stopTime = time.monotonic_ns() |
213 stopTime = time.monotonic_ns() |
206 duration = (stopTime - self.__startTime) / 1_000_000 # ms |
214 duration = (stopTime - self.__startTime) / 1_000_000 # ms |
207 |
215 |
208 super().stopTest(test) |
216 super().stopTest(test) |
209 |
217 |
210 self.__currentTestStatus["duration_ms"] = duration |
218 self.__currentTestStatus["duration_ms"] = duration |
211 self.__writer.write(self.__currentTestStatus) |
219 self.__writer.write(self.__currentTestStatus) |
212 |
220 |
213 def startTestRun(self): |
221 def startTestRun(self): |
214 """ |
222 """ |
215 Public method called once before any tests are executed. |
223 Public method called once before any tests are executed. |
216 """ |
224 """ |
217 self.__totalStartTime = time.monotonic_ns() |
225 self.__totalStartTime = time.monotonic_ns() |
218 self.__testsRun = 0 |
226 self.__testsRun = 0 |
219 |
227 |
220 def stopTestRun(self): |
228 def stopTestRun(self): |
221 """ |
229 """ |
222 Public method called once after all tests are executed. |
230 Public method called once after all tests are executed. |
223 """ |
231 """ |
224 stopTime = time.monotonic_ns() |
232 stopTime = time.monotonic_ns() |
225 duration = (stopTime - self.__totalStartTime) / 1_000_000_000 # s |
233 duration = (stopTime - self.__totalStartTime) / 1_000_000_000 # s |
226 |
234 |
227 self.__writer.write({ |
235 self.__writer.write( |
228 "event": "finished", |
236 { |
229 "duration_s": duration, |
237 "event": "finished", |
230 "tests": self.__testsRun, |
238 "duration_s": duration, |
231 }) |
239 "tests": self.__testsRun, |
|
240 } |
|
241 ) |
232 |
242 |
233 |
243 |
234 def _assembleTestCasesList(suite): |
244 def _assembleTestCasesList(suite): |
235 """ |
245 """ |
236 Protected function to assemble a list of test cases included in a test |
246 Protected function to assemble a list of test cases included in a test |
237 suite. |
247 suite. |
238 |
248 |
239 @param suite test suite to be inspected |
249 @param suite test suite to be inspected |
240 @type unittest.TestSuite |
250 @type unittest.TestSuite |
241 @return list of tuples containing the test case ID, the string |
251 @return list of tuples containing the test case ID, the string |
242 representation and the short description |
252 representation and the short description |
243 @rtype list of tuples of (str, str) |
253 @rtype list of tuples of (str, str) |
247 if isinstance(test, unittest.TestSuite): |
257 if isinstance(test, unittest.TestSuite): |
248 testCases.extend(_assembleTestCasesList(test)) |
258 testCases.extend(_assembleTestCasesList(test)) |
249 else: |
259 else: |
250 testId = test.id() |
260 testId = test.id() |
251 if ( |
261 if ( |
252 "ModuleImportFailure" not in testId and |
262 "ModuleImportFailure" not in testId |
253 "LoadTestsFailure" not in testId and |
263 and "LoadTestsFailure" not in testId |
254 "_FailedTest" not in testId |
264 and "_FailedTest" not in testId |
255 ): |
265 ): |
256 testCases.append( |
266 testCases.append((testId, str(test), test.shortDescription())) |
257 (testId, str(test), test.shortDescription()) |
|
258 ) |
|
259 return testCases |
267 return testCases |
260 |
268 |
261 |
269 |
262 def runtest(argv): |
270 def runtest(argv): |
263 """ |
271 """ |
264 Function to run the tests. |
272 Function to run the tests. |
265 |
273 |
266 @param argv list of command line parameters. |
274 @param argv list of command line parameters. |
267 @type list of str |
275 @type list of str |
268 """ |
276 """ |
269 from EricNetwork.EricJsonStreamWriter import EricJsonWriter |
277 from EricNetwork.EricJsonStreamWriter import EricJsonWriter |
|
278 |
270 writer = EricJsonWriter(argv[0], int(argv[1])) |
279 writer = EricJsonWriter(argv[0], int(argv[1])) |
271 del argv[:2] |
280 del argv[:2] |
272 |
281 |
273 # process arguments |
282 # process arguments |
274 if argv[0] == "discover": |
283 if argv[0] == "discover": |
275 discover = True |
284 discover = True |
276 argv.pop(0) |
285 argv.pop(0) |
277 if argv[0] == "--start-directory": |
286 if argv[0] == "--start-directory": |
278 discoveryStart = argv[1] |
287 discoveryStart = argv[1] |
279 del argv[:2] |
288 del argv[:2] |
280 else: |
289 else: |
281 discover = False |
290 discover = False |
282 discoveryStart = "" |
291 discoveryStart = "" |
283 |
292 |
284 failfast = "--failfast" in argv |
293 failfast = "--failfast" in argv |
285 if failfast: |
294 if failfast: |
286 argv.remove("--failfast") |
295 argv.remove("--failfast") |
287 |
296 |
288 collectCoverage = "--cover" in argv |
297 collectCoverage = "--cover" in argv |
289 if collectCoverage: |
298 if collectCoverage: |
290 argv.remove("--cover") |
299 argv.remove("--cover") |
291 coverageErase = "--cover-erase" in argv |
300 coverageErase = "--cover-erase" in argv |
292 if coverageErase: |
301 if coverageErase: |
293 argv.remove("--cover-erase") |
302 argv.remove("--cover-erase") |
294 if "--cover-file" in argv: |
303 if "--cover-file" in argv: |
295 index = argv.index("--cover-file") |
304 index = argv.index("--cover-file") |
296 covDataFile = argv[index + 1] |
305 covDataFile = argv[index + 1] |
297 del argv[index:index + 2] |
306 del argv[index : index + 2] |
298 else: |
307 else: |
299 covDataFile = "" |
308 covDataFile = "" |
300 |
309 |
301 if argv and argv[0] == "--failed-only": |
310 if argv and argv[0] == "--failed-only": |
302 if discover: |
311 if discover: |
303 testFileName = "" |
312 testFileName = "" |
304 failed = argv[1:] |
313 failed = argv[1:] |
305 else: |
314 else: |
310 if discover: |
319 if discover: |
311 testFileName = testName = "" |
320 testFileName = testName = "" |
312 else: |
321 else: |
313 testFileName, testName = argv[:2] |
322 testFileName, testName = argv[:2] |
314 del argv[:2] |
323 del argv[:2] |
315 |
324 |
316 testCases = argv[:] |
325 testCases = argv[:] |
317 |
326 |
318 if testFileName: |
327 if testFileName: |
319 sys.path.insert(1, os.path.dirname(os.path.abspath(testFileName))) |
328 sys.path.insert(1, os.path.dirname(os.path.abspath(testFileName))) |
320 elif discoveryStart: |
329 elif discoveryStart: |
321 sys.path.insert(1, os.path.abspath(discoveryStart)) |
330 sys.path.insert(1, os.path.abspath(discoveryStart)) |
322 |
331 |
323 # setup test coverage |
332 # setup test coverage |
324 if collectCoverage: |
333 if collectCoverage: |
325 if not covDataFile: |
334 if not covDataFile: |
326 if discover: |
335 if discover: |
327 covname = os.path.join(discoveryStart, "test") |
336 covname = os.path.join(discoveryStart, "test") |
328 elif testFileName: |
337 elif testFileName: |
329 covname = os.path.splitext( |
338 covname = os.path.splitext(os.path.abspath(testFileName))[0] |
330 os.path.abspath(testFileName))[0] |
|
331 else: |
339 else: |
332 covname = "test" |
340 covname = "test" |
333 covDataFile = "{0}.coverage".format(covname) |
341 covDataFile = "{0}.coverage".format(covname) |
334 if not os.path.isabs(covDataFile): |
342 if not os.path.isabs(covDataFile): |
335 covDataFile = os.path.abspath(covDataFile) |
343 covDataFile = os.path.abspath(covDataFile) |
336 |
344 |
337 sys.path.insert( |
345 sys.path.insert( |
338 2, |
346 2, |
339 os.path.abspath(os.path.join( |
347 os.path.abspath( |
340 os.path.dirname(__file__), "..", "..", "DebugClients", "Python" |
348 os.path.join( |
341 )) |
349 os.path.dirname(__file__), "..", "..", "DebugClients", "Python" |
|
350 ) |
|
351 ), |
342 ) |
352 ) |
343 from DebugClients.Python.coverage import Coverage |
353 from DebugClients.Python.coverage import Coverage |
|
354 |
344 cover = Coverage(data_file=covDataFile) |
355 cover = Coverage(data_file=covDataFile) |
345 if coverageErase: |
356 if coverageErase: |
346 cover.erase() |
357 cover.erase() |
347 cover.start() |
358 cover.start() |
348 else: |
359 else: |
349 cover = None |
360 cover = None |
350 |
361 |
351 try: |
362 try: |
352 testLoader = unittest.TestLoader() |
363 testLoader = unittest.TestLoader() |
353 if discover and not failed: |
364 if discover and not failed: |
354 if testCases: |
365 if testCases: |
355 test = testLoader.loadTestsFromNames(testCases) |
366 test = testLoader.loadTestsFromNames(testCases) |
356 else: |
367 else: |
357 test = testLoader.discover(discoveryStart) |
368 test = testLoader.discover(discoveryStart) |
358 else: |
369 else: |
359 if testFileName: |
370 if testFileName: |
360 module = __import__(os.path.splitext( |
371 module = __import__(os.path.splitext(os.path.basename(testFileName))[0]) |
361 os.path.basename(testFileName))[0]) |
|
362 else: |
372 else: |
363 module = None |
373 module = None |
364 if failed: |
374 if failed: |
365 if module: |
375 if module: |
366 failed = [t.split(".", 1)[1] |
376 failed = [t.split(".", 1)[1] for t in failed] |
367 for t in failed] |
377 test = testLoader.loadTestsFromNames(failed, module) |
368 test = testLoader.loadTestsFromNames( |
|
369 failed, module) |
|
370 else: |
378 else: |
371 test = testLoader.loadTestsFromName( |
379 test = testLoader.loadTestsFromName(testName, module) |
372 testName, module) |
|
373 except Exception as err: |
380 except Exception as err: |
374 print("Exception:", str(err)) |
381 print("Exception:", str(err)) |
375 writer.write({ |
382 writer.write( |
376 "event": "collecterror", |
383 { |
377 "error": str(err), |
384 "event": "collecterror", |
378 }) |
385 "error": str(err), |
|
386 } |
|
387 ) |
379 sys.exit(1) |
388 sys.exit(1) |
380 |
389 |
381 collectedTests = { |
390 collectedTests = { |
382 "event": "collected", |
391 "event": "collected", |
383 "tests": [ |
392 "tests": [ |
384 {"id": id, "name": name, "description": desc} |
393 {"id": id, "name": name, "description": desc} |
385 for id, name, desc in _assembleTestCasesList(test) |
394 for id, name, desc in _assembleTestCasesList(test) |
386 ] |
395 ], |
387 } |
396 } |
388 writer.write(collectedTests) |
397 writer.write(collectedTests) |
389 |
398 |
390 testResult = EricTestResult(writer, failfast) |
399 testResult = EricTestResult(writer, failfast) |
391 startTestRun = getattr(testResult, 'startTestRun', None) |
400 startTestRun = getattr(testResult, "startTestRun", None) |
392 if startTestRun is not None: |
401 if startTestRun is not None: |
393 startTestRun() |
402 startTestRun() |
394 try: |
403 try: |
395 test.run(testResult) |
404 test.run(testResult) |
396 finally: |
405 finally: |
397 if cover: |
406 if cover: |
398 cover.stop() |
407 cover.stop() |
399 cover.save() |
408 cover.save() |
400 writer.write({ |
409 writer.write( |
401 "event": "coverage", |
410 { |
402 "file": covDataFile, |
411 "event": "coverage", |
403 }) |
412 "file": covDataFile, |
404 stopTestRun = getattr(testResult, 'stopTestRun', None) |
413 } |
|
414 ) |
|
415 stopTestRun = getattr(testResult, "stopTestRun", None) |
405 if stopTestRun is not None: |
416 if stopTestRun is not None: |
406 stopTestRun() |
417 stopTestRun() |
407 |
418 |
408 writer.close() |
419 writer.close() |
409 sys.exit(0) |
420 sys.exit(0) |
410 |
421 |
411 if __name__ == '__main__': |
422 |
|
423 if __name__ == "__main__": |
412 if len(sys.argv) > 1: |
424 if len(sys.argv) > 1: |
413 command = sys.argv[1] |
425 command = sys.argv[1] |
414 if command == "installed": |
426 if command == "installed": |
415 sys.exit(0) |
427 sys.exit(0) |
416 |
428 |
417 elif command == "versions": |
429 elif command == "versions": |
418 import platform |
430 import platform |
|
431 |
419 versions = { |
432 versions = { |
420 "name": "unittest", |
433 "name": "unittest", |
421 "version": platform.python_version(), |
434 "version": platform.python_version(), |
422 "plugins": [], |
435 "plugins": [], |
423 } |
436 } |
424 print(json.dumps(versions)) |
437 print(json.dumps(versions)) |
425 sys.exit(0) |
438 sys.exit(0) |
426 |
439 |
427 elif command == "runtest": |
440 elif command == "runtest": |
428 runtest(sys.argv[2:]) |
441 runtest(sys.argv[2:]) |
429 sys.exit(0) |
442 sys.exit(0) |
430 |
443 |
431 sys.exit(42) |
444 sys.exit(42) |
432 |
445 |
433 # |
446 # |
434 # eflag: noqa = M801 |
447 # eflag: noqa = M801 |