src/eric7/Testing/Interfaces/UnittestRunner.py

branch
eric7
changeset 9221
bf71ee032bb4
parent 9209
b99e7fd55fd3
child 9264
18a7312cfdb3
child 9313
6bac6775abb2
equal deleted inserted replaced
9220:e9e7eca7efee 9221:bf71ee032bb4
11 import os 11 import os
12 import sys 12 import sys
13 import time 13 import time
14 import unittest 14 import unittest
15 15
16 sys.path.insert( 16 sys.path.insert(2, os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")))
17 2,
18 os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))
19 )
20 17
21 18
22 class EricTestResult(unittest.TestResult): 19 class EricTestResult(unittest.TestResult):
23 """ 20 """
24 Class implementing a TestResult derivative to send the data via a network 21 Class implementing a TestResult derivative to send the data via a network
25 connection. 22 connection.
26 """ 23 """
24
27 def __init__(self, writer, failfast): 25 def __init__(self, writer, failfast):
28 """ 26 """
29 Constructor 27 Constructor
30 28
31 @param writer reference to the object to write the results to 29 @param writer reference to the object to write the results to
32 @type EricJsonWriter 30 @type EricJsonWriter
33 @param failfast flag indicating to stop at the first error 31 @param failfast flag indicating to stop at the first error
34 @type bool 32 @type bool
35 """ 33 """
36 super().__init__() 34 super().__init__()
37 self.__writer = writer 35 self.__writer = writer
38 self.failfast = failfast 36 self.failfast = failfast
39 self.__testsRun = 0 37 self.__testsRun = 0
40 38
41 self.__currentTestStatus = {} 39 self.__currentTestStatus = {}
42 40
43 def addFailure(self, test, err): 41 def addFailure(self, test, err):
44 """ 42 """
45 Public method called if a test failed. 43 Public method called if a test failed.
46 44
47 @param test reference to the test object 45 @param test reference to the test object
48 @type TestCase 46 @type TestCase
49 @param err tuple containing the exception data like sys.exc_info 47 @param err tuple containing the exception data like sys.exc_info
50 (exception type, exception instance, traceback) 48 (exception type, exception instance, traceback)
51 @type tuple 49 @type tuple
52 """ 50 """
53 super().addFailure(test, err) 51 super().addFailure(test, err)
54 tracebackLines = self._exc_info_to_string(err, test) 52 tracebackLines = self._exc_info_to_string(err, test)
55 53
56 self.__currentTestStatus.update({ 54 self.__currentTestStatus.update(
57 "status": "failure", 55 {
58 "traceback": tracebackLines, 56 "status": "failure",
59 }) 57 "traceback": tracebackLines,
60 58 }
59 )
60
61 def addError(self, test, err): 61 def addError(self, test, err):
62 """ 62 """
63 Public method called if a test errored. 63 Public method called if a test errored.
64 64
65 @param test reference to the test object 65 @param test reference to the test object
66 @type TestCase 66 @type TestCase
67 @param err tuple containing the exception data like sys.exc_info 67 @param err tuple containing the exception data like sys.exc_info
68 (exception type, exception instance, traceback) 68 (exception type, exception instance, traceback)
69 @type tuple 69 @type tuple
70 """ 70 """
71 super().addError(test, err) 71 super().addError(test, err)
72 tracebackLines = self._exc_info_to_string(err, test) 72 tracebackLines = self._exc_info_to_string(err, test)
73 73
74 self.__currentTestStatus.update({ 74 self.__currentTestStatus.update(
75 "status": "error", 75 {
76 "traceback": tracebackLines, 76 "status": "error",
77 }) 77 "traceback": tracebackLines,
78 78 }
79 )
80
79 def addSkip(self, test, reason): 81 def addSkip(self, test, reason):
80 """ 82 """
81 Public method called if a test was skipped. 83 Public method called if a test was skipped.
82 84
83 @param test reference to the test object 85 @param test reference to the test object
84 @type TestCase 86 @type TestCase
85 @param reason reason for skipping the test 87 @param reason reason for skipping the test
86 @type str 88 @type str
87 """ 89 """
88 super().addSkip(test, reason) 90 super().addSkip(test, reason)
89 91
90 self.__currentTestStatus.update({ 92 self.__currentTestStatus.update(
91 "status": "skipped", 93 {
92 "shortmsg": reason, 94 "status": "skipped",
93 }) 95 "shortmsg": reason,
94 96 }
97 )
98
95 def addExpectedFailure(self, test, err): 99 def addExpectedFailure(self, test, err):
96 """ 100 """
97 Public method called if a test failed expected. 101 Public method called if a test failed expected.
98 102
99 @param test reference to the test object 103 @param test reference to the test object
100 @type TestCase 104 @type TestCase
101 @param err tuple containing the exception data like sys.exc_info 105 @param err tuple containing the exception data like sys.exc_info
102 (exception type, exception instance, traceback) 106 (exception type, exception instance, traceback)
103 @type tuple 107 @type tuple
104 """ 108 """
105 super().addExpectedFailure(test, err) 109 super().addExpectedFailure(test, err)
106 tracebackLines = self._exc_info_to_string(err, test) 110 tracebackLines = self._exc_info_to_string(err, test)
107 111
108 self.__currentTestStatus.update({ 112 self.__currentTestStatus.update(
109 "status": "expected failure", 113 {
110 "traceback": tracebackLines, 114 "status": "expected failure",
111 }) 115 "traceback": tracebackLines,
112 116 }
117 )
118
113 def addUnexpectedSuccess(self, test): 119 def addUnexpectedSuccess(self, test):
114 """ 120 """
115 Public method called if a test succeeded expectedly. 121 Public method called if a test succeeded expectedly.
116 122
117 @param test reference to the test object 123 @param test reference to the test object
118 @type TestCase 124 @type TestCase
119 """ 125 """
120 super().addUnexpectedSuccess(test) 126 super().addUnexpectedSuccess(test)
121 127
122 self.__currentTestStatus["status"] = "unexpected success" 128 self.__currentTestStatus["status"] = "unexpected success"
123 129
124 def addSubTest(self, test, subtest, err): 130 def addSubTest(self, test, subtest, err):
125 """ 131 """
126 Public method called for each subtest to record its result. 132 Public method called for each subtest to record its result.
127 133
128 @param test reference to the test object 134 @param test reference to the test object
129 @type TestCase 135 @type TestCase
130 @param subtest reference to the subtest object 136 @param subtest reference to the subtest object
131 @type TestCase 137 @type TestCase
132 @param err tuple containing the exception data like sys.exc_info 138 @param err tuple containing the exception data like sys.exc_info
134 @type tuple 140 @type tuple
135 """ 141 """
136 if err is not None: 142 if err is not None:
137 super().addSubTest(test, subtest, err) 143 super().addSubTest(test, subtest, err)
138 tracebackLines = self._exc_info_to_string(err, test) 144 tracebackLines = self._exc_info_to_string(err, test)
139 status = ( 145 status = "failure" if issubclass(err[0], test.failureException) else "error"
140 "failure" 146
141 if issubclass(err[0], test.failureException) else
142 "error"
143 )
144
145 # record the last subtest fail status as the overall status 147 # record the last subtest fail status as the overall status
146 self.__currentTestStatus["status"] = status 148 self.__currentTestStatus["status"] = status
147 149
148 self.__writer.write({ 150 self.__writer.write(
149 "event": "result", 151 {
150 "status": status, 152 "event": "result",
151 "name": str(subtest), 153 "status": status,
152 "id": subtest.id(), 154 "name": str(subtest),
153 "description": subtest.shortDescription(), 155 "id": subtest.id(),
154 "traceback": tracebackLines, 156 "description": subtest.shortDescription(),
155 "subtest": True, 157 "traceback": tracebackLines,
156 }) 158 "subtest": True,
157 159 }
160 )
161
158 if self.failfast: 162 if self.failfast:
159 self.stop() 163 self.stop()
160 else: 164 else:
161 self.__writer.write({ 165 self.__writer.write(
162 "event": "result", 166 {
163 "status": "success", 167 "event": "result",
164 "name": str(subtest), 168 "status": "success",
165 "id": subtest.id(), 169 "name": str(subtest),
166 "description": subtest.shortDescription(), 170 "id": subtest.id(),
167 "subtest": True, 171 "description": subtest.shortDescription(),
168 }) 172 "subtest": True,
169 173 }
174 )
175
170 def startTest(self, test): 176 def startTest(self, test):
171 """ 177 """
172 Public method called at the start of a test. 178 Public method called at the start of a test.
173 179
174 @param test reference to the test object 180 @param test reference to the test object
175 @type TestCase 181 @type TestCase
176 """ 182 """
177 super().startTest(test) 183 super().startTest(test)
178 184
179 self.__testsRun += 1 185 self.__testsRun += 1
180 self.__currentTestStatus = { 186 self.__currentTestStatus = {
181 "event": "result", 187 "event": "result",
182 "status": "success", 188 "status": "success",
183 "name": str(test), 189 "name": str(test),
184 "id": test.id(), 190 "id": test.id(),
185 "description": test.shortDescription(), 191 "description": test.shortDescription(),
186 "subtest": False, 192 "subtest": False,
187 } 193 }
188 194
189 self.__writer.write({ 195 self.__writer.write(
190 "event": "started", 196 {
191 "name": str(test), 197 "event": "started",
192 "id": test.id(), 198 "name": str(test),
193 "description": test.shortDescription(), 199 "id": test.id(),
194 }) 200 "description": test.shortDescription(),
195 201 }
202 )
203
196 self.__startTime = time.monotonic_ns() 204 self.__startTime = time.monotonic_ns()
197 205
198 def stopTest(self, test): 206 def stopTest(self, test):
199 """ 207 """
200 Public method called at the end of a test. 208 Public method called at the end of a test.
201 209
202 @param test reference to the test object 210 @param test reference to the test object
203 @type TestCase 211 @type TestCase
204 """ 212 """
205 stopTime = time.monotonic_ns() 213 stopTime = time.monotonic_ns()
206 duration = (stopTime - self.__startTime) / 1_000_000 # ms 214 duration = (stopTime - self.__startTime) / 1_000_000 # ms
207 215
208 super().stopTest(test) 216 super().stopTest(test)
209 217
210 self.__currentTestStatus["duration_ms"] = duration 218 self.__currentTestStatus["duration_ms"] = duration
211 self.__writer.write(self.__currentTestStatus) 219 self.__writer.write(self.__currentTestStatus)
212 220
213 def startTestRun(self): 221 def startTestRun(self):
214 """ 222 """
215 Public method called once before any tests are executed. 223 Public method called once before any tests are executed.
216 """ 224 """
217 self.__totalStartTime = time.monotonic_ns() 225 self.__totalStartTime = time.monotonic_ns()
218 self.__testsRun = 0 226 self.__testsRun = 0
219 227
220 def stopTestRun(self): 228 def stopTestRun(self):
221 """ 229 """
222 Public method called once after all tests are executed. 230 Public method called once after all tests are executed.
223 """ 231 """
224 stopTime = time.monotonic_ns() 232 stopTime = time.monotonic_ns()
225 duration = (stopTime - self.__totalStartTime) / 1_000_000_000 # s 233 duration = (stopTime - self.__totalStartTime) / 1_000_000_000 # s
226 234
227 self.__writer.write({ 235 self.__writer.write(
228 "event": "finished", 236 {
229 "duration_s": duration, 237 "event": "finished",
230 "tests": self.__testsRun, 238 "duration_s": duration,
231 }) 239 "tests": self.__testsRun,
240 }
241 )
232 242
233 243
234 def _assembleTestCasesList(suite): 244 def _assembleTestCasesList(suite):
235 """ 245 """
236 Protected function to assemble a list of test cases included in a test 246 Protected function to assemble a list of test cases included in a test
237 suite. 247 suite.
238 248
239 @param suite test suite to be inspected 249 @param suite test suite to be inspected
240 @type unittest.TestSuite 250 @type unittest.TestSuite
241 @return list of tuples containing the test case ID, the string 251 @return list of tuples containing the test case ID, the string
242 representation and the short description 252 representation and the short description
243 @rtype list of tuples of (str, str) 253 @rtype list of tuples of (str, str)
247 if isinstance(test, unittest.TestSuite): 257 if isinstance(test, unittest.TestSuite):
248 testCases.extend(_assembleTestCasesList(test)) 258 testCases.extend(_assembleTestCasesList(test))
249 else: 259 else:
250 testId = test.id() 260 testId = test.id()
251 if ( 261 if (
252 "ModuleImportFailure" not in testId and 262 "ModuleImportFailure" not in testId
253 "LoadTestsFailure" not in testId and 263 and "LoadTestsFailure" not in testId
254 "_FailedTest" not in testId 264 and "_FailedTest" not in testId
255 ): 265 ):
256 testCases.append( 266 testCases.append((testId, str(test), test.shortDescription()))
257 (testId, str(test), test.shortDescription())
258 )
259 return testCases 267 return testCases
260 268
261 269
262 def runtest(argv): 270 def runtest(argv):
263 """ 271 """
264 Function to run the tests. 272 Function to run the tests.
265 273
266 @param argv list of command line parameters. 274 @param argv list of command line parameters.
267 @type list of str 275 @type list of str
268 """ 276 """
269 from EricNetwork.EricJsonStreamWriter import EricJsonWriter 277 from EricNetwork.EricJsonStreamWriter import EricJsonWriter
278
270 writer = EricJsonWriter(argv[0], int(argv[1])) 279 writer = EricJsonWriter(argv[0], int(argv[1]))
271 del argv[:2] 280 del argv[:2]
272 281
273 # process arguments 282 # process arguments
274 if argv[0] == "discover": 283 if argv[0] == "discover":
275 discover = True 284 discover = True
276 argv.pop(0) 285 argv.pop(0)
277 if argv[0] == "--start-directory": 286 if argv[0] == "--start-directory":
278 discoveryStart = argv[1] 287 discoveryStart = argv[1]
279 del argv[:2] 288 del argv[:2]
280 else: 289 else:
281 discover = False 290 discover = False
282 discoveryStart = "" 291 discoveryStart = ""
283 292
284 failfast = "--failfast" in argv 293 failfast = "--failfast" in argv
285 if failfast: 294 if failfast:
286 argv.remove("--failfast") 295 argv.remove("--failfast")
287 296
288 collectCoverage = "--cover" in argv 297 collectCoverage = "--cover" in argv
289 if collectCoverage: 298 if collectCoverage:
290 argv.remove("--cover") 299 argv.remove("--cover")
291 coverageErase = "--cover-erase" in argv 300 coverageErase = "--cover-erase" in argv
292 if coverageErase: 301 if coverageErase:
293 argv.remove("--cover-erase") 302 argv.remove("--cover-erase")
294 if "--cover-file" in argv: 303 if "--cover-file" in argv:
295 index = argv.index("--cover-file") 304 index = argv.index("--cover-file")
296 covDataFile = argv[index + 1] 305 covDataFile = argv[index + 1]
297 del argv[index:index + 2] 306 del argv[index : index + 2]
298 else: 307 else:
299 covDataFile = "" 308 covDataFile = ""
300 309
301 if argv and argv[0] == "--failed-only": 310 if argv and argv[0] == "--failed-only":
302 if discover: 311 if discover:
303 testFileName = "" 312 testFileName = ""
304 failed = argv[1:] 313 failed = argv[1:]
305 else: 314 else:
310 if discover: 319 if discover:
311 testFileName = testName = "" 320 testFileName = testName = ""
312 else: 321 else:
313 testFileName, testName = argv[:2] 322 testFileName, testName = argv[:2]
314 del argv[:2] 323 del argv[:2]
315 324
316 testCases = argv[:] 325 testCases = argv[:]
317 326
318 if testFileName: 327 if testFileName:
319 sys.path.insert(1, os.path.dirname(os.path.abspath(testFileName))) 328 sys.path.insert(1, os.path.dirname(os.path.abspath(testFileName)))
320 elif discoveryStart: 329 elif discoveryStart:
321 sys.path.insert(1, os.path.abspath(discoveryStart)) 330 sys.path.insert(1, os.path.abspath(discoveryStart))
322 331
323 # setup test coverage 332 # setup test coverage
324 if collectCoverage: 333 if collectCoverage:
325 if not covDataFile: 334 if not covDataFile:
326 if discover: 335 if discover:
327 covname = os.path.join(discoveryStart, "test") 336 covname = os.path.join(discoveryStart, "test")
328 elif testFileName: 337 elif testFileName:
329 covname = os.path.splitext( 338 covname = os.path.splitext(os.path.abspath(testFileName))[0]
330 os.path.abspath(testFileName))[0]
331 else: 339 else:
332 covname = "test" 340 covname = "test"
333 covDataFile = "{0}.coverage".format(covname) 341 covDataFile = "{0}.coverage".format(covname)
334 if not os.path.isabs(covDataFile): 342 if not os.path.isabs(covDataFile):
335 covDataFile = os.path.abspath(covDataFile) 343 covDataFile = os.path.abspath(covDataFile)
336 344
337 sys.path.insert( 345 sys.path.insert(
338 2, 346 2,
339 os.path.abspath(os.path.join( 347 os.path.abspath(
340 os.path.dirname(__file__), "..", "..", "DebugClients", "Python" 348 os.path.join(
341 )) 349 os.path.dirname(__file__), "..", "..", "DebugClients", "Python"
350 )
351 ),
342 ) 352 )
343 from DebugClients.Python.coverage import Coverage 353 from DebugClients.Python.coverage import Coverage
354
344 cover = Coverage(data_file=covDataFile) 355 cover = Coverage(data_file=covDataFile)
345 if coverageErase: 356 if coverageErase:
346 cover.erase() 357 cover.erase()
347 cover.start() 358 cover.start()
348 else: 359 else:
349 cover = None 360 cover = None
350 361
351 try: 362 try:
352 testLoader = unittest.TestLoader() 363 testLoader = unittest.TestLoader()
353 if discover and not failed: 364 if discover and not failed:
354 if testCases: 365 if testCases:
355 test = testLoader.loadTestsFromNames(testCases) 366 test = testLoader.loadTestsFromNames(testCases)
356 else: 367 else:
357 test = testLoader.discover(discoveryStart) 368 test = testLoader.discover(discoveryStart)
358 else: 369 else:
359 if testFileName: 370 if testFileName:
360 module = __import__(os.path.splitext( 371 module = __import__(os.path.splitext(os.path.basename(testFileName))[0])
361 os.path.basename(testFileName))[0])
362 else: 372 else:
363 module = None 373 module = None
364 if failed: 374 if failed:
365 if module: 375 if module:
366 failed = [t.split(".", 1)[1] 376 failed = [t.split(".", 1)[1] for t in failed]
367 for t in failed] 377 test = testLoader.loadTestsFromNames(failed, module)
368 test = testLoader.loadTestsFromNames(
369 failed, module)
370 else: 378 else:
371 test = testLoader.loadTestsFromName( 379 test = testLoader.loadTestsFromName(testName, module)
372 testName, module)
373 except Exception as err: 380 except Exception as err:
374 print("Exception:", str(err)) 381 print("Exception:", str(err))
375 writer.write({ 382 writer.write(
376 "event": "collecterror", 383 {
377 "error": str(err), 384 "event": "collecterror",
378 }) 385 "error": str(err),
386 }
387 )
379 sys.exit(1) 388 sys.exit(1)
380 389
381 collectedTests = { 390 collectedTests = {
382 "event": "collected", 391 "event": "collected",
383 "tests": [ 392 "tests": [
384 {"id": id, "name": name, "description": desc} 393 {"id": id, "name": name, "description": desc}
385 for id, name, desc in _assembleTestCasesList(test) 394 for id, name, desc in _assembleTestCasesList(test)
386 ] 395 ],
387 } 396 }
388 writer.write(collectedTests) 397 writer.write(collectedTests)
389 398
390 testResult = EricTestResult(writer, failfast) 399 testResult = EricTestResult(writer, failfast)
391 startTestRun = getattr(testResult, 'startTestRun', None) 400 startTestRun = getattr(testResult, "startTestRun", None)
392 if startTestRun is not None: 401 if startTestRun is not None:
393 startTestRun() 402 startTestRun()
394 try: 403 try:
395 test.run(testResult) 404 test.run(testResult)
396 finally: 405 finally:
397 if cover: 406 if cover:
398 cover.stop() 407 cover.stop()
399 cover.save() 408 cover.save()
400 writer.write({ 409 writer.write(
401 "event": "coverage", 410 {
402 "file": covDataFile, 411 "event": "coverage",
403 }) 412 "file": covDataFile,
404 stopTestRun = getattr(testResult, 'stopTestRun', None) 413 }
414 )
415 stopTestRun = getattr(testResult, "stopTestRun", None)
405 if stopTestRun is not None: 416 if stopTestRun is not None:
406 stopTestRun() 417 stopTestRun()
407 418
408 writer.close() 419 writer.close()
409 sys.exit(0) 420 sys.exit(0)
410 421
411 if __name__ == '__main__': 422
423 if __name__ == "__main__":
412 if len(sys.argv) > 1: 424 if len(sys.argv) > 1:
413 command = sys.argv[1] 425 command = sys.argv[1]
414 if command == "installed": 426 if command == "installed":
415 sys.exit(0) 427 sys.exit(0)
416 428
417 elif command == "versions": 429 elif command == "versions":
418 import platform 430 import platform
431
419 versions = { 432 versions = {
420 "name": "unittest", 433 "name": "unittest",
421 "version": platform.python_version(), 434 "version": platform.python_version(),
422 "plugins": [], 435 "plugins": [],
423 } 436 }
424 print(json.dumps(versions)) 437 print(json.dumps(versions))
425 sys.exit(0) 438 sys.exit(0)
426 439
427 elif command == "runtest": 440 elif command == "runtest":
428 runtest(sys.argv[2:]) 441 runtest(sys.argv[2:])
429 sys.exit(0) 442 sys.exit(0)
430 443
431 sys.exit(42) 444 sys.exit(42)
432 445
433 # 446 #
434 # eflag: noqa = M801 447 # eflag: noqa = M801

eric ide

mercurial