Mercurial Repositories > eric / file comparison
comparison: src/eric7/Plugins/CheckerPlugins/CodeStyleChecker/CodeStyleChecker.py
src/eric7/Plugins/CheckerPlugins/CodeStyleChecker/CodeStyleChecker.py
- branch
- eric7
- changeset 9209
- b99e7fd55fd3
- parent 8881
- 54e42bc2437a
- child 9219
- 964a326c58d4
equal
deleted
inserted
replaced
| 9208:3fc8dfeb6ebe | 9209:b99e7fd55fd3 |
|---|---|
| 1 # -*- coding: utf-8 -*- | |
| 2 | |
| 3 # Copyright (c) 2011 - 2022 Detlev Offenbach <detlev@die-offenbachs.de> | |
| 4 # | |
| 5 | |
| 6 """ | |
| 7 Module implementing the code style checker. | |
| 8 """ | |
| 9 | |
| 10 import queue | |
| 11 import ast | |
| 12 import sys | |
| 13 import multiprocessing | |
| 14 import contextlib | |
| 15 | |
| 16 import pycodestyle | |
| 17 | |
| 18 from Complexity.ComplexityChecker import ComplexityChecker | |
| 19 from DocStyle.DocStyleChecker import DocStyleChecker | |
| 20 from Imports.ImportsChecker import ImportsChecker | |
| 21 from Miscellaneous.MiscellaneousChecker import MiscellaneousChecker | |
| 22 from Naming.NamingStyleChecker import NamingStyleChecker | |
| 23 from PathLib.PathlibChecker import PathlibChecker | |
| 24 from Security.SecurityChecker import SecurityChecker | |
| 25 from Simplify.SimplifyChecker import SimplifyChecker | |
| 26 | |
| 27 # register the name checker | |
| 28 pycodestyle.register_check(NamingStyleChecker, NamingStyleChecker.Codes) | |
| 29 | |
| 30 | |
| 31 def initService(): | |
| 32 """ | |
| 33 Initialize the service and return the entry point. | |
| 34 | |
| 35 @return the entry point for the background client (function) | |
| 36 """ | |
| 37 return codeStyleCheck | |
| 38 | |
| 39 | |
| 40 def initBatchService(): | |
| 41 """ | |
| 42 Initialize the batch service and return the entry point. | |
| 43 | |
| 44 @return the entry point for the background client (function) | |
| 45 """ | |
| 46 return codeStyleBatchCheck | |
| 47 | |
| 48 | |
| 49 class CodeStyleCheckerReport(pycodestyle.BaseReport): | |
| 50 """ | |
| 51 Class implementing a special report to be used with our dialog. | |
| 52 """ | |
| 53 def __init__(self, options): | |
| 54 """ | |
| 55 Constructor | |
| 56 | |
| 57 @param options options for the report (optparse.Values) | |
| 58 """ | |
| 59 super().__init__(options) | |
| 60 | |
| 61 self.__repeat = options.repeat | |
| 62 self.errors = [] | |
| 63 | |
| 64 def error_args(self, line_number, offset, code, check, *args): | |
| 65 """ | |
| 66 Public method to collect the error messages. | |
| 67 | |
| 68 @param line_number line number of the issue (integer) | |
| 69 @param offset position within line of the issue (integer) | |
| 70 @param code message code (string) | |
| 71 @param check reference to the checker function (function) | |
| 72 @param args arguments for the message (list) | |
| 73 @return error code (string) | |
| 74 """ | |
| 75 code = super().error_args( | |
| 76 line_number, offset, code, check, *args) | |
| 77 if code and (self.counters[code] == 1 or self.__repeat): | |
| 78 self.errors.append( | |
| 79 { | |
| 80 "file": self.filename, | |
| 81 "line": line_number, | |
| 82 "offset": offset, | |
| 83 "code": code, | |
| 84 "args": args, | |
| 85 } | |
| 86 ) | |
| 87 return code | |
| 88 | |
| 89 | |
| 90 def extractLineFlags(line, startComment="#", endComment="", flagsLine=False): | |
| 91 """ | |
| 92 Function to extract flags starting and ending with '__' from a line | |
| 93 comment. | |
| 94 | |
| 95 @param line line to extract flags from (string) | |
| 96 @param startComment string identifying the start of the comment (string) | |
| 97 @param endComment string identifying the end of a comment (string) | |
| 98 @param flagsLine flag indicating to check for a flags only line (bool) | |
| 99 @return list containing the extracted flags (list of strings) | |
| 100 """ | |
| 101 flags = [] | |
| 102 | |
| 103 if not flagsLine or ( | |
| 104 flagsLine and line.strip().startswith(startComment)): | |
| 105 pos = line.rfind(startComment) | |
| 106 if pos >= 0: | |
| 107 comment = line[pos + len(startComment):].strip() | |
| 108 if endComment: | |
| 109 endPos = line.rfind(endComment) | |
| 110 if endPos >= 0: | |
| 111 comment = comment[:endPos] | |
| 112 flags = [f.strip() for f in comment.split() | |
| 113 if (f.startswith("__") and f.endswith("__"))] | |
| 114 flags += [f.strip().lower() for f in comment.split() | |
| 115 if f in ("noqa", "NOQA", | |
| 116 "nosec", "NOSEC", | |
| 117 "secok", "SECOK")] | |
| 118 return flags | |
| 119 | |
| 120 | |
| 121 def ignoreCode(code, lineFlags): | |
| 122 """ | |
| 123 Function to check, if the given code should be ignored as per line flags. | |
| 124 | |
| 125 @param code error code to be checked | |
| 126 @type str | |
| 127 @param lineFlags list of line flags to check against | |
| 128 @type list of str | |
| 129 @return flag indicating to ignore the code | |
| 130 @rtype bool | |
| 131 """ | |
| 132 if lineFlags: | |
| 133 | |
| 134 if ( | |
| 135 "__IGNORE_WARNING__" in lineFlags or | |
| 136 "noqa" in lineFlags or | |
| 137 "nosec" in lineFlags | |
| 138 ): | |
| 139 # ignore all warning codes | |
| 140 return True | |
| 141 | |
| 142 for flag in lineFlags: | |
| 143 # check individual warning code | |
| 144 if flag.startswith("__IGNORE_WARNING_"): | |
| 145 ignoredCode = flag[2:-2].rsplit("_", 1)[-1] | |
| 146 if code.startswith(ignoredCode): | |
| 147 return True | |
| 148 | |
| 149 return False | |
| 150 | |
| 151 | |
| 152 def securityOk(code, lineFlags): | |
| 153 """ | |
| 154 Function to check, if the given code is an acknowledged security report. | |
| 155 | |
| 156 @param code error code to be checked | |
| 157 @type str | |
| 158 @param lineFlags list of line flags to check against | |
| 159 @type list of str | |
| 160 @return flag indicating an acknowledged security report | |
| 161 @rtype bool | |
| 162 """ | |
| 163 if lineFlags: | |
| 164 return "secok" in lineFlags | |
| 165 | |
| 166 return False | |
| 167 | |
| 168 | |
| 169 def codeStyleCheck(filename, source, args): | |
| 170 """ | |
| 171 Do the code style check and/or fix found errors. | |
| 172 | |
| 173 @param filename source filename | |
| 174 @type str | |
| 175 @param source string containing the code to check | |
| 176 @type str | |
| 177 @param args arguments used by the codeStyleCheck function (list of | |
| 178 excludeMessages, includeMessages, repeatMessages, fixCodes, | |
| 179 noFixCodes, fixIssues, maxLineLength, maxDocLineLength, blankLines, | |
| 180 hangClosing, docType, codeComplexityArgs, miscellaneousArgs, | |
| 181 annotationArgs, securityArgs, importsArgs, errors, eol, encoding, | |
| 182 backup) | |
| 183 @type list of (str, str, bool, str, str, bool, int, list of (int, int), | |
| 184 bool, str, dict, dict, dict, dict, list of str, str, str, bool) | |
| 185 @return tuple of statistics (dict) and list of results (tuple for each | |
| 186 found violation of style (lineno, position, text, ignored, fixed, | |
| 187 autofixing, fixedMsg)) | |
| 188 @rtype tuple of (dict, list of tuples of (int, int, str, bool, bool, bool, | |
| 189 str)) | |
| 190 """ | |
| 191 return __checkCodeStyle(filename, source, args) | |
| 192 | |
| 193 | |
| 194 def codeStyleBatchCheck(argumentsList, send, fx, cancelled, maxProcesses=0): | |
| 195 """ | |
| 196 Module function to check code style for a batch of files. | |
| 197 | |
| 198 @param argumentsList list of arguments tuples as given for codeStyleCheck | |
| 199 @type list | |
| 200 @param send reference to send function | |
| 201 @type func | |
| 202 @param fx registered service name | |
| 203 @type str | |
| 204 @param cancelled reference to function checking for a cancellation | |
| 205 @type func | |
| 206 @param maxProcesses number of processes to be used | |
| 207 @type int | |
| 208 """ | |
| 209 if maxProcesses == 0: | |
| 210 # determine based on CPU count | |
| 211 try: | |
| 212 NumberOfProcesses = multiprocessing.cpu_count() | |
| 213 if NumberOfProcesses >= 1: | |
| 214 NumberOfProcesses -= 1 | |
| 215 except NotImplementedError: | |
| 216 NumberOfProcesses = 1 | |
| 217 else: | |
| 218 NumberOfProcesses = maxProcesses | |
| 219 | |
| 220 # Create queues | |
| 221 taskQueue = multiprocessing.Queue() | |
| 222 doneQueue = multiprocessing.Queue() | |
| 223 | |
| 224 # Submit tasks (initially two time number of processes | |
| 225 initialTasks = 2 * NumberOfProcesses | |
| 226 for task in argumentsList[:initialTasks]: | |
| 227 taskQueue.put(task) | |
| 228 | |
| 229 # Start worker processes | |
| 230 workers = [ | |
| 231 multiprocessing.Process( | |
| 232 target=workerTask, args=(taskQueue, doneQueue) | |
| 233 ) for _ in range(NumberOfProcesses) | |
| 234 ] | |
| 235 for worker in workers: | |
| 236 worker.start() | |
| 237 | |
| 238 # Get and send results | |
| 239 endIndex = len(argumentsList) - initialTasks | |
| 240 for i in range(len(argumentsList)): | |
| 241 resultSent = False | |
| 242 wasCancelled = False | |
| 243 | |
| 244 while not resultSent: | |
| 245 try: | |
| 246 # get result (waiting max. 3 seconds and send it to frontend | |
| 247 filename, result = doneQueue.get(timeout=3) | |
| 248 send(fx, filename, result) | |
| 249 resultSent = True | |
| 250 except queue.Empty: | |
| 251 # ignore empty queue, just carry on | |
| 252 if cancelled(): | |
| 253 wasCancelled = True | |
| 254 break | |
| 255 | |
| 256 if wasCancelled or cancelled(): | |
| 257 # just exit the loop ignoring the results of queued tasks | |
| 258 break | |
| 259 | |
| 260 if i < endIndex: | |
| 261 taskQueue.put(argumentsList[i + initialTasks]) | |
| 262 | |
| 263 # Tell child processes to stop | |
| 264 for _ in range(NumberOfProcesses): | |
| 265 taskQueue.put('STOP') | |
| 266 | |
| 267 for worker in workers: | |
| 268 worker.join() | |
| 269 worker.close() | |
| 270 | |
| 271 | |
| 272 def workerTask(inputQueue, outputQueue): | |
| 273 """ | |
| 274 Module function acting as the parallel worker for the style check. | |
| 275 | |
| 276 @param inputQueue input queue (multiprocessing.Queue) | |
| 277 @param outputQueue output queue (multiprocessing.Queue) | |
| 278 """ | |
| 279 for filename, source, args in iter(inputQueue.get, 'STOP'): | |
| 280 result = __checkCodeStyle(filename, source, args) | |
| 281 outputQueue.put((filename, result)) | |
| 282 | |
| 283 | |
| 284 def __checkSyntax(filename, source): | |
| 285 """ | |
| 286 Private module function to perform a syntax check. | |
| 287 | |
| 288 @param filename source filename | |
| 289 @type str | |
| 290 @param source string containing the code to check | |
| 291 @type str | |
| 292 @return tuple containing the error dictionary with syntax error details, | |
| 293 a statistics dictionary and None or a tuple containing two None and | |
| 294 the generated AST tree | |
| 295 @rtype tuple of (dict, dict, None) or tuple of (None, None, ast.Module) | |
| 296 """ | |
| 297 src = "".join(source) | |
| 298 | |
| 299 try: | |
| 300 tree = ( | |
| 301 ast.parse(src, filename, 'exec', type_comments=True) | |
| 302 # need the 'type_comments' parameter to include type annotations | |
| 303 if sys.version_info >= (3, 8) else | |
| 304 ast.parse(src, filename, 'exec') | |
| 305 ) | |
| 306 return None, None, tree | |
| 307 except (SyntaxError, TypeError): | |
| 308 exc_type, exc = sys.exc_info()[:2] | |
| 309 if len(exc.args) > 1: | |
| 310 offset = exc.args[1] | |
| 311 if len(offset) > 2: | |
| 312 offset = offset[1:3] | |
| 313 else: | |
| 314 offset = (1, 0) | |
| 315 return ( | |
| 316 { | |
| 317 "file": filename, | |
| 318 "line": offset[0], | |
| 319 "offset": offset[1], | |
| 320 "code": "E901", | |
| 321 "args": [exc_type.__name__, exc.args[0]], | |
| 322 }, { | |
| 323 "E901": 1, | |
| 324 }, | |
| 325 None | |
| 326 ) | |
| 327 | |
| 328 | |
| 329 def __checkCodeStyle(filename, source, args): | |
| 330 """ | |
| 331 Private module function to perform the code style check and/or fix | |
| 332 found errors. | |
| 333 | |
| 334 @param filename source filename | |
| 335 @type str | |
| 336 @param source string containing the code to check | |
| 337 @type str | |
| 338 @param args arguments used by the codeStyleCheck function (list of | |
| 339 excludeMessages, includeMessages, repeatMessages, fixCodes, | |
| 340 noFixCodes, fixIssues, maxLineLength, maxDocLineLength, blankLines, | |
| 341 hangClosing, docType, codeComplexityArgs, miscellaneousArgs, | |
| 342 annotationArgs, securityArgs, importsArgs, errors, eol, encoding, | |
| 343 backup) | |
| 344 @type list of (str, str, bool, str, str, bool, int, list of (int, int), | |
| 345 bool, str, dict, dict, dict, dict, list of str, str, str, bool) | |
| 346 @return tuple of statistics data and list of result dictionaries with | |
| 347 keys: | |
| 348 <ul> | |
| 349 <li>file: file name</li> | |
| 350 <li>line: line_number</li> | |
| 351 <li>offset: offset within line</li> | |
| 352 <li>code: message code</li> | |
| 353 <li>args: list of arguments to format the message</li> | |
| 354 <li>ignored: flag indicating this issue was ignored</li> | |
| 355 <li>fixed: flag indicating this issue was fixed</li> | |
| 356 <li>autofixing: flag indicating that a fix can be done</li> | |
| 357 <li>fixcode: message code for the fix</li> | |
| 358 <li>fixargs: list of arguments to format the fix message</li> | |
| 359 </ul> | |
| 360 @rtype tuple of (dict, list of dict) | |
| 361 """ | |
| 362 (excludeMessages, includeMessages, repeatMessages, fixCodes, noFixCodes, | |
| 363 fixIssues, maxLineLength, maxDocLineLength, blankLines, hangClosing, | |
| 364 docType, codeComplexityArgs, miscellaneousArgs, annotationArgs, | |
| 365 securityArgs, importsArgs, errors, eol, encoding, backup) = args | |
| 366 | |
| 367 stats = {} | |
| 368 | |
| 369 if fixIssues: | |
| 370 from CodeStyleFixer import CodeStyleFixer | |
| 371 fixer = CodeStyleFixer( | |
| 372 filename, source, fixCodes, noFixCodes, | |
| 373 maxLineLength, blankLines, True, eol, backup) | |
| 374 # always fix in place | |
| 375 else: | |
| 376 fixer = None | |
| 377 | |
| 378 if not errors: | |
| 379 if includeMessages: | |
| 380 select = [s.strip() for s in | |
| 381 includeMessages.split(',') if s.strip()] | |
| 382 else: | |
| 383 select = [] | |
| 384 if excludeMessages: | |
| 385 ignore = [i.strip() for i in | |
| 386 excludeMessages.split(',') if i.strip()] | |
| 387 else: | |
| 388 ignore = [] | |
| 389 | |
| 390 syntaxError, syntaxStats, tree = __checkSyntax(filename, source) | |
| 391 | |
| 392 # perform the checks only, if syntax is ok and AST tree was generated | |
| 393 if tree: | |
| 394 # check coding style | |
| 395 pycodestyle.BLANK_LINES_CONFIG = { | |
| 396 # Top level class and function. | |
| 397 'top_level': blankLines[0], | |
| 398 # Methods and nested class and function. | |
| 399 'method': blankLines[1], | |
| 400 } | |
| 401 styleGuide = pycodestyle.StyleGuide( | |
| 402 reporter=CodeStyleCheckerReport, | |
| 403 repeat=repeatMessages, | |
| 404 select=select, | |
| 405 ignore=ignore, | |
| 406 max_line_length=maxLineLength, | |
| 407 max_doc_length=maxDocLineLength, | |
| 408 hang_closing=hangClosing, | |
| 409 ) | |
| 410 report = styleGuide.check_files([filename]) | |
| 411 stats.update(report.counters) | |
| 412 errors = report.errors | |
| 413 | |
| 414 # check documentation style | |
| 415 docStyleChecker = DocStyleChecker( | |
| 416 source, filename, select, ignore, [], repeatMessages, | |
| 417 maxLineLength=maxDocLineLength, docType=docType) | |
| 418 docStyleChecker.run() | |
| 419 stats.update(docStyleChecker.counters) | |
| 420 errors += docStyleChecker.errors | |
| 421 | |
| 422 # miscellaneous additional checks | |
| 423 miscellaneousChecker = MiscellaneousChecker( | |
| 424 source, filename, tree, select, ignore, [], repeatMessages, | |
| 425 miscellaneousArgs) | |
| 426 miscellaneousChecker.run() | |
| 427 stats.update(miscellaneousChecker.counters) | |
| 428 errors += miscellaneousChecker.errors | |
| 429 | |
| 430 # check code complexity | |
| 431 complexityChecker = ComplexityChecker( | |
| 432 source, filename, tree, select, ignore, codeComplexityArgs) | |
| 433 complexityChecker.run() | |
| 434 stats.update(complexityChecker.counters) | |
| 435 errors += complexityChecker.errors | |
| 436 | |
| 437 # check function annotations | |
| 438 if sys.version_info >= (3, 8, 0): | |
| 439 # annotations with type comments are supported from | |
| 440 # Python 3.8 on | |
| 441 from Annotations.AnnotationsChecker import AnnotationsChecker | |
| 442 annotationsChecker = AnnotationsChecker( | |
| 443 source, filename, tree, select, ignore, [], repeatMessages, | |
| 444 annotationArgs) | |
| 445 annotationsChecker.run() | |
| 446 stats.update(annotationsChecker.counters) | |
| 447 errors += annotationsChecker.errors | |
| 448 | |
| 449 # check for security issues | |
| 450 securityChecker = SecurityChecker( | |
| 451 source, filename, tree, select, ignore, [], repeatMessages, | |
| 452 securityArgs) | |
| 453 securityChecker.run() | |
| 454 stats.update(securityChecker.counters) | |
| 455 errors += securityChecker.errors | |
| 456 | |
| 457 # check for pathlib usage | |
| 458 pathlibChecker = PathlibChecker( | |
| 459 source, filename, tree, select, ignore, [], repeatMessages) | |
| 460 pathlibChecker.run() | |
| 461 stats.update(pathlibChecker.counters) | |
| 462 errors += pathlibChecker.errors | |
| 463 | |
| 464 # check for code simplifications | |
| 465 simplifyChecker = SimplifyChecker( | |
| 466 source, filename, tree, select, ignore, [], repeatMessages) | |
| 467 simplifyChecker.run() | |
| 468 stats.update(simplifyChecker.counters) | |
| 469 errors += simplifyChecker.errors | |
| 470 | |
| 471 # check import statements | |
| 472 importsChecker = ImportsChecker( | |
| 473 source, filename, tree, select, ignore, [], repeatMessages, | |
| 474 importsArgs) | |
| 475 importsChecker.run() | |
| 476 stats.update(importsChecker.counters) | |
| 477 errors += importsChecker.errors | |
| 478 | |
| 479 elif syntaxError: | |
| 480 errors = [syntaxError] | |
| 481 stats.update(syntaxStats) | |
| 482 | |
| 483 errorsDict = {} | |
| 484 for error in errors: | |
| 485 if error["line"] > len(source): | |
| 486 error["line"] = len(source) | |
| 487 # inverse processing of messages and fixes | |
| 488 errorLine = errorsDict.setdefault(error["line"], []) | |
| 489 errorLine.append((error["offset"], error)) | |
| 490 deferredFixes = {} | |
| 491 results = [] | |
| 492 for lineno, errorsList in errorsDict.items(): | |
| 493 errorsList.sort(key=lambda x: x[0], reverse=True) | |
| 494 for _, error in errorsList: | |
| 495 error.update({ | |
| 496 "ignored": False, | |
| 497 "fixed": False, | |
| 498 "autofixing": False, | |
| 499 "fixcode": "", | |
| 500 "fixargs": [], | |
| 501 "securityOk": False, | |
| 502 }) | |
| 503 | |
| 504 if source: | |
| 505 code = error["code"] | |
| 506 lineFlags = extractLineFlags(source[lineno - 1].strip()) | |
| 507 with contextlib.suppress(IndexError): | |
| 508 lineFlags += extractLineFlags(source[lineno].strip(), | |
| 509 flagsLine=True) | |
| 510 | |
| 511 if securityOk(code, lineFlags): | |
| 512 error["securityOk"] = True | |
| 513 | |
| 514 if ignoreCode(code, lineFlags): | |
| 515 error["ignored"] = True | |
| 516 else: | |
| 517 if fixer: | |
| 518 res, fixcode, fixargs, id_ = fixer.fixIssue( | |
| 519 lineno, error["offset"], code) | |
| 520 if res == -1: | |
| 521 deferredFixes[id_] = error | |
| 522 else: | |
| 523 error.update({ | |
| 524 "fixed": res == 1, | |
| 525 "autofixing": True, | |
| 526 "fixcode": fixcode, | |
| 527 "fixargs": fixargs, | |
| 528 }) | |
| 529 | |
| 530 results.append(error) | |
| 531 | |
| 532 if fixer: | |
| 533 deferredResults = fixer.finalize() | |
| 534 for id_ in deferredResults: | |
| 535 fixed, fixcode, fixargs = deferredResults[id_] | |
| 536 error = deferredFixes[id_] | |
| 537 error.update({ | |
| 538 "ignored": False, | |
| 539 "fixed": fixed == 1, | |
| 540 "autofixing": True, | |
| 541 "fixcode": fixcode, | |
| 542 "fixargs": fixargs, | |
| 543 }) | |
| 544 | |
| 545 saveError = fixer.saveFile(encoding) | |
| 546 if saveError: | |
| 547 for error in results: | |
| 548 error.update({ | |
| 549 "fixcode": saveError[0], | |
| 550 "fixargs": saveError[1], | |
| 551 }) | |
| 552 | |
| 553 return stats, results |
