Mercurial Repositories > eric / file comparison
comparison: src/eric7/Plugins/CheckerPlugins/SyntaxChecker/pyCheckSyntax.py
src/eric7/Plugins/CheckerPlugins/SyntaxChecker/pyCheckSyntax.py
- branch
- eric7
- changeset 9508
- 5a02bdb1dcba
- parent 9507
- 1f39839655ea
- child 9591
- 3c56c81a70be
equal
deleted
inserted
replaced
| 9507:1f39839655ea | 9508:5a02bdb1dcba |
|---|---|
| 1 # -*- coding: utf-8 -*- | |
| 2 | |
| 3 # Copyright (c) 2011 - 2022 Detlev Offenbach <detlev@die-offenbachs.de> | |
| 4 # | |
| 5 | |
| 6 """ | |
| 7 Module implementing the syntax check for Python 3. | |
| 8 """ | |
| 9 | |
| 10 import ast | |
| 11 import builtins | |
| 12 import contextlib | |
| 13 import multiprocessing | |
| 14 import queue | |
| 15 import re | |
| 16 import traceback | |
| 17 | |
| 18 with contextlib.suppress(ImportError): | |
| 19 from pyflakes.checker import Checker | |
| 20 from pyflakes.messages import ImportStarUsage, ImportStarUsed | |
| 21 | |
| 22 VcsConflictMarkerRegExpList = ( | |
| 23 re.compile( | |
| 24 r"""^<<<<<<< .*?\|\|\|\|\|\|\| .*?=======.*?>>>>>>> .*?$""", | |
| 25 re.MULTILINE | re.DOTALL, | |
| 26 ), | |
| 27 re.compile(r"""^<<<<<<< .*?=======.*?>>>>>>> .*?$""", re.MULTILINE | re.DOTALL), | |
| 28 ) | |
| 29 | |
| 30 | |
| 31 def initService(): | |
| 32 """ | |
| 33 Initialize the service and return the entry point. | |
| 34 | |
| 35 @return the entry point for the background client | |
| 36 @rtype function | |
| 37 """ | |
| 38 return pySyntaxAndPyflakesCheck | |
| 39 | |
| 40 | |
| 41 def initBatchService(): | |
| 42 """ | |
| 43 Initialize the batch service and return the entry point. | |
| 44 | |
| 45 @return the entry point for the background client | |
| 46 @rtype function | |
| 47 """ | |
| 48 return pySyntaxAndPyflakesBatchCheck | |
| 49 | |
| 50 | |
| 51 def extractLineFlags(line, startComment="#", endComment="", flagsLine=False): | |
| 52 """ | |
| 53 Function to extract flags starting and ending with '__' from a line | |
| 54 comment. | |
| 55 | |
| 56 @param line line to extract flags from | |
| 57 @type str | |
| 58 @param startComment string identifying the start of the comment | |
| 59 @type str | |
| 60 @param endComment string identifying the end of a comment | |
| 61 @type str | |
| 62 @param flagsLine flag indicating to check for a flags only line | |
| 63 @type bool | |
| 64 @return list containing the extracted flags | |
| 65 @rtype list of str | |
| 66 """ | |
| 67 flags = [] | |
| 68 | |
| 69 if not flagsLine or (flagsLine and line.strip().startswith(startComment)): | |
| 70 pos = line.rfind(startComment) | |
| 71 if pos >= 0: | |
| 72 comment = line[pos + len(startComment) :].strip() | |
| 73 if endComment: | |
| 74 endPos = line.rfind(endComment) | |
| 75 if endPos >= 0: | |
| 76 comment = comment[:endPos] | |
| 77 flags = [ | |
| 78 f.strip() | |
| 79 for f in comment.split() | |
| 80 if (f.startswith("__") and f.endswith("__")) | |
| 81 ] | |
| 82 flags += [ | |
| 83 f.strip().lower() for f in comment.split() if f in ("noqa", "NOQA") | |
| 84 ] | |
| 85 return flags | |
| 86 | |
| 87 | |
| 88 def pySyntaxAndPyflakesCheck( | |
| 89 filename, codestring, checkFlakes=True, ignoreStarImportWarnings=False | |
| 90 ): | |
| 91 """ | |
| 92 Function to compile one Python source file to Python bytecode | |
| 93 and to perform a pyflakes check. | |
| 94 | |
| 95 @param filename source filename | |
| 96 @type str | |
| 97 @param codestring string containing the code to compile | |
| 98 @type str | |
| 99 @param checkFlakes flag indicating to do a pyflakes check | |
| 100 @type bool | |
| 101 @param ignoreStarImportWarnings flag indicating to ignore 'star import' warnings | |
| 102 @type bool | |
| 103 @return dictionary with the keys 'error' and 'warnings' which | |
| 104 hold a list containing details about the error/warnings | |
| 105 (file name, line number, column, codestring (only at syntax | |
| 106 errors), the message, a list with arguments for the message) | |
| 107 @rtype dict | |
| 108 """ | |
| 109 return __pySyntaxAndPyflakesCheck( | |
| 110 filename, codestring, checkFlakes, ignoreStarImportWarnings | |
| 111 ) | |
| 112 | |
| 113 | |
| 114 def pySyntaxAndPyflakesBatchCheck(argumentsList, send, fx, cancelled, maxProcesses=0): | |
| 115 """ | |
| 116 Module function to check syntax for a batch of files. | |
| 117 | |
| 118 @param argumentsList list of arguments tuples as given for pySyntaxAndPyflakesCheck | |
| 119 @type list | |
| 120 @param send reference to send function | |
| 121 @type func | |
| 122 @param fx registered service name | |
| 123 @type str | |
| 124 @param cancelled reference to function checking for a cancellation | |
| 125 @type func | |
| 126 @param maxProcesses number of processes to be used | |
| 127 @type int | |
| 128 """ | |
| 129 if maxProcesses == 0: | |
| 130 # determine based on CPU count | |
| 131 try: | |
| 132 NumberOfProcesses = multiprocessing.cpu_count() | |
| 133 if NumberOfProcesses >= 1: | |
| 134 NumberOfProcesses -= 1 | |
| 135 except NotImplementedError: | |
| 136 NumberOfProcesses = 1 | |
| 137 else: | |
| 138 NumberOfProcesses = maxProcesses | |
| 139 | |
| 140 # Create queues | |
| 141 taskQueue = multiprocessing.Queue() | |
| 142 doneQueue = multiprocessing.Queue() | |
| 143 | |
| 144 # Submit tasks (initially two times the number of processes) | |
| 145 tasks = len(argumentsList) | |
| 146 initialTasks = min(2 * NumberOfProcesses, tasks) | |
| 147 for _ in range(initialTasks): | |
| 148 taskQueue.put(argumentsList.pop(0)) | |
| 149 | |
| 150 # Start worker processes | |
| 151 workers = [ | |
| 152 multiprocessing.Process(target=workerTask, args=(taskQueue, doneQueue)) | |
| 153 for _ in range(NumberOfProcesses) | |
| 154 ] | |
| 155 for worker in workers: | |
| 156 worker.start() | |
| 157 | |
| 158 # Get and send results | |
| 159 for _ in range(tasks): | |
| 160 resultSent = False | |
| 161 wasCancelled = False | |
| 162 | |
| 163 while not resultSent: | |
| 164 try: | |
| 165 # get result (waiting max. 3 seconds and send it to frontend | |
| 166 filename, result = doneQueue.get() | |
| 167 send(fx, filename, result) | |
| 168 resultSent = True | |
| 169 except queue.Empty: | |
| 170 # ignore empty queue, just carry on | |
| 171 if cancelled(): | |
| 172 wasCancelled = True | |
| 173 break | |
| 174 | |
| 175 if wasCancelled or cancelled(): | |
| 176 # just exit the loop ignoring the results of queued tasks | |
| 177 break | |
| 178 | |
| 179 if argumentsList: | |
| 180 taskQueue.put(argumentsList.pop(0)) | |
| 181 | |
| 182 # Tell child processes to stop | |
| 183 for _ in range(NumberOfProcesses): | |
| 184 taskQueue.put("STOP") | |
| 185 | |
| 186 for worker in workers: | |
| 187 worker.join() | |
| 188 worker.close() | |
| 189 | |
| 190 taskQueue.close() | |
| 191 doneQueue.close() | |
| 192 | |
| 193 | |
| 194 def workerTask(inputQueue, outputQueue): | |
| 195 """ | |
| 196 Module function acting as the parallel worker for the syntax check. | |
| 197 | |
| 198 @param inputQueue input queue | |
| 199 @type multiprocessing.Queue | |
| 200 @param outputQueue output queue | |
| 201 @type multiprocessing.Queue | |
| 202 """ | |
| 203 for filename, args in iter(inputQueue.get, "STOP"): | |
| 204 source, checkFlakes, ignoreStarImportWarnings = args | |
| 205 result = __pySyntaxAndPyflakesCheck( | |
| 206 filename, source, checkFlakes, ignoreStarImportWarnings | |
| 207 ) | |
| 208 outputQueue.put((filename, result)) | |
| 209 | |
| 210 | |
| 211 def __pySyntaxAndPyflakesCheck( | |
| 212 filename, codestring, checkFlakes=True, ignoreStarImportWarnings=False | |
| 213 ): | |
| 214 """ | |
| 215 Function to compile one Python source file to Python bytecode | |
| 216 and to perform a pyflakes check. | |
| 217 | |
| 218 @param filename source filename | |
| 219 @type str | |
| 220 @param codestring string containing the code to compile | |
| 221 @type str | |
| 222 @param checkFlakes flag indicating to do a pyflakes check | |
| 223 @type bool | |
| 224 @param ignoreStarImportWarnings flag indicating to | |
| 225 ignore 'star import' warnings | |
| 226 @type bool | |
| 227 @return dictionary with the keys 'error' and 'warnings' which | |
| 228 hold a list containing details about the error/ warnings | |
| 229 (file name, line number, column, codestring (only at syntax | |
| 230 errors), the message, a list with arguments for the message) | |
| 231 @rtype dict | |
| 232 """ | |
| 233 try: | |
| 234 # Check for VCS conflict markers | |
| 235 for conflictMarkerRe in VcsConflictMarkerRegExpList: | |
| 236 conflict = conflictMarkerRe.search(codestring) | |
| 237 if conflict is not None: | |
| 238 start, i = conflict.span() | |
| 239 lineindex = 1 + codestring.count("\n", 0, start) | |
| 240 return [ | |
| 241 {"error": (filename, lineindex, 0, "", "VCS conflict marker found")} | |
| 242 ] | |
| 243 | |
| 244 if filename.endswith(".ptl"): | |
| 245 try: | |
| 246 import quixote.ptl_compile # __IGNORE_WARNING_I10__ | |
| 247 except ImportError: | |
| 248 return [{"error": (filename, 0, 0, "", "Quixote plugin not found.")}] | |
| 249 template = quixote.ptl_compile.Template(codestring, filename) | |
| 250 template.compile() | |
| 251 else: | |
| 252 module = builtins.compile(codestring, filename, "exec", ast.PyCF_ONLY_AST) | |
| 253 except SyntaxError as detail: | |
| 254 index = 0 | |
| 255 code = "" | |
| 256 error = "" | |
| 257 lines = traceback.format_exception_only(SyntaxError, detail) | |
| 258 match = re.match( | |
| 259 r'\s*File "(.+)", line (\d+)', lines[0].replace("<string>", filename) | |
| 260 ) | |
| 261 if match is not None: | |
| 262 fn, line = match.group(1, 2) | |
| 263 if lines[1].startswith("SyntaxError:"): | |
| 264 error = re.match("SyntaxError: (.+)", lines[1]).group(1) | |
| 265 else: | |
| 266 code = re.match("(.+)", lines[1]).group(1) | |
| 267 for seLine in lines[2:]: | |
| 268 if seLine.startswith("SyntaxError:"): | |
| 269 error = re.match("SyntaxError: (.+)", seLine).group(1) | |
| 270 elif seLine.rstrip().endswith("^"): | |
| 271 index = len(seLine.rstrip()) - 4 | |
| 272 else: | |
| 273 fn = detail.filename | |
| 274 line = detail.lineno or 1 | |
| 275 error = detail.msg | |
| 276 return [{"error": (fn, int(line), index, code.strip(), error)}] | |
| 277 except ValueError as detail: | |
| 278 try: | |
| 279 fn = detail.filename | |
| 280 line = detail.lineno | |
| 281 error = detail.msg | |
| 282 except AttributeError: | |
| 283 fn = filename | |
| 284 line = 1 | |
| 285 error = str(detail) | |
| 286 return [{"error": (fn, line, 0, "", error)}] | |
| 287 except Exception as detail: | |
| 288 with contextlib.suppress(AttributeError): | |
| 289 fn = detail.filename | |
| 290 line = detail.lineno | |
| 291 error = detail.msg | |
| 292 return [{"error": (fn, line, 0, "", error)}] | |
| 293 | |
| 294 # pyflakes | |
| 295 if not checkFlakes: | |
| 296 return [{}] | |
| 297 | |
| 298 results = [] | |
| 299 lines = codestring.splitlines() | |
| 300 try: | |
| 301 warnings = Checker(module, filename, withDoctest=True) | |
| 302 warnings.messages.sort(key=lambda a: a.lineno) | |
| 303 for warning in warnings.messages: | |
| 304 if ignoreStarImportWarnings and isinstance( | |
| 305 warning, (ImportStarUsed, ImportStarUsage) | |
| 306 ): | |
| 307 continue | |
| 308 | |
| 309 _fn, lineno, col, message, msg_args = warning.getMessageData() | |
| 310 lineFlags = extractLineFlags(lines[lineno - 1].strip()) | |
| 311 with contextlib.suppress(IndexError): | |
| 312 lineFlags += extractLineFlags(lines[lineno].strip(), flagsLine=True) | |
| 313 if "__IGNORE_WARNING__" not in lineFlags and "noqa" not in lineFlags: | |
| 314 results.append((_fn, lineno, col, "", message, msg_args)) | |
| 315 except SyntaxError as err: | |
| 316 msg = err.text.strip() if err.text.strip() else err.msg | |
| 317 results.append((filename, err.lineno, 0, "FLAKES_ERROR", msg, [])) | |
| 318 | |
| 319 return [{"warnings": results}] |
