Mercurial Repositories > eric / file comparison
comparison: src/eric7/Plugins/CheckerPlugins/SyntaxChecker/SyntaxCheck.py
src/eric7/Plugins/CheckerPlugins/SyntaxChecker/SyntaxCheck.py
- branch
- eric7
- changeset 9209
- b99e7fd55fd3
- parent 8881
- 54e42bc2437a
- child 9221
- bf71ee032bb4
equal
deleted
inserted
replaced
| 9208:3fc8dfeb6ebe | 9209:b99e7fd55fd3 |
|---|---|
| 1 # -*- coding: utf-8 -*- | |
| 2 | |
| 3 # Copyright (c) 2011 - 2022 Detlev Offenbach <detlev@die-offenbachs.de> | |
| 4 # | |
| 5 | |
| 6 """ | |
| 7 Module implementing the syntax check for Python 3. | |
| 8 """ | |
| 9 | |
| 10 import queue | |
| 11 import ast | |
| 12 import re | |
| 13 import traceback | |
| 14 import multiprocessing | |
| 15 import contextlib | |
| 16 | |
| 17 with contextlib.suppress(ImportError): | |
| 18 from pyflakes.checker import Checker | |
| 19 from pyflakes.messages import ImportStarUsed, ImportStarUsage | |
| 20 | |
| 21 VcsConflictMarkerRegExpList = ( | |
| 22 re.compile( | |
| 23 r"""^<<<<<<< .*?\|\|\|\|\|\|\| .*?=======.*?>>>>>>> .*?$""", | |
| 24 re.MULTILINE | re.DOTALL | |
| 25 ), | |
| 26 re.compile( | |
| 27 r"""^<<<<<<< .*?=======.*?>>>>>>> .*?$""", | |
| 28 re.MULTILINE | re.DOTALL | |
| 29 ), | |
| 30 ) | |
| 31 | |
| 32 | |
| 33 def initService(): | |
| 34 """ | |
| 35 Initialize the service and return the entry point. | |
| 36 | |
| 37 @return the entry point for the background client (function) | |
| 38 """ | |
| 39 return syntaxAndPyflakesCheck | |
| 40 | |
| 41 | |
| 42 def initBatchService(): | |
| 43 """ | |
| 44 Initialize the batch service and return the entry point. | |
| 45 | |
| 46 @return the entry point for the background client (function) | |
| 47 """ | |
| 48 return syntaxAndPyflakesBatchCheck | |
| 49 | |
| 50 | |
| 51 def normalizeCode(codestring): | |
| 52 """ | |
| 53 Function to normalize the given code. | |
| 54 | |
| 55 @param codestring code to be normalized (string) | |
| 56 @return normalized code (string) | |
| 57 """ | |
| 58 codestring = codestring.replace("\r\n", "\n").replace("\r", "\n") | |
| 59 | |
| 60 if codestring and codestring[-1] != '\n': | |
| 61 codestring += '\n' | |
| 62 | |
| 63 return codestring | |
| 64 | |
| 65 | |
| 66 def extractLineFlags(line, startComment="#", endComment="", flagsLine=False): | |
| 67 """ | |
| 68 Function to extract flags starting and ending with '__' from a line | |
| 69 comment. | |
| 70 | |
| 71 @param line line to extract flags from (string) | |
| 72 @param startComment string identifying the start of the comment (string) | |
| 73 @param endComment string identifying the end of a comment (string) | |
| 74 @param flagsLine flag indicating to check for a flags only line (bool) | |
| 75 @return list containing the extracted flags (list of strings) | |
| 76 """ | |
| 77 flags = [] | |
| 78 | |
| 79 if not flagsLine or ( | |
| 80 flagsLine and line.strip().startswith(startComment)): | |
| 81 pos = line.rfind(startComment) | |
| 82 if pos >= 0: | |
| 83 comment = line[pos + len(startComment):].strip() | |
| 84 if endComment: | |
| 85 endPos = line.rfind(endComment) | |
| 86 if endPos >= 0: | |
| 87 comment = comment[:endPos] | |
| 88 flags = [f.strip() for f in comment.split() | |
| 89 if (f.startswith("__") and f.endswith("__"))] | |
| 90 flags += [f.strip().lower() for f in comment.split() | |
| 91 if f in ("noqa", "NOQA")] | |
| 92 return flags | |
| 93 | |
| 94 | |
| 95 def syntaxAndPyflakesCheck(filename, codestring, checkFlakes=True, | |
| 96 ignoreStarImportWarnings=False): | |
| 97 """ | |
| 98 Function to compile one Python source file to Python bytecode | |
| 99 and to perform a pyflakes check. | |
| 100 | |
| 101 @param filename source filename (string) | |
| 102 @param codestring string containing the code to compile (string) | |
| 103 @param checkFlakes flag indicating to do a pyflakes check (boolean) | |
| 104 @param ignoreStarImportWarnings flag indicating to | |
| 105 ignore 'star import' warnings (boolean) | |
| 106 @return dictionary with the keys 'error' and 'warnings' which | |
| 107 hold a list containing details about the error/ warnings | |
| 108 (file name, line number, column, codestring (only at syntax | |
| 109 errors), the message, a list with arguments for the message) | |
| 110 """ | |
| 111 return __syntaxAndPyflakesCheck(filename, codestring, checkFlakes, | |
| 112 ignoreStarImportWarnings) | |
| 113 | |
| 114 | |
| 115 def syntaxAndPyflakesBatchCheck(argumentsList, send, fx, cancelled, | |
| 116 maxProcesses=0): | |
| 117 """ | |
| 118 Module function to check syntax for a batch of files. | |
| 119 | |
| 120 @param argumentsList list of arguments tuples as given for | |
| 121 syntaxAndPyflakesCheck | |
| 122 @type list | |
| 123 @param send reference to send function | |
| 124 @type func | |
| 125 @param fx registered service name | |
| 126 @type str | |
| 127 @param cancelled reference to function checking for a cancellation | |
| 128 @type func | |
| 129 @param maxProcesses number of processes to be used | |
| 130 @type int | |
| 131 """ | |
| 132 if maxProcesses == 0: | |
| 133 # determine based on CPU count | |
| 134 try: | |
| 135 NumberOfProcesses = multiprocessing.cpu_count() | |
| 136 if NumberOfProcesses >= 1: | |
| 137 NumberOfProcesses -= 1 | |
| 138 except NotImplementedError: | |
| 139 NumberOfProcesses = 1 | |
| 140 else: | |
| 141 NumberOfProcesses = maxProcesses | |
| 142 | |
| 143 # Create queues | |
| 144 taskQueue = multiprocessing.Queue() | |
| 145 doneQueue = multiprocessing.Queue() | |
| 146 | |
| 147 # Submit tasks (initially two time number of processes | |
| 148 initialTasks = 2 * NumberOfProcesses | |
| 149 for task in argumentsList[:initialTasks]: | |
| 150 taskQueue.put(task) | |
| 151 | |
| 152 # Start worker processes | |
| 153 workers = [ | |
| 154 multiprocessing.Process( | |
| 155 target=workerTask, args=(taskQueue, doneQueue) | |
| 156 ) for _ in range(NumberOfProcesses) | |
| 157 ] | |
| 158 for worker in workers: | |
| 159 worker.start() | |
| 160 | |
| 161 # Get and send results | |
| 162 endIndex = len(argumentsList) - initialTasks | |
| 163 for i in range(len(argumentsList)): | |
| 164 resultSent = False | |
| 165 wasCancelled = False | |
| 166 | |
| 167 while not resultSent: | |
| 168 try: | |
| 169 # get result (waiting max. 3 seconds and send it to frontend | |
| 170 filename, result = doneQueue.get() | |
| 171 send(fx, filename, result) | |
| 172 resultSent = True | |
| 173 except queue.Empty: | |
| 174 # ignore empty queue, just carry on | |
| 175 if cancelled(): | |
| 176 wasCancelled = True | |
| 177 break | |
| 178 | |
| 179 if wasCancelled or cancelled(): | |
| 180 # just exit the loop ignoring the results of queued tasks | |
| 181 break | |
| 182 | |
| 183 if i < endIndex: | |
| 184 taskQueue.put(argumentsList[i + initialTasks]) | |
| 185 | |
| 186 # Tell child processes to stop | |
| 187 for _ in range(NumberOfProcesses): | |
| 188 taskQueue.put('STOP') | |
| 189 | |
| 190 for worker in workers: | |
| 191 worker.join() | |
| 192 worker.close() | |
| 193 | |
| 194 | |
| 195 def workerTask(inputQueue, outputQueue): | |
| 196 """ | |
| 197 Module function acting as the parallel worker for the syntax check. | |
| 198 | |
| 199 @param inputQueue input queue (multiprocessing.Queue) | |
| 200 @param outputQueue output queue (multiprocessing.Queue) | |
| 201 """ | |
| 202 for filename, args in iter(inputQueue.get, 'STOP'): | |
| 203 source, checkFlakes, ignoreStarImportWarnings = args | |
| 204 result = __syntaxAndPyflakesCheck(filename, source, checkFlakes, | |
| 205 ignoreStarImportWarnings) | |
| 206 outputQueue.put((filename, result)) | |
| 207 | |
| 208 | |
| 209 def __syntaxAndPyflakesCheck(filename, codestring, checkFlakes=True, | |
| 210 ignoreStarImportWarnings=False): | |
| 211 """ | |
| 212 Function to compile one Python source file to Python bytecode | |
| 213 and to perform a pyflakes check. | |
| 214 | |
| 215 @param filename source filename | |
| 216 @type str | |
| 217 @param codestring string containing the code to compile | |
| 218 @type str | |
| 219 @param checkFlakes flag indicating to do a pyflakes check | |
| 220 @type bool | |
| 221 @param ignoreStarImportWarnings flag indicating to | |
| 222 ignore 'star import' warnings | |
| 223 @type bool | |
| 224 @return dictionary with the keys 'error' and 'warnings' which | |
| 225 hold a list containing details about the error/ warnings | |
| 226 (file name, line number, column, codestring (only at syntax | |
| 227 errors), the message, a list with arguments for the message) | |
| 228 @rtype dict | |
| 229 """ | |
| 230 import builtins | |
| 231 | |
| 232 try: | |
| 233 codestring = normalizeCode(codestring) | |
| 234 | |
| 235 # Check for VCS conflict markers | |
| 236 for conflictMarkerRe in VcsConflictMarkerRegExpList: | |
| 237 conflict = conflictMarkerRe.search(codestring) | |
| 238 if conflict is not None: | |
| 239 start, i = conflict.span() | |
| 240 lineindex = 1 + codestring.count("\n", 0, start) | |
| 241 return [{'error': | |
| 242 (filename, lineindex, 0, "", | |
| 243 "VCS conflict marker found") | |
| 244 }] | |
| 245 | |
| 246 if filename.endswith('.ptl'): | |
| 247 try: | |
| 248 import quixote.ptl_compile | |
| 249 except ImportError: | |
| 250 return [{'error': (filename, 0, 0, '', | |
| 251 'Quixote plugin not found.')}] | |
| 252 template = quixote.ptl_compile.Template(codestring, filename) | |
| 253 template.compile() | |
| 254 else: | |
| 255 module = builtins.compile( | |
| 256 codestring, filename, 'exec', ast.PyCF_ONLY_AST) | |
| 257 except SyntaxError as detail: | |
| 258 index = 0 | |
| 259 code = "" | |
| 260 error = "" | |
| 261 lines = traceback.format_exception_only(SyntaxError, detail) | |
| 262 match = re.match(r'\s*File "(.+)", line (\d+)', | |
| 263 lines[0].replace('<string>', filename)) | |
| 264 if match is not None: | |
| 265 fn, line = match.group(1, 2) | |
| 266 if lines[1].startswith('SyntaxError:'): | |
| 267 error = re.match('SyntaxError: (.+)', lines[1]).group(1) | |
| 268 else: | |
| 269 code = re.match('(.+)', lines[1]).group(1) | |
| 270 for seLine in lines[2:]: | |
| 271 if seLine.startswith('SyntaxError:'): | |
| 272 error = re.match('SyntaxError: (.+)', seLine).group(1) | |
| 273 elif seLine.rstrip().endswith('^'): | |
| 274 index = len(seLine.rstrip()) - 4 | |
| 275 else: | |
| 276 fn = detail.filename | |
| 277 line = detail.lineno or 1 | |
| 278 error = detail.msg | |
| 279 return [{'error': (fn, int(line), index, code.strip(), error)}] | |
| 280 except ValueError as detail: | |
| 281 try: | |
| 282 fn = detail.filename | |
| 283 line = detail.lineno | |
| 284 error = detail.msg | |
| 285 except AttributeError: | |
| 286 fn = filename | |
| 287 line = 1 | |
| 288 error = str(detail) | |
| 289 return [{'error': (fn, line, 0, "", error)}] | |
| 290 except Exception as detail: | |
| 291 with contextlib.suppress(Exception): | |
| 292 fn = detail.filename | |
| 293 line = detail.lineno | |
| 294 error = detail.msg | |
| 295 return [{'error': (fn, line, 0, "", error)}] | |
| 296 | |
| 297 # pyflakes | |
| 298 if not checkFlakes: | |
| 299 return [{}] | |
| 300 | |
| 301 results = [] | |
| 302 lines = codestring.splitlines() | |
| 303 try: | |
| 304 warnings = Checker(module, filename, withDoctest=True) | |
| 305 warnings.messages.sort(key=lambda a: a.lineno) | |
| 306 for warning in warnings.messages: | |
| 307 if ( | |
| 308 ignoreStarImportWarnings and | |
| 309 isinstance(warning, (ImportStarUsed, ImportStarUsage)) | |
| 310 ): | |
| 311 continue | |
| 312 | |
| 313 _fn, lineno, col, message, msg_args = warning.getMessageData() | |
| 314 lineFlags = extractLineFlags(lines[lineno - 1].strip()) | |
| 315 with contextlib.suppress(IndexError): | |
| 316 lineFlags += extractLineFlags(lines[lineno].strip(), | |
| 317 flagsLine=True) | |
| 318 if ( | |
| 319 "__IGNORE_WARNING__" not in lineFlags and | |
| 320 "noqa" not in lineFlags | |
| 321 ): | |
| 322 results.append((_fn, lineno, col, "", message, msg_args)) | |
| 323 except SyntaxError as err: | |
| 324 msg = err.text.strip() if err.text.strip() else err.msg | |
| 325 results.append((filename, err.lineno, 0, "FLAKES_ERROR", msg, [])) | |
| 326 | |
| 327 return [{'warnings': results}] |
