Mercurial Repositories > eric / file comparison
comparison: eric7/Plugins/CheckerPlugins/SyntaxChecker/SyntaxCheck.py
eric7/Plugins/CheckerPlugins/SyntaxChecker/SyntaxCheck.py
- branch
- eric7
- changeset 8312
- 800c432b34c8
- parent 8243
- cc717c2ae956
- child 8650
- 100726f55a9a
equal
deleted
inserted
replaced
| 8311:4e8b98454baa | 8312:800c432b34c8 |
|---|---|
| 1 # -*- coding: utf-8 -*- | |
| 2 | |
| 3 # Copyright (c) 2011 - 2021 Detlev Offenbach <detlev@die-offenbachs.de> | |
| 4 # | |
| 5 | |
| 6 """ | |
| 7 Module implementing the syntax check for Python 3. | |
| 8 """ | |
| 9 | |
| 10 import queue | |
| 11 import ast | |
| 12 import re | |
| 13 import traceback | |
| 14 import multiprocessing | |
| 15 import contextlib | |
| 16 | |
| 17 with contextlib.suppress(ImportError): | |
| 18 from pyflakes.checker import Checker | |
| 19 from pyflakes.messages import ImportStarUsed, ImportStarUsage | |
| 20 | |
| 21 VcsConflictMarkerRegExpList = ( | |
| 22 re.compile( | |
| 23 r"""^<<<<<<< .*?\|\|\|\|\|\|\| .*?=======.*?>>>>>>> .*?$""", | |
| 24 re.MULTILINE | re.DOTALL | |
| 25 ), | |
| 26 re.compile( | |
| 27 r"""^<<<<<<< .*?=======.*?>>>>>>> .*?$""", | |
| 28 re.MULTILINE | re.DOTALL | |
| 29 ), | |
| 30 ) | |
| 31 | |
| 32 | |
| 33 def initService(): | |
| 34 """ | |
| 35 Initialize the service and return the entry point. | |
| 36 | |
| 37 @return the entry point for the background client (function) | |
| 38 """ | |
| 39 return syntaxAndPyflakesCheck | |
| 40 | |
| 41 | |
| 42 def initBatchService(): | |
| 43 """ | |
| 44 Initialize the batch service and return the entry point. | |
| 45 | |
| 46 @return the entry point for the background client (function) | |
| 47 """ | |
| 48 return syntaxAndPyflakesBatchCheck | |
| 49 | |
| 50 | |
| 51 def normalizeCode(codestring): | |
| 52 """ | |
| 53 Function to normalize the given code. | |
| 54 | |
| 55 @param codestring code to be normalized (string) | |
| 56 @return normalized code (string) | |
| 57 """ | |
| 58 codestring = codestring.replace("\r\n", "\n").replace("\r", "\n") | |
| 59 | |
| 60 if codestring and codestring[-1] != '\n': | |
| 61 codestring += '\n' | |
| 62 | |
| 63 return codestring | |
| 64 | |
| 65 | |
| 66 def extractLineFlags(line, startComment="#", endComment="", flagsLine=False): | |
| 67 """ | |
| 68 Function to extract flags starting and ending with '__' from a line | |
| 69 comment. | |
| 70 | |
| 71 @param line line to extract flags from (string) | |
| 72 @param startComment string identifying the start of the comment (string) | |
| 73 @param endComment string identifying the end of a comment (string) | |
| 74 @param flagsLine flag indicating to check for a flags only line (bool) | |
| 75 @return list containing the extracted flags (list of strings) | |
| 76 """ | |
| 77 flags = [] | |
| 78 | |
| 79 if not flagsLine or ( | |
| 80 flagsLine and line.strip().startswith(startComment)): | |
| 81 pos = line.rfind(startComment) | |
| 82 if pos >= 0: | |
| 83 comment = line[pos + len(startComment):].strip() | |
| 84 if endComment: | |
| 85 endPos = line.rfind(endComment) | |
| 86 if endPos >= 0: | |
| 87 comment = comment[:endPos] | |
| 88 flags = [f.strip() for f in comment.split() | |
| 89 if (f.startswith("__") and f.endswith("__"))] | |
| 90 flags += [f.strip().lower() for f in comment.split() | |
| 91 if f in ("noqa", "NOQA")] | |
| 92 return flags | |
| 93 | |
| 94 | |
| 95 def syntaxAndPyflakesCheck(filename, codestring, checkFlakes=True, | |
| 96 ignoreStarImportWarnings=False): | |
| 97 """ | |
| 98 Function to compile one Python source file to Python bytecode | |
| 99 and to perform a pyflakes check. | |
| 100 | |
| 101 @param filename source filename (string) | |
| 102 @param codestring string containing the code to compile (string) | |
| 103 @param checkFlakes flag indicating to do a pyflakes check (boolean) | |
| 104 @param ignoreStarImportWarnings flag indicating to | |
| 105 ignore 'star import' warnings (boolean) | |
| 106 @return dictionary with the keys 'error' and 'warnings' which | |
| 107 hold a list containing details about the error/ warnings | |
| 108 (file name, line number, column, codestring (only at syntax | |
| 109 errors), the message, a list with arguments for the message) | |
| 110 """ | |
| 111 return __syntaxAndPyflakesCheck(filename, codestring, checkFlakes, | |
| 112 ignoreStarImportWarnings) | |
| 113 | |
| 114 | |
| 115 def syntaxAndPyflakesBatchCheck(argumentsList, send, fx, cancelled, | |
| 116 maxProcesses=0): | |
| 117 """ | |
| 118 Module function to check syntax for a batch of files. | |
| 119 | |
| 120 @param argumentsList list of arguments tuples as given for | |
| 121 syntaxAndPyflakesCheck | |
| 122 @type list | |
| 123 @param send reference to send function | |
| 124 @type func | |
| 125 @param fx registered service name | |
| 126 @type str | |
| 127 @param cancelled reference to function checking for a cancellation | |
| 128 @type func | |
| 129 @param maxProcesses number of processes to be used | |
| 130 @type int | |
| 131 """ | |
| 132 if maxProcesses == 0: | |
| 133 # determine based on CPU count | |
| 134 try: | |
| 135 NumberOfProcesses = multiprocessing.cpu_count() | |
| 136 if NumberOfProcesses >= 1: | |
| 137 NumberOfProcesses -= 1 | |
| 138 except NotImplementedError: | |
| 139 NumberOfProcesses = 1 | |
| 140 else: | |
| 141 NumberOfProcesses = maxProcesses | |
| 142 | |
| 143 # Create queues | |
| 144 taskQueue = multiprocessing.Queue() | |
| 145 doneQueue = multiprocessing.Queue() | |
| 146 | |
| 147 # Submit tasks (initially two time number of processes | |
| 148 initialTasks = 2 * NumberOfProcesses | |
| 149 for task in argumentsList[:initialTasks]: | |
| 150 taskQueue.put(task) | |
| 151 | |
| 152 # Start worker processes | |
| 153 for _ in range(NumberOfProcesses): | |
| 154 multiprocessing.Process( | |
| 155 target=worker, args=(taskQueue, doneQueue) | |
| 156 ).start() | |
| 157 | |
| 158 # Get and send results | |
| 159 endIndex = len(argumentsList) - initialTasks | |
| 160 for i in range(len(argumentsList)): | |
| 161 resultSent = False | |
| 162 wasCancelled = False | |
| 163 | |
| 164 while not resultSent: | |
| 165 try: | |
| 166 # get result (waiting max. 3 seconds and send it to frontend | |
| 167 filename, result = doneQueue.get() | |
| 168 send(fx, filename, result) | |
| 169 resultSent = True | |
| 170 except queue.Empty: | |
| 171 # ignore empty queue, just carry on | |
| 172 if cancelled(): | |
| 173 wasCancelled = True | |
| 174 break | |
| 175 | |
| 176 if wasCancelled or cancelled(): | |
| 177 # just exit the loop ignoring the results of queued tasks | |
| 178 break | |
| 179 | |
| 180 if i < endIndex: | |
| 181 taskQueue.put(argumentsList[i + initialTasks]) | |
| 182 | |
| 183 # Tell child processes to stop | |
| 184 for _ in range(NumberOfProcesses): | |
| 185 taskQueue.put('STOP') | |
| 186 | |
| 187 | |
| 188 def worker(inputQueue, outputQueue): | |
| 189 """ | |
| 190 Module function acting as the parallel worker for the syntax check. | |
| 191 | |
| 192 @param inputQueue input queue (multiprocessing.Queue) | |
| 193 @param outputQueue output queue (multiprocessing.Queue) | |
| 194 """ | |
| 195 for filename, args in iter(inputQueue.get, 'STOP'): | |
| 196 source, checkFlakes, ignoreStarImportWarnings = args | |
| 197 result = __syntaxAndPyflakesCheck(filename, source, checkFlakes, | |
| 198 ignoreStarImportWarnings) | |
| 199 outputQueue.put((filename, result)) | |
| 200 | |
| 201 | |
| 202 def __syntaxAndPyflakesCheck(filename, codestring, checkFlakes=True, | |
| 203 ignoreStarImportWarnings=False): | |
| 204 """ | |
| 205 Function to compile one Python source file to Python bytecode | |
| 206 and to perform a pyflakes check. | |
| 207 | |
| 208 @param filename source filename | |
| 209 @type str | |
| 210 @param codestring string containing the code to compile | |
| 211 @type str | |
| 212 @param checkFlakes flag indicating to do a pyflakes check | |
| 213 @type bool | |
| 214 @param ignoreStarImportWarnings flag indicating to | |
| 215 ignore 'star import' warnings | |
| 216 @type bool | |
| 217 @return dictionary with the keys 'error' and 'warnings' which | |
| 218 hold a list containing details about the error/ warnings | |
| 219 (file name, line number, column, codestring (only at syntax | |
| 220 errors), the message, a list with arguments for the message) | |
| 221 @rtype dict | |
| 222 """ | |
| 223 import builtins | |
| 224 | |
| 225 try: | |
| 226 codestring = normalizeCode(codestring) | |
| 227 | |
| 228 # Check for VCS conflict markers | |
| 229 for conflictMarkerRe in VcsConflictMarkerRegExpList: | |
| 230 conflict = conflictMarkerRe.search(codestring) | |
| 231 if conflict is not None: | |
| 232 start, i = conflict.span() | |
| 233 lineindex = 1 + codestring.count("\n", 0, start) | |
| 234 return [{'error': | |
| 235 (filename, lineindex, 0, "", | |
| 236 "VCS conflict marker found") | |
| 237 }] | |
| 238 | |
| 239 if filename.endswith('.ptl'): | |
| 240 try: | |
| 241 import quixote.ptl_compile | |
| 242 except ImportError: | |
| 243 return [{'error': (filename, 0, 0, '', | |
| 244 'Quixote plugin not found.')}] | |
| 245 template = quixote.ptl_compile.Template(codestring, filename) | |
| 246 template.compile() | |
| 247 else: | |
| 248 module = builtins.compile( | |
| 249 codestring, filename, 'exec', ast.PyCF_ONLY_AST) | |
| 250 except SyntaxError as detail: | |
| 251 index = 0 | |
| 252 code = "" | |
| 253 error = "" | |
| 254 lines = traceback.format_exception_only(SyntaxError, detail) | |
| 255 match = re.match(r'\s*File "(.+)", line (\d+)', | |
| 256 lines[0].replace('<string>', filename)) | |
| 257 if match is not None: | |
| 258 fn, line = match.group(1, 2) | |
| 259 if lines[1].startswith('SyntaxError:'): | |
| 260 error = re.match('SyntaxError: (.+)', lines[1]).group(1) | |
| 261 else: | |
| 262 code = re.match('(.+)', lines[1]).group(1) | |
| 263 for seLine in lines[2:]: | |
| 264 if seLine.startswith('SyntaxError:'): | |
| 265 error = re.match('SyntaxError: (.+)', seLine).group(1) | |
| 266 elif seLine.rstrip().endswith('^'): | |
| 267 index = len(seLine.rstrip()) - 4 | |
| 268 else: | |
| 269 fn = detail.filename | |
| 270 line = detail.lineno or 1 | |
| 271 error = detail.msg | |
| 272 return [{'error': (fn, int(line), index, code.strip(), error)}] | |
| 273 except ValueError as detail: | |
| 274 try: | |
| 275 fn = detail.filename | |
| 276 line = detail.lineno | |
| 277 error = detail.msg | |
| 278 except AttributeError: | |
| 279 fn = filename | |
| 280 line = 1 | |
| 281 error = str(detail) | |
| 282 return [{'error': (fn, line, 0, "", error)}] | |
| 283 except Exception as detail: | |
| 284 with contextlib.suppress(Exception): | |
| 285 fn = detail.filename | |
| 286 line = detail.lineno | |
| 287 error = detail.msg | |
| 288 return [{'error': (fn, line, 0, "", error)}] | |
| 289 | |
| 290 # pyflakes | |
| 291 if not checkFlakes: | |
| 292 return [{}] | |
| 293 | |
| 294 results = [] | |
| 295 lines = codestring.splitlines() | |
| 296 try: | |
| 297 warnings = Checker(module, filename, withDoctest=True) | |
| 298 warnings.messages.sort(key=lambda a: a.lineno) | |
| 299 for warning in warnings.messages: | |
| 300 if ( | |
| 301 ignoreStarImportWarnings and | |
| 302 isinstance(warning, (ImportStarUsed, ImportStarUsage)) | |
| 303 ): | |
| 304 continue | |
| 305 | |
| 306 _fn, lineno, col, message, msg_args = warning.getMessageData() | |
| 307 lineFlags = extractLineFlags(lines[lineno - 1].strip()) | |
| 308 with contextlib.suppress(IndexError): | |
| 309 lineFlags += extractLineFlags(lines[lineno].strip(), | |
| 310 flagsLine=True) | |
| 311 if ( | |
| 312 "__IGNORE_WARNING__" not in lineFlags and | |
| 313 "noqa" not in lineFlags | |
| 314 ): | |
| 315 results.append((_fn, lineno, col, "", message, msg_args)) | |
| 316 except SyntaxError as err: | |
| 317 msg = err.text.strip() if err.text.strip() else err.msg | |
| 318 results.append((filename, err.lineno, 0, "FLAKES_ERROR", msg, [])) | |
| 319 | |
| 320 return [{'warnings': results}] |
