Mercurial Repositories > eric / file comparison
comparison: eric6/Plugins/CheckerPlugins/SyntaxChecker/SyntaxCheck.py
eric6/Plugins/CheckerPlugins/SyntaxChecker/SyntaxCheck.py
- changeset 6942
- 2602857055c5
- parent 6645
- ad476851d7e0
- child 7256
- 4ef3b78ebb4e
equal
deleted
inserted
replaced
| 6941:f99d60d6b59b | 6942:2602857055c5 |
|---|---|
| 1 # -*- coding: utf-8 -*- | |
| 2 | |
| 3 # Copyright (c) 2011 - 2019 Detlev Offenbach <detlev@die-offenbachs.de> | |
| 4 # | |
| 5 # pylint: disable=C0103 | |
| 6 | |
| 7 """ | |
| 8 Module implementing the syntax check for Python 2/3. | |
| 9 """ | |
| 10 | |
| 11 try: # Only for Py2 | |
| 12 import Queue as queue | |
| 13 except ImportError: | |
| 14 import queue | |
| 15 | |
| 16 import ast | |
| 17 import re | |
| 18 import sys | |
| 19 import traceback | |
| 20 import multiprocessing | |
| 21 | |
| 22 | |
| 23 try: | |
| 24 from pyflakes.checker import Checker | |
| 25 from pyflakes.messages import ImportStarUsed, ImportStarUsage | |
| 26 except ImportError: | |
| 27 pass | |
| 28 | |
| 29 VcsConflictMarkerRegExpList = ( | |
| 30 re.compile( | |
| 31 r"""^<<<<<<< .*?\|\|\|\|\|\|\| .*?=======.*?>>>>>>> .*?$""", | |
| 32 re.MULTILINE | re.DOTALL | |
| 33 ), | |
| 34 re.compile( | |
| 35 r"""^<<<<<<< .*?=======.*?>>>>>>> .*?$""", | |
| 36 re.MULTILINE | re.DOTALL | |
| 37 ), | |
| 38 ) | |
| 39 | |
| 40 | |
| 41 def initService(): | |
| 42 """ | |
| 43 Initialize the service and return the entry point. | |
| 44 | |
| 45 @return the entry point for the background client (function) | |
| 46 """ | |
| 47 return syntaxAndPyflakesCheck | |
| 48 | |
| 49 | |
| 50 def initBatchService(): | |
| 51 """ | |
| 52 Initialize the batch service and return the entry point. | |
| 53 | |
| 54 @return the entry point for the background client (function) | |
| 55 """ | |
| 56 return syntaxAndPyflakesBatchCheck | |
| 57 | |
| 58 | |
| 59 def normalizeCode(codestring): | |
| 60 """ | |
| 61 Function to normalize the given code. | |
| 62 | |
| 63 @param codestring code to be normalized (string) | |
| 64 @return normalized code (string) | |
| 65 """ | |
| 66 codestring = codestring.replace("\r\n", "\n").replace("\r", "\n") | |
| 67 | |
| 68 if codestring and codestring[-1] != '\n': | |
| 69 codestring = codestring + '\n' | |
| 70 | |
| 71 # Check type for py2: if not str it's unicode | |
| 72 if sys.version_info[0] == 2: | |
| 73 try: | |
| 74 codestring = codestring.encode('utf-8') | |
| 75 except UnicodeError: | |
| 76 pass | |
| 77 | |
| 78 return codestring | |
| 79 | |
| 80 | |
| 81 def extractLineFlags(line, startComment="#", endComment="", flagsLine=False): | |
| 82 """ | |
| 83 Function to extract flags starting and ending with '__' from a line | |
| 84 comment. | |
| 85 | |
| 86 @param line line to extract flags from (string) | |
| 87 @keyparam startComment string identifying the start of the comment (string) | |
| 88 @keyparam endComment string identifying the end of a comment (string) | |
| 89 @keyparam flagsLine flag indicating to check for a flags only line (bool) | |
| 90 @return list containing the extracted flags (list of strings) | |
| 91 """ | |
| 92 flags = [] | |
| 93 | |
| 94 if not flagsLine or ( | |
| 95 flagsLine and line.strip().startswith(startComment)): | |
| 96 pos = line.rfind(startComment) | |
| 97 if pos >= 0: | |
| 98 comment = line[pos + len(startComment):].strip() | |
| 99 if endComment: | |
| 100 endPos = line.rfind(endComment) | |
| 101 if endPos >= 0: | |
| 102 comment = comment[:endPos] | |
| 103 flags = [f.strip() for f in comment.split() | |
| 104 if (f.startswith("__") and f.endswith("__"))] | |
| 105 flags += [f.strip().lower() for f in comment.split() | |
| 106 if f in ("noqa", "NOQA")] | |
| 107 return flags | |
| 108 | |
| 109 | |
| 110 def syntaxAndPyflakesCheck(filename, codestring, checkFlakes=True, | |
| 111 ignoreStarImportWarnings=False): | |
| 112 """ | |
| 113 Function to compile one Python source file to Python bytecode | |
| 114 and to perform a pyflakes check. | |
| 115 | |
| 116 @param filename source filename (string) | |
| 117 @param codestring string containing the code to compile (string) | |
| 118 @keyparam checkFlakes flag indicating to do a pyflakes check (boolean) | |
| 119 @keyparam ignoreStarImportWarnings flag indicating to | |
| 120 ignore 'star import' warnings (boolean) | |
| 121 @return dictionary with the keys 'error' and 'warnings' which | |
| 122 hold a list containing details about the error/ warnings | |
| 123 (file name, line number, column, codestring (only at syntax | |
| 124 errors), the message, a list with arguments for the message) | |
| 125 """ | |
| 126 return __syntaxAndPyflakesCheck(filename, codestring, checkFlakes, | |
| 127 ignoreStarImportWarnings) | |
| 128 | |
| 129 | |
| 130 def syntaxAndPyflakesBatchCheck(argumentsList, send, fx, cancelled, | |
| 131 maxProcesses=0): | |
| 132 """ | |
| 133 Module function to check syntax for a batch of files. | |
| 134 | |
| 135 @param argumentsList list of arguments tuples as given for | |
| 136 syntaxAndPyflakesCheck | |
| 137 @type list | |
| 138 @param send reference to send function | |
| 139 @type func | |
| 140 @param fx registered service name | |
| 141 @type str | |
| 142 @param cancelled reference to function checking for a cancellation | |
| 143 @type func | |
| 144 @param maxProcesses number of processes to be used | |
| 145 @type int | |
| 146 """ | |
| 147 if maxProcesses == 0: | |
| 148 # determine based on CPU count | |
| 149 try: | |
| 150 NumberOfProcesses = multiprocessing.cpu_count() | |
| 151 if NumberOfProcesses >= 1: | |
| 152 NumberOfProcesses -= 1 | |
| 153 except NotImplementedError: | |
| 154 NumberOfProcesses = 1 | |
| 155 else: | |
| 156 NumberOfProcesses = maxProcesses | |
| 157 | |
| 158 # Create queues | |
| 159 taskQueue = multiprocessing.Queue() | |
| 160 doneQueue = multiprocessing.Queue() | |
| 161 | |
| 162 # Submit tasks (initially two time number of processes | |
| 163 initialTasks = 2 * NumberOfProcesses | |
| 164 for task in argumentsList[:initialTasks]: | |
| 165 taskQueue.put(task) | |
| 166 | |
| 167 # Start worker processes | |
| 168 for _ in range(NumberOfProcesses): | |
| 169 multiprocessing.Process(target=worker, args=(taskQueue, doneQueue))\ | |
| 170 .start() | |
| 171 | |
| 172 # Get and send results | |
| 173 endIndex = len(argumentsList) - initialTasks | |
| 174 for i in range(len(argumentsList)): | |
| 175 resultSent = False | |
| 176 wasCancelled = False | |
| 177 | |
| 178 while not resultSent: | |
| 179 try: | |
| 180 # get result (waiting max. 3 seconds and send it to frontend | |
| 181 filename, result = doneQueue.get() | |
| 182 send(fx, filename, result) | |
| 183 resultSent = True | |
| 184 except queue.Empty: | |
| 185 # ignore empty queue, just carry on | |
| 186 if cancelled(): | |
| 187 wasCancelled = True | |
| 188 break | |
| 189 | |
| 190 if wasCancelled or cancelled(): | |
| 191 # just exit the loop ignoring the results of queued tasks | |
| 192 break | |
| 193 | |
| 194 if i < endIndex: | |
| 195 taskQueue.put(argumentsList[i + initialTasks]) | |
| 196 | |
| 197 # Tell child processes to stop | |
| 198 for _ in range(NumberOfProcesses): | |
| 199 taskQueue.put('STOP') | |
| 200 | |
| 201 | |
| 202 def worker(inputQueue, outputQueue): | |
| 203 """ | |
| 204 Module function acting as the parallel worker for the style check. | |
| 205 | |
| 206 @param inputQueue input queue (multiprocessing.Queue) | |
| 207 @param outputQueue output queue (multiprocessing.Queue) | |
| 208 """ | |
| 209 for filename, args in iter(inputQueue.get, 'STOP'): | |
| 210 source, checkFlakes, ignoreStarImportWarnings = args | |
| 211 result = __syntaxAndPyflakesCheck(filename, source, checkFlakes, | |
| 212 ignoreStarImportWarnings) | |
| 213 outputQueue.put((filename, result)) | |
| 214 | |
| 215 | |
| 216 def __syntaxAndPyflakesCheck(filename, codestring, checkFlakes=True, | |
| 217 ignoreStarImportWarnings=False): | |
| 218 """ | |
| 219 Function to compile one Python source file to Python bytecode | |
| 220 and to perform a pyflakes check. | |
| 221 | |
| 222 @param filename source filename (string) | |
| 223 @param codestring string containing the code to compile (string) | |
| 224 @keyparam checkFlakes flag indicating to do a pyflakes check (boolean) | |
| 225 @keyparam ignoreStarImportWarnings flag indicating to | |
| 226 ignore 'star import' warnings (boolean) | |
| 227 @return dictionary with the keys 'error' and 'warnings' which | |
| 228 hold a list containing details about the error/ warnings | |
| 229 (file name, line number, column, codestring (only at syntax | |
| 230 errors), the message, a list with arguments for the message) | |
| 231 """ | |
| 232 try: | |
| 233 import builtins | |
| 234 except ImportError: | |
| 235 import __builtin__ as builtins # __IGNORE_WARNING__ | |
| 236 | |
| 237 try: | |
| 238 if sys.version_info[0] == 2: | |
| 239 file_enc = filename.encode(sys.getfilesystemencoding()) | |
| 240 else: | |
| 241 file_enc = filename | |
| 242 | |
| 243 # It also encode the code back to avoid 'Encoding declaration in | |
| 244 # unicode string' exception on Python2 | |
| 245 codestring = normalizeCode(codestring) | |
| 246 | |
| 247 # Check for VCS conflict markers | |
| 248 for conflictMarkerRe in VcsConflictMarkerRegExpList: | |
| 249 conflict = conflictMarkerRe.search(codestring) | |
| 250 if conflict is not None: | |
| 251 start, i = conflict.span() | |
| 252 lineindex = 1 + codestring.count("\n", 0, start) | |
| 253 return [{'error': | |
| 254 (file_enc, lineindex, 0, "", | |
| 255 "VCS conflict marker found") | |
| 256 }] | |
| 257 | |
| 258 if filename.endswith('.ptl'): | |
| 259 try: | |
| 260 import quixote.ptl_compile | |
| 261 except ImportError: | |
| 262 return [{'error': (filename, 0, 0, '', | |
| 263 'Quixote plugin not found.')}] | |
| 264 template = quixote.ptl_compile.Template(codestring, file_enc) | |
| 265 template.compile() | |
| 266 else: | |
| 267 module = builtins.compile( | |
| 268 codestring, file_enc, 'exec', ast.PyCF_ONLY_AST) | |
| 269 except SyntaxError as detail: | |
| 270 index = 0 | |
| 271 code = "" | |
| 272 error = "" | |
| 273 lines = traceback.format_exception_only(SyntaxError, detail) | |
| 274 if sys.version_info[0] == 2: | |
| 275 lines = [x.decode(sys.getfilesystemencoding()) for x in lines] | |
| 276 match = re.match(r'\s*File "(.+)", line (\d+)', | |
| 277 lines[0].replace('<string>', filename)) | |
| 278 if match is not None: | |
| 279 fn, line = match.group(1, 2) | |
| 280 if lines[1].startswith('SyntaxError:'): | |
| 281 error = re.match('SyntaxError: (.+)', lines[1]).group(1) | |
| 282 else: | |
| 283 code = re.match('(.+)', lines[1]).group(1) | |
| 284 for seLine in lines[2:]: | |
| 285 if seLine.startswith('SyntaxError:'): | |
| 286 error = re.match('SyntaxError: (.+)', seLine).group(1) | |
| 287 elif seLine.rstrip().endswith('^'): | |
| 288 index = len(seLine.rstrip()) - 4 | |
| 289 else: | |
| 290 fn = detail.filename | |
| 291 line = detail.lineno or 1 | |
| 292 error = detail.msg | |
| 293 return [{'error': (fn, int(line), index, code.strip(), error)}] | |
| 294 except ValueError as detail: | |
| 295 try: | |
| 296 fn = detail.filename | |
| 297 line = detail.lineno | |
| 298 error = detail.msg | |
| 299 except AttributeError: | |
| 300 fn = filename | |
| 301 line = 1 | |
| 302 error = str(detail) | |
| 303 return [{'error': (fn, line, 0, "", error)}] | |
| 304 except Exception as detail: | |
| 305 try: | |
| 306 fn = detail.filename | |
| 307 line = detail.lineno | |
| 308 error = detail.msg | |
| 309 return [{'error': (fn, line, 0, "", error)}] | |
| 310 except Exception: | |
| 311 pass | |
| 312 | |
| 313 # pyflakes | |
| 314 if not checkFlakes: | |
| 315 return [{}] | |
| 316 | |
| 317 results = [] | |
| 318 lines = codestring.splitlines() | |
| 319 try: | |
| 320 warnings = Checker(module, filename, withDoctest=True) | |
| 321 warnings.messages.sort(key=lambda a: a.lineno) | |
| 322 for warning in warnings.messages: | |
| 323 if ignoreStarImportWarnings and ( | |
| 324 isinstance(warning, ImportStarUsed) or | |
| 325 isinstance(warning, ImportStarUsage) | |
| 326 ): | |
| 327 continue | |
| 328 | |
| 329 _fn, lineno, col, message, msg_args = warning.getMessageData() | |
| 330 lineFlags = extractLineFlags(lines[lineno - 1].strip()) | |
| 331 try: | |
| 332 lineFlags += extractLineFlags(lines[lineno].strip(), | |
| 333 flagsLine=True) | |
| 334 except IndexError: | |
| 335 pass | |
| 336 if "__IGNORE_WARNING__" not in lineFlags and \ | |
| 337 "noqa" not in lineFlags: | |
| 338 results.append((_fn, lineno, col, "", message, msg_args)) | |
| 339 except SyntaxError as err: | |
| 340 if err.text.strip(): | |
| 341 msg = err.text.strip() | |
| 342 else: | |
| 343 msg = err.msg | |
| 344 results.append((filename, err.lineno, 0, "FLAKES_ERROR", msg, [])) | |
| 345 | |
| 346 return [{'warnings': results}] | |
| 347 | |
| 348 # | |
| 349 # eflag: noqa = M702 |
