Mercurial Repositories > eric / file comparison

comparison: eric6/ThirdParty/Pygments/pygments/lexers/python.py

eric6/ThirdParty/Pygments/pygments/lexers/python.py

changeset 7547
21b0534faebc
parent 6942
2602857055c5
child 7701
25f42e208e08
equal deleted inserted replaced
7546:bf5f777260a6 7547:21b0534faebc
3 pygments.lexers.python 3 pygments.lexers.python
4 ~~~~~~~~~~~~~~~~~~~~~~ 4 ~~~~~~~~~~~~~~~~~~~~~~
5 5
6 Lexers for Python and related languages. 6 Lexers for Python and related languages.
7 7
8 :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. 8 :copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS.
9 :license: BSD, see LICENSE for details. 9 :license: BSD, see LICENSE for details.
10 """ 10 """
11 11
12 import re 12 import re
13 13
17 from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ 17 from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
18 Number, Punctuation, Generic, Other, Error 18 Number, Punctuation, Generic, Other, Error
19 from pygments import unistring as uni 19 from pygments import unistring as uni
20 20
21 __all__ = ['PythonLexer', 'PythonConsoleLexer', 'PythonTracebackLexer', 21 __all__ = ['PythonLexer', 'PythonConsoleLexer', 'PythonTracebackLexer',
22 'Python3Lexer', 'Python3TracebackLexer', 'CythonLexer', 22 'Python2Lexer', 'Python2TracebackLexer',
23 'DgLexer', 'NumPyLexer'] 23 'CythonLexer', 'DgLexer', 'NumPyLexer']
24 24
25 line_re = re.compile('.*?\n') 25 line_re = re.compile('.*?\n')
26 26
27 27
28 class PythonLexer(RegexLexer): 28 class PythonLexer(RegexLexer):
29 """ 29 """
30 For `Python <http://www.python.org>`_ source code. 30 For `Python <http://www.python.org>`_ source code (version 3.x).
31
32 .. versionadded:: 0.10
33
34 .. versionchanged:: 2.5
35 This is now the default ``PythonLexer``. It is still available as the
36 alias ``Python3Lexer``.
31 """ 37 """
32 38
33 name = 'Python' 39 name = 'Python'
34 aliases = ['python', 'py', 'sage'] 40 aliases = ['python', 'py', 'sage', 'python3', 'py3']
35 filenames = ['*.py', '*.pyw', '*.sc', 'SConstruct', 'SConscript', '*.tac', '*.sage'] 41 filenames = [
36 mimetypes = ['text/x-python', 'application/x-python'] 42 '*.py',
43 '*.pyw',
44 # Jython
45 '*.jy',
46 # Sage
47 '*.sage',
48 # SCons
49 '*.sc',
50 'SConstruct',
51 'SConscript',
52 # Skylark/Starlark (used by Bazel, Buck, and Pants)
53 '*.bzl',
54 'BUCK',
55 'BUILD',
56 'BUILD.bazel',
57 'WORKSPACE',
58 # Twisted Application infrastructure
59 '*.tac',
60 ]
61 mimetypes = ['text/x-python', 'application/x-python',
62 'text/x-python3', 'application/x-python3']
63
64 flags = re.MULTILINE | re.UNICODE
65
66 uni_name = "[%s][%s]*" % (uni.xid_start, uni.xid_continue)
67
68 def innerstring_rules(ttype):
69 return [
70 # the old style '%s' % (...) string formatting (still valid in Py3)
71 (r'%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?'
72 '[hlL]?[E-GXc-giorsaux%]', String.Interpol),
73 # the new style '{}'.format(...) string formatting
74 (r'\{'
75 r'((\w+)((\.\w+)|(\[[^\]]+\]))*)?' # field name
76 r'(\![sra])?' # conversion
77 r'(\:(.?[<>=\^])?[-+ ]?#?0?(\d+)?,?(\.\d+)?[E-GXb-gnosx%]?)?'
78 r'\}', String.Interpol),
79
80 # backslashes, quotes and formatting signs must be parsed one at a time
81 (r'[^\\\'"%{\n]+', ttype),
82 (r'[\'"\\]', ttype),
83 # unhandled string formatting sign
84 (r'%|(\{{1,2})', ttype)
85 # newlines are an error (use "nl" state)
86 ]
87
88 def fstring_rules(ttype):
89 return [
90 # Assuming that a '}' is the closing brace after format specifier.
91 # Sadly, this means that we won't detect syntax error. But it's
92 # more important to parse correct syntax correctly, than to
93 # highlight invalid syntax.
94 (r'\}', String.Interpol),
95 (r'\{', String.Interpol, 'expr-inside-fstring'),
96 # backslashes, quotes and formatting signs must be parsed one at a time
97 (r'[^\\\'"{}\n]+', ttype),
98 (r'[\'"\\]', ttype),
99 # newlines are an error (use "nl" state)
100 ]
101
102 tokens = {
103 'root': [
104 (r'\n', Text),
105 (r'^(\s*)([rRuUbB]{,2})("""(?:.|\n)*?""")',
106 bygroups(Text, String.Affix, String.Doc)),
107 (r"^(\s*)([rRuUbB]{,2})('''(?:.|\n)*?''')",
108 bygroups(Text, String.Affix, String.Doc)),
109 (r'\A#!.+$', Comment.Hashbang),
110 (r'#.*$', Comment.Single),
111 (r'\\\n', Text),
112 (r'\\', Text),
113 include('keywords'),
114 (r'(def)((?:\s|\\\s)+)', bygroups(Keyword, Text), 'funcname'),
115 (r'(class)((?:\s|\\\s)+)', bygroups(Keyword, Text), 'classname'),
116 (r'(from)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Text),
117 'fromimport'),
118 (r'(import)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Text),
119 'import'),
120 include('expr'),
121 ],
122 'expr': [
123 # raw f-strings
124 ('(?i)(rf|fr)(""")',
125 bygroups(String.Affix, String.Double), 'tdqf'),
126 ("(?i)(rf|fr)(''')",
127 bygroups(String.Affix, String.Single), 'tsqf'),
128 ('(?i)(rf|fr)(")',
129 bygroups(String.Affix, String.Double), 'dqf'),
130 ("(?i)(rf|fr)(')",
131 bygroups(String.Affix, String.Single), 'sqf'),
132 # non-raw f-strings
133 ('([fF])(""")', bygroups(String.Affix, String.Double),
134 combined('fstringescape', 'tdqf')),
135 ("([fF])(''')", bygroups(String.Affix, String.Single),
136 combined('fstringescape', 'tsqf')),
137 ('([fF])(")', bygroups(String.Affix, String.Double),
138 combined('fstringescape', 'dqf')),
139 ("([fF])(')", bygroups(String.Affix, String.Single),
140 combined('fstringescape', 'sqf')),
141 # raw strings
142 ('(?i)(rb|br|r)(""")',
143 bygroups(String.Affix, String.Double), 'tdqs'),
144 ("(?i)(rb|br|r)(''')",
145 bygroups(String.Affix, String.Single), 'tsqs'),
146 ('(?i)(rb|br|r)(")',
147 bygroups(String.Affix, String.Double), 'dqs'),
148 ("(?i)(rb|br|r)(')",
149 bygroups(String.Affix, String.Single), 'sqs'),
150 # non-raw strings
151 ('([uUbB]?)(""")', bygroups(String.Affix, String.Double),
152 combined('stringescape', 'tdqs')),
153 ("([uUbB]?)(''')", bygroups(String.Affix, String.Single),
154 combined('stringescape', 'tsqs')),
155 ('([uUbB]?)(")', bygroups(String.Affix, String.Double),
156 combined('stringescape', 'dqs')),
157 ("([uUbB]?)(')", bygroups(String.Affix, String.Single),
158 combined('stringescape', 'sqs')),
159 (r'[^\S\n]+', Text),
160 (r'!=|==|<<|>>|:=|[-~+/*%=<>&^|.]', Operator),
161 (r'[]{}:(),;[]', Punctuation),
162 (r'(in|is|and|or|not)\b', Operator.Word),
163 include('expr-keywords'),
164 include('builtins'),
165 include('magicfuncs'),
166 include('magicvars'),
167 include('name'),
168 include('numbers'),
169 ],
170 'expr-inside-fstring': [
171 (r'[{([]', Punctuation, 'expr-inside-fstring-inner'),
172 # without format specifier
173 (r'(=\s*)?' # debug (https://bugs.python.org/issue36817)
174 r'(\![sraf])?' # conversion
175 r'}', String.Interpol, '#pop'),
176 # with format specifier
177 # we'll catch the remaining '}' in the outer scope
178 (r'(=\s*)?' # debug (https://bugs.python.org/issue36817)
179 r'(\![sraf])?' # conversion
180 r':', String.Interpol, '#pop'),
181 (r'[^\S]+', Text), # allow new lines
182 include('expr'),
183 ],
184 'expr-inside-fstring-inner': [
185 (r'[{([]', Punctuation, 'expr-inside-fstring-inner'),
186 (r'[])}]', Punctuation, '#pop'),
187 (r'[^\S]+', Text), # allow new lines
188 include('expr'),
189 ],
190 'expr-keywords': [
191 # Based on https://docs.python.org/3/reference/expressions.html
192 (words((
193 'async for', 'await', 'else', 'for', 'if', 'lambda',
194 'yield', 'yield from'), suffix=r'\b'),
195 Keyword),
196 (words(('True', 'False', 'None'), suffix=r'\b'), Keyword.Constant),
197 ],
198 'keywords': [
199 (words((
200 'assert', 'async', 'await', 'break', 'continue', 'del', 'elif',
201 'else', 'except', 'finally', 'for', 'global', 'if', 'lambda',
202 'pass', 'raise', 'nonlocal', 'return', 'try', 'while', 'yield',
203 'yield from', 'as', 'with'), suffix=r'\b'),
204 Keyword),
205 (words(('True', 'False', 'None'), suffix=r'\b'), Keyword.Constant),
206 ],
207 'builtins': [
208 (words((
209 '__import__', 'abs', 'all', 'any', 'bin', 'bool', 'bytearray',
210 'bytes', 'chr', 'classmethod', 'cmp', 'compile', 'complex',
211 'delattr', 'dict', 'dir', 'divmod', 'enumerate', 'eval', 'filter',
212 'float', 'format', 'frozenset', 'getattr', 'globals', 'hasattr',
213 'hash', 'hex', 'id', 'input', 'int', 'isinstance', 'issubclass',
214 'iter', 'len', 'list', 'locals', 'map', 'max', 'memoryview',
215 'min', 'next', 'object', 'oct', 'open', 'ord', 'pow', 'print',
216 'property', 'range', 'repr', 'reversed', 'round', 'set', 'setattr',
217 'slice', 'sorted', 'staticmethod', 'str', 'sum', 'super', 'tuple',
218 'type', 'vars', 'zip'), prefix=r'(?<!\.)', suffix=r'\b'),
219 Name.Builtin),
220 (r'(?<!\.)(self|Ellipsis|NotImplemented|cls)\b', Name.Builtin.Pseudo),
221 (words((
222 'ArithmeticError', 'AssertionError', 'AttributeError',
223 'BaseException', 'BufferError', 'BytesWarning', 'DeprecationWarning',
224 'EOFError', 'EnvironmentError', 'Exception', 'FloatingPointError',
225 'FutureWarning', 'GeneratorExit', 'IOError', 'ImportError',
226 'ImportWarning', 'IndentationError', 'IndexError', 'KeyError',
227 'KeyboardInterrupt', 'LookupError', 'MemoryError', 'NameError',
228 'NotImplementedError', 'OSError', 'OverflowError',
229 'PendingDeprecationWarning', 'ReferenceError', 'ResourceWarning',
230 'RuntimeError', 'RuntimeWarning', 'StopIteration',
231 'SyntaxError', 'SyntaxWarning', 'SystemError', 'SystemExit',
232 'TabError', 'TypeError', 'UnboundLocalError', 'UnicodeDecodeError',
233 'UnicodeEncodeError', 'UnicodeError', 'UnicodeTranslateError',
234 'UnicodeWarning', 'UserWarning', 'ValueError', 'VMSError',
235 'Warning', 'WindowsError', 'ZeroDivisionError',
236 # new builtin exceptions from PEP 3151
237 'BlockingIOError', 'ChildProcessError', 'ConnectionError',
238 'BrokenPipeError', 'ConnectionAbortedError', 'ConnectionRefusedError',
239 'ConnectionResetError', 'FileExistsError', 'FileNotFoundError',
240 'InterruptedError', 'IsADirectoryError', 'NotADirectoryError',
241 'PermissionError', 'ProcessLookupError', 'TimeoutError',
242 # others new in Python 3
243 'StopAsyncIteration', 'ModuleNotFoundError', 'RecursionError'),
244 prefix=r'(?<!\.)', suffix=r'\b'),
245 Name.Exception),
246 ],
247 'magicfuncs': [
248 (words((
249 '__abs__', '__add__', '__aenter__', '__aexit__', '__aiter__',
250 '__and__', '__anext__', '__await__', '__bool__', '__bytes__',
251 '__call__', '__complex__', '__contains__', '__del__', '__delattr__',
252 '__delete__', '__delitem__', '__dir__', '__divmod__', '__enter__',
253 '__eq__', '__exit__', '__float__', '__floordiv__', '__format__',
254 '__ge__', '__get__', '__getattr__', '__getattribute__',
255 '__getitem__', '__gt__', '__hash__', '__iadd__', '__iand__',
256 '__ifloordiv__', '__ilshift__', '__imatmul__', '__imod__',
257 '__imul__', '__index__', '__init__', '__instancecheck__',
258 '__int__', '__invert__', '__ior__', '__ipow__', '__irshift__',
259 '__isub__', '__iter__', '__itruediv__', '__ixor__', '__le__',
260 '__len__', '__length_hint__', '__lshift__', '__lt__', '__matmul__',
261 '__missing__', '__mod__', '__mul__', '__ne__', '__neg__',
262 '__new__', '__next__', '__or__', '__pos__', '__pow__',
263 '__prepare__', '__radd__', '__rand__', '__rdivmod__', '__repr__',
264 '__reversed__', '__rfloordiv__', '__rlshift__', '__rmatmul__',
265 '__rmod__', '__rmul__', '__ror__', '__round__', '__rpow__',
266 '__rrshift__', '__rshift__', '__rsub__', '__rtruediv__',
267 '__rxor__', '__set__', '__setattr__', '__setitem__', '__str__',
268 '__sub__', '__subclasscheck__', '__truediv__',
269 '__xor__'), suffix=r'\b'),
270 Name.Function.Magic),
271 ],
272 'magicvars': [
273 (words((
274 '__annotations__', '__bases__', '__class__', '__closure__',
275 '__code__', '__defaults__', '__dict__', '__doc__', '__file__',
276 '__func__', '__globals__', '__kwdefaults__', '__module__',
277 '__mro__', '__name__', '__objclass__', '__qualname__',
278 '__self__', '__slots__', '__weakref__'), suffix=r'\b'),
279 Name.Variable.Magic),
280 ],
281 'numbers': [
282 (r'(\d(?:_?\d)*\.(?:\d(?:_?\d)*)?|(?:\d(?:_?\d)*)?\.\d(?:_?\d)*)'
283 r'([eE][+-]?\d(?:_?\d)*)?', Number.Float),
284 (r'\d(?:_?\d)*[eE][+-]?\d(?:_?\d)*j?', Number.Float),
285 (r'0[oO](?:_?[0-7])+', Number.Oct),
286 (r'0[bB](?:_?[01])+', Number.Bin),
287 (r'0[xX](?:_?[a-fA-F0-9])+', Number.Hex),
288 (r'\d(?:_?\d)*', Number.Integer),
289 ],
290 'name': [
291 (r'@' + uni_name, Name.Decorator),
292 (r'@', Operator), # new matrix multiplication operator
293 (uni_name, Name),
294 ],
295 'funcname': [
296 include('magicfuncs'),
297 (uni_name, Name.Function, '#pop'),
298 default('#pop'),
299 ],
300 'classname': [
301 (uni_name, Name.Class, '#pop'),
302 ],
303 'import': [
304 (r'(\s+)(as)(\s+)', bygroups(Text, Keyword, Text)),
305 (r'\.', Name.Namespace),
306 (uni_name, Name.Namespace),
307 (r'(\s*)(,)(\s*)', bygroups(Text, Operator, Text)),
308 default('#pop') # all else: go back
309 ],
310 'fromimport': [
311 (r'(\s+)(import)\b', bygroups(Text, Keyword.Namespace), '#pop'),
312 (r'\.', Name.Namespace),
313 # if None occurs here, it's "raise x from None", since None can
314 # never be a module name
315 (r'None\b', Name.Builtin.Pseudo, '#pop'),
316 (uni_name, Name.Namespace),
317 default('#pop'),
318 ],
319 'fstringescape': [
320 ('{{', String.Escape),
321 ('}}', String.Escape),
322 include('stringescape'),
323 ],
324 'stringescape': [
325 (r'\\([\\abfnrtv"\']|\n|N\{.*?\}|u[a-fA-F0-9]{4}|'
326 r'U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape)
327 ],
328 'fstrings-single': fstring_rules(String.Single),
329 'fstrings-double': fstring_rules(String.Double),
330 'strings-single': innerstring_rules(String.Single),
331 'strings-double': innerstring_rules(String.Double),
332 'dqf': [
333 (r'"', String.Double, '#pop'),
334 (r'\\\\|\\"|\\\n', String.Escape), # included here for raw strings
335 include('fstrings-double')
336 ],
337 'sqf': [
338 (r"'", String.Single, '#pop'),
339 (r"\\\\|\\'|\\\n", String.Escape), # included here for raw strings
340 include('fstrings-single')
341 ],
342 'dqs': [
343 (r'"', String.Double, '#pop'),
344 (r'\\\\|\\"|\\\n', String.Escape), # included here for raw strings
345 include('strings-double')
346 ],
347 'sqs': [
348 (r"'", String.Single, '#pop'),
349 (r"\\\\|\\'|\\\n", String.Escape), # included here for raw strings
350 include('strings-single')
351 ],
352 'tdqf': [
353 (r'"""', String.Double, '#pop'),
354 include('fstrings-double'),
355 (r'\n', String.Double)
356 ],
357 'tsqf': [
358 (r"'''", String.Single, '#pop'),
359 include('fstrings-single'),
360 (r'\n', String.Single)
361 ],
362 'tdqs': [
363 (r'"""', String.Double, '#pop'),
364 include('strings-double'),
365 (r'\n', String.Double)
366 ],
367 'tsqs': [
368 (r"'''", String.Single, '#pop'),
369 include('strings-single'),
370 (r'\n', String.Single)
371 ],
372 }
373
374 def analyse_text(text):
375 return shebang_matches(text, r'pythonw?(3(\.\d)?)?')
376
377
378 Python3Lexer = PythonLexer
379
380
381 class Python2Lexer(RegexLexer):
382 """
383 For `Python 2.x <http://www.python.org>`_ source code.
384
385 .. versionchanged:: 2.5
386 This class has been renamed from ``PythonLexer``. ``PythonLexer`` now
387 refers to the Python 3 variant. File name patterns like ``*.py`` have
388 been moved to Python 3 as well.
389 """
390
391 name = 'Python 2.x'
392 aliases = ['python2', 'py2']
393 filenames = [] # now taken over by PythonLexer (3.x)
394 mimetypes = ['text/x-python2', 'application/x-python2']
37 395
38 def innerstring_rules(ttype): 396 def innerstring_rules(ttype):
39 return [ 397 return [
40 # the old style '%s' % (...) string formatting 398 # the old style '%s' % (...) string formatting
41 (r'%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?' 399 (r'%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?'
122 'ArithmeticError', 'AssertionError', 'AttributeError', 480 'ArithmeticError', 'AssertionError', 'AttributeError',
123 'BaseException', 'DeprecationWarning', 'EOFError', 'EnvironmentError', 481 'BaseException', 'DeprecationWarning', 'EOFError', 'EnvironmentError',
124 'Exception', 'FloatingPointError', 'FutureWarning', 'GeneratorExit', 482 'Exception', 'FloatingPointError', 'FutureWarning', 'GeneratorExit',
125 'IOError', 'ImportError', 'ImportWarning', 'IndentationError', 483 'IOError', 'ImportError', 'ImportWarning', 'IndentationError',
126 'IndexError', 'KeyError', 'KeyboardInterrupt', 'LookupError', 484 'IndexError', 'KeyError', 'KeyboardInterrupt', 'LookupError',
127 'MemoryError', 'ModuleNotFoundError', 'NameError', 'NotImplemented', 'NotImplementedError', 485 'MemoryError', 'NameError',
128 'OSError', 'OverflowError', 'OverflowWarning', 'PendingDeprecationWarning', 486 'NotImplementedError', 'OSError', 'OverflowError', 'OverflowWarning',
129 'RecursionError', 'ReferenceError', 'RuntimeError', 'RuntimeWarning', 'StandardError', 487 'PendingDeprecationWarning', 'ReferenceError',
130 'StopIteration', 'StopAsyncIteration', 'SyntaxError', 'SyntaxWarning', 'SystemError', 488 'RuntimeError', 'RuntimeWarning', 'StandardError', 'StopIteration',
131 'SystemExit', 'TabError', 'TypeError', 'UnboundLocalError', 489 'SyntaxError', 'SyntaxWarning', 'SystemError', 'SystemExit',
132 'UnicodeDecodeError', 'UnicodeEncodeError', 'UnicodeError', 490 'TabError', 'TypeError', 'UnboundLocalError', 'UnicodeDecodeError',
133 'UnicodeTranslateError', 'UnicodeWarning', 'UserWarning', 491 'UnicodeEncodeError', 'UnicodeError', 'UnicodeTranslateError',
134 'ValueError', 'VMSError', 'Warning', 'WindowsError', 492 'UnicodeWarning', 'UserWarning', 'ValueError', 'VMSError', 'Warning',
135 'ZeroDivisionError'), prefix=r'(?<!\.)', suffix=r'\b'), 493 'WindowsError', 'ZeroDivisionError'), prefix=r'(?<!\.)', suffix=r'\b'),
136 Name.Exception), 494 Name.Exception),
137 ], 495 ],
138 'magicfuncs': [ 496 'magicfuncs': [
139 (words(( 497 (words((
140 '__abs__', '__add__', '__and__', '__call__', '__cmp__', '__coerce__', 498 '__abs__', '__add__', '__and__', '__call__', '__cmp__', '__coerce__',
236 (r'\n', String.Single) 594 (r'\n', String.Single)
237 ], 595 ],
238 } 596 }
239 597
240 def analyse_text(text): 598 def analyse_text(text):
241 return shebang_matches(text, r'pythonw?(2(\.\d)?)?') or \ 599 return shebang_matches(text, r'pythonw?2(\.\d)?') or \
242 'import ' in text[:1000] 600 'import ' in text[:1000]
243
244
245 class Python3Lexer(RegexLexer):
246 """
247 For `Python <http://www.python.org>`_ source code (version 3.0).
248
249 .. versionadded:: 0.10
250 """
251
252 name = 'Python 3'
253 aliases = ['python3', 'py3']
254 filenames = [] # Nothing until Python 3 gets widespread
255 mimetypes = ['text/x-python3', 'application/x-python3']
256
257 flags = re.MULTILINE | re.UNICODE
258
259 uni_name = "[%s][%s]*" % (uni.xid_start, uni.xid_continue)
260
261 def innerstring_rules(ttype):
262 return [
263 # the old style '%s' % (...) string formatting (still valid in Py3)
264 (r'%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?'
265 '[hlL]?[E-GXc-giorsaux%]', String.Interpol),
266 # the new style '{}'.format(...) string formatting
267 (r'\{'
268 r'((\w+)((\.\w+)|(\[[^\]]+\]))*)?' # field name
269 r'(\![sra])?' # conversion
270 r'(\:(.?[<>=\^])?[-+ ]?#?0?(\d+)?,?(\.\d+)?[E-GXb-gnosx%]?)?'
271 r'\}', String.Interpol),
272
273 # backslashes, quotes and formatting signs must be parsed one at a time
274 (r'[^\\\'"%{\n]+', ttype),
275 (r'[\'"\\]', ttype),
276 # unhandled string formatting sign
277 (r'%|(\{{1,2})', ttype)
278 # newlines are an error (use "nl" state)
279 ]
280
281 tokens = PythonLexer.tokens.copy()
282 tokens['keywords'] = [
283 (words((
284 'assert', 'async', 'await', 'break', 'continue', 'del', 'elif',
285 'else', 'except', 'finally', 'for', 'global', 'if', 'lambda', 'pass',
286 'raise', 'nonlocal', 'return', 'try', 'while', 'yield', 'yield from',
287 'as', 'with'), suffix=r'\b'),
288 Keyword),
289 (words((
290 'True', 'False', 'None'), suffix=r'\b'),
291 Keyword.Constant),
292 ]
293 tokens['builtins'] = [
294 (words((
295 '__import__', 'abs', 'all', 'any', 'bin', 'bool', 'bytearray', 'bytes',
296 'chr', 'classmethod', 'cmp', 'compile', 'complex', 'delattr', 'dict',
297 'dir', 'divmod', 'enumerate', 'eval', 'filter', 'float', 'format',
298 'frozenset', 'getattr', 'globals', 'hasattr', 'hash', 'hex', 'id',
299 'input', 'int', 'isinstance', 'issubclass', 'iter', 'len', 'list',
300 'locals', 'map', 'max', 'memoryview', 'min', 'next', 'object', 'oct',
301 'open', 'ord', 'pow', 'print', 'property', 'range', 'repr', 'reversed',
302 'round', 'set', 'setattr', 'slice', 'sorted', 'staticmethod', 'str',
303 'sum', 'super', 'tuple', 'type', 'vars', 'zip'), prefix=r'(?<!\.)',
304 suffix=r'\b'),
305 Name.Builtin),
306 (r'(?<!\.)(self|Ellipsis|NotImplemented|cls)\b', Name.Builtin.Pseudo),
307 (words((
308 'ArithmeticError', 'AssertionError', 'AttributeError',
309 'BaseException', 'BufferError', 'BytesWarning', 'DeprecationWarning',
310 'EOFError', 'EnvironmentError', 'Exception', 'FloatingPointError',
311 'FutureWarning', 'GeneratorExit', 'IOError', 'ImportError',
312 'ImportWarning', 'IndentationError', 'IndexError', 'KeyError',
313 'KeyboardInterrupt', 'LookupError', 'MemoryError', 'NameError',
314 'NotImplementedError', 'OSError', 'OverflowError',
315 'PendingDeprecationWarning', 'ReferenceError', 'ResourceWarning',
316 'RuntimeError', 'RuntimeWarning', 'StopIteration',
317 'SyntaxError', 'SyntaxWarning', 'SystemError', 'SystemExit', 'TabError',
318 'TypeError', 'UnboundLocalError', 'UnicodeDecodeError',
319 'UnicodeEncodeError', 'UnicodeError', 'UnicodeTranslateError',
320 'UnicodeWarning', 'UserWarning', 'ValueError', 'VMSError', 'Warning',
321 'WindowsError', 'ZeroDivisionError',
322 # new builtin exceptions from PEP 3151
323 'BlockingIOError', 'ChildProcessError', 'ConnectionError',
324 'BrokenPipeError', 'ConnectionAbortedError', 'ConnectionRefusedError',
325 'ConnectionResetError', 'FileExistsError', 'FileNotFoundError',
326 'InterruptedError', 'IsADirectoryError', 'NotADirectoryError',
327 'PermissionError', 'ProcessLookupError', 'TimeoutError'),
328 prefix=r'(?<!\.)', suffix=r'\b'),
329 Name.Exception),
330 ]
331 tokens['magicfuncs'] = [
332 (words((
333 '__abs__', '__add__', '__aenter__', '__aexit__', '__aiter__', '__and__',
334 '__anext__', '__await__', '__bool__', '__bytes__', '__call__',
335 '__complex__', '__contains__', '__del__', '__delattr__', '__delete__',
336 '__delitem__', '__dir__', '__divmod__', '__enter__', '__eq__', '__exit__',
337 '__float__', '__floordiv__', '__format__', '__ge__', '__get__',
338 '__getattr__', '__getattribute__', '__getitem__', '__gt__', '__hash__',
339 '__iadd__', '__iand__', '__ifloordiv__', '__ilshift__', '__imatmul__',
340 '__imod__', '__import__', '__imul__', '__index__', '__init__',
341 '__instancecheck__', '__int__', '__invert__', '__ior__', '__ipow__',
342 '__irshift__', '__isub__', '__iter__', '__itruediv__', '__ixor__',
343 '__le__', '__len__', '__length_hint__', '__lshift__', '__lt__',
344 '__matmul__', '__missing__', '__mod__', '__mul__', '__ne__', '__neg__',
345 '__new__', '__next__', '__or__', '__pos__', '__pow__', '__prepare__',
346 '__radd__', '__rand__', '__rdivmod__', '__repr__', '__reversed__',
347 '__rfloordiv__', '__rlshift__', '__rmatmul__', '__rmod__', '__rmul__',
348 '__ror__', '__round__', '__rpow__', '__rrshift__', '__rshift__',
349 '__rsub__', '__rtruediv__', '__rxor__', '__set__', '__setattr__',
350 '__setitem__', '__str__', '__sub__', '__subclasscheck__', '__truediv__',
351 '__xor__'), suffix=r'\b'),
352 Name.Function.Magic),
353 ]
354 tokens['magicvars'] = [
355 (words((
356 '__annotations__', '__bases__', '__class__', '__closure__', '__code__',
357 '__defaults__', '__dict__', '__doc__', '__file__', '__func__',
358 '__globals__', '__kwdefaults__', '__module__', '__mro__', '__name__',
359 '__objclass__', '__qualname__', '__self__', '__slots__', '__weakref__'),
360 suffix=r'\b'),
361 Name.Variable.Magic),
362 ]
363 tokens['numbers'] = [
364 (r'(\d(?:_?\d)*\.(?:\d(?:_?\d)*)?|(?:\d(?:_?\d)*)?\.\d(?:_?\d)*)([eE][+-]?\d(?:_?\d)*)?', Number.Float),
365 (r'\d(?:_?\d)*[eE][+-]?\d(?:_?\d)*j?', Number.Float),
366 (r'0[oO](?:_?[0-7])+', Number.Oct),
367 (r'0[bB](?:_?[01])+', Number.Bin),
368 (r'0[xX](?:_?[a-fA-F0-9])+', Number.Hex),
369 (r'\d(?:_?\d)*', Number.Integer)
370 ]
371 tokens['backtick'] = []
372 tokens['name'] = [
373 (r'@\w+', Name.Decorator),
374 (r'@', Operator), # new matrix multiplication operator
375 (uni_name, Name),
376 ]
377 tokens['funcname'] = [
378 (uni_name, Name.Function, '#pop')
379 ]
380 tokens['classname'] = [
381 (uni_name, Name.Class, '#pop')
382 ]
383 tokens['import'] = [
384 (r'(\s+)(as)(\s+)', bygroups(Text, Keyword, Text)),
385 (r'\.', Name.Namespace),
386 (uni_name, Name.Namespace),
387 (r'(\s*)(,)(\s*)', bygroups(Text, Operator, Text)),
388 default('#pop') # all else: go back
389 ]
390 tokens['fromimport'] = [
391 (r'(\s+)(import)\b', bygroups(Text, Keyword), '#pop'),
392 (r'\.', Name.Namespace),
393 (uni_name, Name.Namespace),
394 default('#pop'),
395 ]
396 tokens['strings-single'] = innerstring_rules(String.Single)
397 tokens['strings-double'] = innerstring_rules(String.Double)
398
399
400 def analyse_text(text):
401 return shebang_matches(text, r'pythonw?3(\.\d)?')
402 601
403 602
404 class PythonConsoleLexer(Lexer): 603 class PythonConsoleLexer(Lexer):
405 """ 604 """
406 For Python console output or doctests, such as: 605 For Python console output or doctests, such as:
416 ZeroDivisionError: integer division or modulo by zero 615 ZeroDivisionError: integer division or modulo by zero
417 616
418 Additional options: 617 Additional options:
419 618
420 `python3` 619 `python3`
421 Use Python 3 lexer for code. Default is ``False``. 620 Use Python 3 lexer for code. Default is ``True``.
422 621
423 .. versionadded:: 1.0 622 .. versionadded:: 1.0
623 .. versionchanged:: 2.5
624 Now defaults to ``True``.
424 """ 625 """
425 name = 'Python console session' 626 name = 'Python console session'
426 aliases = ['pycon'] 627 aliases = ['pycon']
427 mimetypes = ['text/x-python-doctest'] 628 mimetypes = ['text/x-python-doctest']
428 629
429 def __init__(self, **options): 630 def __init__(self, **options):
430 self.python3 = get_bool_opt(options, 'python3', False) 631 self.python3 = get_bool_opt(options, 'python3', True)
431 Lexer.__init__(self, **options) 632 Lexer.__init__(self, **options)
432 633
433 def get_tokens_unprocessed(self, text): 634 def get_tokens_unprocessed(self, text):
434 if self.python3: 635 if self.python3:
435 pylexer = Python3Lexer(**self.options)
436 tblexer = Python3TracebackLexer(**self.options)
437 else:
438 pylexer = PythonLexer(**self.options) 636 pylexer = PythonLexer(**self.options)
439 tblexer = PythonTracebackLexer(**self.options) 637 tblexer = PythonTracebackLexer(**self.options)
638 else:
639 pylexer = Python2Lexer(**self.options)
640 tblexer = Python2TracebackLexer(**self.options)
440 641
441 curcode = '' 642 curcode = ''
442 insertions = [] 643 insertions = []
443 curtb = '' 644 curtb = ''
444 tbindex = 0 645 tbindex = 0
489 yield tbindex+i, t, v 690 yield tbindex+i, t, v
490 691
491 692
492 class PythonTracebackLexer(RegexLexer): 693 class PythonTracebackLexer(RegexLexer):
493 """ 694 """
494 For Python tracebacks. 695 For Python 3.x tracebacks, with support for chained exceptions.
495 696
496 .. versionadded:: 0.7 697 .. versionadded:: 1.0
698
699 .. versionchanged:: 2.5
700 This is now the default ``PythonTracebackLexer``. It is still available
701 as the alias ``Python3TracebackLexer``.
497 """ 702 """
498 703
499 name = 'Python Traceback' 704 name = 'Python Traceback'
500 aliases = ['pytb'] 705 aliases = ['pytb', 'py3tb']
501 filenames = ['*.pytb'] 706 filenames = ['*.pytb', '*.py3tb']
502 mimetypes = ['text/x-python-traceback'] 707 mimetypes = ['text/x-python-traceback', 'text/x-python3-traceback']
503 708
504 tokens = { 709 tokens = {
505 'root': [ 710 'root': [
506 # Cover both (most recent call last) and (innermost last) 711 (r'\n', Text),
507 # The optional ^C allows us to catch keyboard interrupt signals. 712 (r'^Traceback \(most recent call last\):\n', Generic.Traceback, 'intb'),
508 (r'^(\^C)?(Traceback.*\n)', 713 (r'^During handling of the above exception, another '
509 bygroups(Text, Generic.Traceback), 'intb'), 714 r'exception occurred:\n\n', Generic.Traceback),
510 # SyntaxError starts with this. 715 (r'^The above exception was the direct cause of the '
716 r'following exception:\n\n', Generic.Traceback),
511 (r'^(?= File "[^"]+", line \d+)', Generic.Traceback, 'intb'), 717 (r'^(?= File "[^"]+", line \d+)', Generic.Traceback, 'intb'),
512 (r'^.*\n', Other), 718 (r'^.*\n', Other),
513 ], 719 ],
514 'intb': [ 720 'intb': [
515 (r'^( File )("[^"]+")(, line )(\d+)(, in )(.+)(\n)', 721 (r'^( File )("[^"]+")(, line )(\d+)(, in )(.+)(\n)',
526 bygroups(Generic.Error, Text), '#pop') 732 bygroups(Generic.Error, Text), '#pop')
527 ], 733 ],
528 } 734 }
529 735
530 736
531 class Python3TracebackLexer(RegexLexer): 737 Python3TracebackLexer = PythonTracebackLexer
532 """ 738
533 For Python 3.0 tracebacks, with support for chained exceptions. 739
534 740 class Python2TracebackLexer(RegexLexer):
535 .. versionadded:: 1.0 741 """
536 """ 742 For Python tracebacks.
537 743
538 name = 'Python 3.0 Traceback' 744 .. versionadded:: 0.7
539 aliases = ['py3tb'] 745
540 filenames = ['*.py3tb'] 746 .. versionchanged:: 2.5
541 mimetypes = ['text/x-python3-traceback'] 747 This class has been renamed from ``PythonTracebackLexer``.
748 ``PythonTracebackLexer`` now refers to the Python 3 variant.
749 """
750
751 name = 'Python 2.x Traceback'
752 aliases = ['py2tb']
753 filenames = ['*.py2tb']
754 mimetypes = ['text/x-python2-traceback']
542 755
543 tokens = { 756 tokens = {
544 'root': [ 757 'root': [
545 (r'\n', Text), 758 # Cover both (most recent call last) and (innermost last)
546 (r'^Traceback \(most recent call last\):\n', Generic.Traceback, 'intb'), 759 # The optional ^C allows us to catch keyboard interrupt signals.
547 (r'^During handling of the above exception, another ' 760 (r'^(\^C)?(Traceback.*\n)',
548 r'exception occurred:\n\n', Generic.Traceback), 761 bygroups(Text, Generic.Traceback), 'intb'),
549 (r'^The above exception was the direct cause of the ' 762 # SyntaxError starts with this.
550 r'following exception:\n\n', Generic.Traceback),
551 (r'^(?= File "[^"]+", line \d+)', Generic.Traceback, 'intb'), 763 (r'^(?= File "[^"]+", line \d+)', Generic.Traceback, 'intb'),
764 (r'^.*\n', Other),
552 ], 765 ],
553 'intb': [ 766 'intb': [
554 (r'^( File )("[^"]+")(, line )(\d+)(, in )(.+)(\n)', 767 (r'^( File )("[^"]+")(, line )(\d+)(, in )(.+)(\n)',
555 bygroups(Text, Name.Builtin, Text, Number, Text, Name, Text)), 768 bygroups(Text, Name.Builtin, Text, Number, Text, Name, Text)),
556 (r'^( File )("[^"]+")(, line )(\d+)(\n)', 769 (r'^( File )("[^"]+")(, line )(\d+)(\n)',
557 bygroups(Text, Name.Builtin, Text, Number, Text)), 770 bygroups(Text, Name.Builtin, Text, Number, Text)),
558 (r'^( )(.+)(\n)', 771 (r'^( )(.+)(\n)',
559 bygroups(Text, using(Python3Lexer), Text)), 772 bygroups(Text, using(Python2Lexer), Text)),
560 (r'^([ \t]*)(\.\.\.)(\n)', 773 (r'^([ \t]*)(\.\.\.)(\n)',
561 bygroups(Text, Comment, Text)), # for doctests... 774 bygroups(Text, Comment, Text)), # for doctests...
562 (r'^([^:]+)(: )(.+)(\n)', 775 (r'^([^:]+)(: )(.+)(\n)',
563 bygroups(Generic.Error, Text, Name, Text), '#pop'), 776 bygroups(Generic.Error, Text, Name, Text), '#pop'),
564 (r'^([a-zA-Z_]\w*)(:?\n)', 777 (r'^([a-zA-Z_]\w*)(:?\n)',
857 1070
858 # override the mimetypes to not inherit them from python 1071 # override the mimetypes to not inherit them from python
859 mimetypes = [] 1072 mimetypes = []
860 filenames = [] 1073 filenames = []
861 1074
862 EXTRA_KEYWORDS = set(( 1075 EXTRA_KEYWORDS = {
863 'abs', 'absolute', 'accumulate', 'add', 'alen', 'all', 'allclose', 1076 'abs', 'absolute', 'accumulate', 'add', 'alen', 'all', 'allclose',
864 'alltrue', 'alterdot', 'amax', 'amin', 'angle', 'any', 'append', 1077 'alltrue', 'alterdot', 'amax', 'amin', 'angle', 'any', 'append',
865 'apply_along_axis', 'apply_over_axes', 'arange', 'arccos', 'arccosh', 1078 'apply_along_axis', 'apply_over_axes', 'arange', 'arccos', 'arccosh',
866 'arcsin', 'arcsinh', 'arctan', 'arctan2', 'arctanh', 'argmax', 'argmin', 1079 'arcsin', 'arcsinh', 'arctan', 'arctan2', 'arctanh', 'argmax', 'argmin',
867 'argsort', 'argwhere', 'around', 'array', 'array2string', 'array_equal', 1080 'argsort', 'argwhere', 'around', 'array', 'array2string', 'array_equal',
922 'test', 'tile', 'tofile', 'tolist', 'tostring', 'trace', 'transpose', 1135 'test', 'tile', 'tofile', 'tolist', 'tostring', 'trace', 'transpose',
923 'trapz', 'tri', 'tril', 'trim_zeros', 'triu', 'true_divide', 'typeDict', 1136 'trapz', 'tri', 'tril', 'trim_zeros', 'triu', 'true_divide', 'typeDict',
924 'typename', 'uniform', 'union1d', 'unique', 'unique1d', 'unravel_index', 1137 'typename', 'uniform', 'union1d', 'unique', 'unique1d', 'unravel_index',
925 'unwrap', 'vander', 'var', 'vdot', 'vectorize', 'view', 'vonmises', 1138 'unwrap', 'vander', 'var', 'vdot', 'vectorize', 'view', 'vonmises',
926 'vsplit', 'vstack', 'weibull', 'where', 'who', 'zeros', 'zeros_like' 1139 'vsplit', 'vstack', 'weibull', 'where', 'who', 'zeros', 'zeros_like'
927 )) 1140 }
928 1141
929 def get_tokens_unprocessed(self, text): 1142 def get_tokens_unprocessed(self, text):
930 for index, token, value in \ 1143 for index, token, value in \
931 PythonLexer.get_tokens_unprocessed(self, text): 1144 PythonLexer.get_tokens_unprocessed(self, text):
932 if token is Name and value in self.EXTRA_KEYWORDS: 1145 if token is Name and value in self.EXTRA_KEYWORDS:
933 yield index, Keyword.Pseudo, value 1146 yield index, Keyword.Pseudo, value
934 else: 1147 else:
935 yield index, token, value 1148 yield index, token, value
936 1149
937 def analyse_text(text): 1150 def analyse_text(text):
938 return (shebang_matches(text, r'pythonw?(2(\.\d)?)?') or 1151 return (shebang_matches(text, r'pythonw?(3(\.\d)?)?') or
939 'import ' in text[:1000]) \ 1152 'import ' in text[:1000]) \
940 and ('import numpy' in text or 'from numpy import' in text) 1153 and ('import numpy' in text or 'from numpy import' in text)

Mercurial Repository: eric

eric ide

mercurial