Mercurial Repositories > eric / file comparison
comparison: eric6/ThirdParty/Pygments/pygments/lexers/python.py
eric6/ThirdParty/Pygments/pygments/lexers/python.py
- changeset 8258
- 82b608e352ec
- parent 8257
- 28146736bbfc
- child 8259
- 2bbec88047dd
equal
deleted
inserted
replaced
| 8257:28146736bbfc | 8258:82b608e352ec |
|---|---|
| 1 # -*- coding: utf-8 -*- | |
| 2 """ | |
| 3 pygments.lexers.python | |
| 4 ~~~~~~~~~~~~~~~~~~~~~~ | |
| 5 | |
| 6 Lexers for Python and related languages. | |
| 7 | |
| 8 :copyright: Copyright 2006-2021 by the Pygments team, see AUTHORS. | |
| 9 :license: BSD, see LICENSE for details. | |
| 10 """ | |
| 11 | |
| 12 import re | |
| 13 | |
| 14 from pygments.lexer import Lexer, RegexLexer, include, bygroups, using, \ | |
| 15 default, words, combined, do_insertions | |
| 16 from pygments.util import get_bool_opt, shebang_matches | |
| 17 from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ | |
| 18 Number, Punctuation, Generic, Other, Error | |
| 19 from pygments import unistring as uni | |
| 20 | |
| 21 __all__ = ['PythonLexer', 'PythonConsoleLexer', 'PythonTracebackLexer', | |
| 22 'Python2Lexer', 'Python2TracebackLexer', | |
| 23 'CythonLexer', 'DgLexer', 'NumPyLexer'] | |
| 24 | |
| 25 line_re = re.compile('.*?\n') | |
| 26 | |
| 27 | |
| 28 class PythonLexer(RegexLexer): | |
| 29 """ | |
| 30 For `Python <http://www.python.org>`_ source code (version 3.x). | |
| 31 | |
| 32 .. versionadded:: 0.10 | |
| 33 | |
| 34 .. versionchanged:: 2.5 | |
| 35 This is now the default ``PythonLexer``. It is still available as the | |
| 36 alias ``Python3Lexer``. | |
| 37 """ | |
| 38 | |
| 39 name = 'Python' | |
| 40 aliases = ['python', 'py', 'sage', 'python3', 'py3'] | |
| 41 filenames = [ | |
| 42 '*.py', | |
| 43 '*.pyw', | |
| 44 # Jython | |
| 45 '*.jy', | |
| 46 # Sage | |
| 47 '*.sage', | |
| 48 # SCons | |
| 49 '*.sc', | |
| 50 'SConstruct', | |
| 51 'SConscript', | |
| 52 # Skylark/Starlark (used by Bazel, Buck, and Pants) | |
| 53 '*.bzl', | |
| 54 'BUCK', | |
| 55 'BUILD', | |
| 56 'BUILD.bazel', | |
| 57 'WORKSPACE', | |
| 58 # Twisted Application infrastructure | |
| 59 '*.tac', | |
| 60 ] | |
| 61 mimetypes = ['text/x-python', 'application/x-python', | |
| 62 'text/x-python3', 'application/x-python3'] | |
| 63 | |
| 64 flags = re.MULTILINE | re.UNICODE | |
| 65 | |
| 66 uni_name = "[%s][%s]*" % (uni.xid_start, uni.xid_continue) | |
| 67 | |
| 68 def innerstring_rules(ttype): | |
| 69 return [ | |
| 70 # the old style '%s' % (...) string formatting (still valid in Py3) | |
| 71 (r'%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?' | |
| 72 '[hlL]?[E-GXc-giorsaux%]', String.Interpol), | |
| 73 # the new style '{}'.format(...) string formatting | |
| 74 (r'\{' | |
| 75 r'((\w+)((\.\w+)|(\[[^\]]+\]))*)?' # field name | |
| 76 r'(\![sra])?' # conversion | |
| 77 r'(\:(.?[<>=\^])?[-+ ]?#?0?(\d+)?,?(\.\d+)?[E-GXb-gnosx%]?)?' | |
| 78 r'\}', String.Interpol), | |
| 79 | |
| 80 # backslashes, quotes and formatting signs must be parsed one at a time | |
| 81 (r'[^\\\'"%{\n]+', ttype), | |
| 82 (r'[\'"\\]', ttype), | |
| 83 # unhandled string formatting sign | |
| 84 (r'%|(\{{1,2})', ttype) | |
| 85 # newlines are an error (use "nl" state) | |
| 86 ] | |
| 87 | |
| 88 def fstring_rules(ttype): | |
| 89 return [ | |
| 90 # Assuming that a '}' is the closing brace after format specifier. | |
| 91 # Sadly, this means that we won't detect syntax error. But it's | |
| 92 # more important to parse correct syntax correctly, than to | |
| 93 # highlight invalid syntax. | |
| 94 (r'\}', String.Interpol), | |
| 95 (r'\{', String.Interpol, 'expr-inside-fstring'), | |
| 96 # backslashes, quotes and formatting signs must be parsed one at a time | |
| 97 (r'[^\\\'"{}\n]+', ttype), | |
| 98 (r'[\'"\\]', ttype), | |
| 99 # newlines are an error (use "nl" state) | |
| 100 ] | |
| 101 | |
| 102 tokens = { | |
| 103 'root': [ | |
| 104 (r'\n', Text), | |
| 105 (r'^(\s*)([rRuUbB]{,2})("""(?:.|\n)*?""")', | |
| 106 bygroups(Text, String.Affix, String.Doc)), | |
| 107 (r"^(\s*)([rRuUbB]{,2})('''(?:.|\n)*?''')", | |
| 108 bygroups(Text, String.Affix, String.Doc)), | |
| 109 (r'\A#!.+$', Comment.Hashbang), | |
| 110 (r'#.*$', Comment.Single), | |
| 111 (r'\\\n', Text), | |
| 112 (r'\\', Text), | |
| 113 include('keywords'), | |
| 114 (r'(def)((?:\s|\\\s)+)', bygroups(Keyword, Text), 'funcname'), | |
| 115 (r'(class)((?:\s|\\\s)+)', bygroups(Keyword, Text), 'classname'), | |
| 116 (r'(from)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Text), | |
| 117 'fromimport'), | |
| 118 (r'(import)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Text), | |
| 119 'import'), | |
| 120 include('expr'), | |
| 121 ], | |
| 122 'expr': [ | |
| 123 # raw f-strings | |
| 124 ('(?i)(rf|fr)(""")', | |
| 125 bygroups(String.Affix, String.Double), 'tdqf'), | |
| 126 ("(?i)(rf|fr)(''')", | |
| 127 bygroups(String.Affix, String.Single), 'tsqf'), | |
| 128 ('(?i)(rf|fr)(")', | |
| 129 bygroups(String.Affix, String.Double), 'dqf'), | |
| 130 ("(?i)(rf|fr)(')", | |
| 131 bygroups(String.Affix, String.Single), 'sqf'), | |
| 132 # non-raw f-strings | |
| 133 ('([fF])(""")', bygroups(String.Affix, String.Double), | |
| 134 combined('fstringescape', 'tdqf')), | |
| 135 ("([fF])(''')", bygroups(String.Affix, String.Single), | |
| 136 combined('fstringescape', 'tsqf')), | |
| 137 ('([fF])(")', bygroups(String.Affix, String.Double), | |
| 138 combined('fstringescape', 'dqf')), | |
| 139 ("([fF])(')", bygroups(String.Affix, String.Single), | |
| 140 combined('fstringescape', 'sqf')), | |
| 141 # raw strings | |
| 142 ('(?i)(rb|br|r)(""")', | |
| 143 bygroups(String.Affix, String.Double), 'tdqs'), | |
| 144 ("(?i)(rb|br|r)(''')", | |
| 145 bygroups(String.Affix, String.Single), 'tsqs'), | |
| 146 ('(?i)(rb|br|r)(")', | |
| 147 bygroups(String.Affix, String.Double), 'dqs'), | |
| 148 ("(?i)(rb|br|r)(')", | |
| 149 bygroups(String.Affix, String.Single), 'sqs'), | |
| 150 # non-raw strings | |
| 151 ('([uUbB]?)(""")', bygroups(String.Affix, String.Double), | |
| 152 combined('stringescape', 'tdqs')), | |
| 153 ("([uUbB]?)(''')", bygroups(String.Affix, String.Single), | |
| 154 combined('stringescape', 'tsqs')), | |
| 155 ('([uUbB]?)(")', bygroups(String.Affix, String.Double), | |
| 156 combined('stringescape', 'dqs')), | |
| 157 ("([uUbB]?)(')", bygroups(String.Affix, String.Single), | |
| 158 combined('stringescape', 'sqs')), | |
| 159 (r'[^\S\n]+', Text), | |
| 160 (r'!=|==|<<|>>|:=|[-~+/*%=<>&^|.]', Operator), | |
| 161 (r'[]{}:(),;[]', Punctuation), | |
| 162 (r'(in|is|and|or|not)\b', Operator.Word), | |
| 163 include('expr-keywords'), | |
| 164 include('builtins'), | |
| 165 include('magicfuncs'), | |
| 166 include('magicvars'), | |
| 167 include('name'), | |
| 168 include('numbers'), | |
| 169 ], | |
| 170 'expr-inside-fstring': [ | |
| 171 (r'[{([]', Punctuation, 'expr-inside-fstring-inner'), | |
| 172 # without format specifier | |
| 173 (r'(=\s*)?' # debug (https://bugs.python.org/issue36817) | |
| 174 r'(\![sraf])?' # conversion | |
| 175 r'\}', String.Interpol, '#pop'), | |
| 176 # with format specifier | |
| 177 # we'll catch the remaining '}' in the outer scope | |
| 178 (r'(=\s*)?' # debug (https://bugs.python.org/issue36817) | |
| 179 r'(\![sraf])?' # conversion | |
| 180 r':', String.Interpol, '#pop'), | |
| 181 (r'\s+', Text), # allow new lines | |
| 182 include('expr'), | |
| 183 ], | |
| 184 'expr-inside-fstring-inner': [ | |
| 185 (r'[{([]', Punctuation, 'expr-inside-fstring-inner'), | |
| 186 (r'[])}]', Punctuation, '#pop'), | |
| 187 (r'\s+', Text), # allow new lines | |
| 188 include('expr'), | |
| 189 ], | |
| 190 'expr-keywords': [ | |
| 191 # Based on https://docs.python.org/3/reference/expressions.html | |
| 192 (words(( | |
| 193 'async for', 'await', 'else', 'for', 'if', 'lambda', | |
| 194 'yield', 'yield from'), suffix=r'\b'), | |
| 195 Keyword), | |
| 196 (words(('True', 'False', 'None'), suffix=r'\b'), Keyword.Constant), | |
| 197 ], | |
| 198 'keywords': [ | |
| 199 (words(( | |
| 200 'assert', 'async', 'await', 'break', 'continue', 'del', 'elif', | |
| 201 'else', 'except', 'finally', 'for', 'global', 'if', 'lambda', | |
| 202 'pass', 'raise', 'nonlocal', 'return', 'try', 'while', 'yield', | |
| 203 'yield from', 'as', 'with'), suffix=r'\b'), | |
| 204 Keyword), | |
| 205 (words(('True', 'False', 'None'), suffix=r'\b'), Keyword.Constant), | |
| 206 ], | |
| 207 'builtins': [ | |
| 208 (words(( | |
| 209 '__import__', 'abs', 'all', 'any', 'bin', 'bool', 'bytearray', | |
| 210 'bytes', 'chr', 'classmethod', 'compile', 'complex', | |
| 211 'delattr', 'dict', 'dir', 'divmod', 'enumerate', 'eval', 'filter', | |
| 212 'float', 'format', 'frozenset', 'getattr', 'globals', 'hasattr', | |
| 213 'hash', 'hex', 'id', 'input', 'int', 'isinstance', 'issubclass', | |
| 214 'iter', 'len', 'list', 'locals', 'map', 'max', 'memoryview', | |
| 215 'min', 'next', 'object', 'oct', 'open', 'ord', 'pow', 'print', | |
| 216 'property', 'range', 'repr', 'reversed', 'round', 'set', 'setattr', | |
| 217 'slice', 'sorted', 'staticmethod', 'str', 'sum', 'super', 'tuple', | |
| 218 'type', 'vars', 'zip'), prefix=r'(?<!\.)', suffix=r'\b'), | |
| 219 Name.Builtin), | |
| 220 (r'(?<!\.)(self|Ellipsis|NotImplemented|cls)\b', Name.Builtin.Pseudo), | |
| 221 (words(( | |
| 222 'ArithmeticError', 'AssertionError', 'AttributeError', | |
| 223 'BaseException', 'BufferError', 'BytesWarning', 'DeprecationWarning', | |
| 224 'EOFError', 'EnvironmentError', 'Exception', 'FloatingPointError', | |
| 225 'FutureWarning', 'GeneratorExit', 'IOError', 'ImportError', | |
| 226 'ImportWarning', 'IndentationError', 'IndexError', 'KeyError', | |
| 227 'KeyboardInterrupt', 'LookupError', 'MemoryError', 'NameError', | |
| 228 'NotImplementedError', 'OSError', 'OverflowError', | |
| 229 'PendingDeprecationWarning', 'ReferenceError', 'ResourceWarning', | |
| 230 'RuntimeError', 'RuntimeWarning', 'StopIteration', | |
| 231 'SyntaxError', 'SyntaxWarning', 'SystemError', 'SystemExit', | |
| 232 'TabError', 'TypeError', 'UnboundLocalError', 'UnicodeDecodeError', | |
| 233 'UnicodeEncodeError', 'UnicodeError', 'UnicodeTranslateError', | |
| 234 'UnicodeWarning', 'UserWarning', 'ValueError', 'VMSError', | |
| 235 'Warning', 'WindowsError', 'ZeroDivisionError', | |
| 236 # new builtin exceptions from PEP 3151 | |
| 237 'BlockingIOError', 'ChildProcessError', 'ConnectionError', | |
| 238 'BrokenPipeError', 'ConnectionAbortedError', 'ConnectionRefusedError', | |
| 239 'ConnectionResetError', 'FileExistsError', 'FileNotFoundError', | |
| 240 'InterruptedError', 'IsADirectoryError', 'NotADirectoryError', | |
| 241 'PermissionError', 'ProcessLookupError', 'TimeoutError', | |
| 242 # others new in Python 3 | |
| 243 'StopAsyncIteration', 'ModuleNotFoundError', 'RecursionError'), | |
| 244 prefix=r'(?<!\.)', suffix=r'\b'), | |
| 245 Name.Exception), | |
| 246 ], | |
| 247 'magicfuncs': [ | |
| 248 (words(( | |
| 249 '__abs__', '__add__', '__aenter__', '__aexit__', '__aiter__', | |
| 250 '__and__', '__anext__', '__await__', '__bool__', '__bytes__', | |
| 251 '__call__', '__complex__', '__contains__', '__del__', '__delattr__', | |
| 252 '__delete__', '__delitem__', '__dir__', '__divmod__', '__enter__', | |
| 253 '__eq__', '__exit__', '__float__', '__floordiv__', '__format__', | |
| 254 '__ge__', '__get__', '__getattr__', '__getattribute__', | |
| 255 '__getitem__', '__gt__', '__hash__', '__iadd__', '__iand__', | |
| 256 '__ifloordiv__', '__ilshift__', '__imatmul__', '__imod__', | |
| 257 '__imul__', '__index__', '__init__', '__instancecheck__', | |
| 258 '__int__', '__invert__', '__ior__', '__ipow__', '__irshift__', | |
| 259 '__isub__', '__iter__', '__itruediv__', '__ixor__', '__le__', | |
| 260 '__len__', '__length_hint__', '__lshift__', '__lt__', '__matmul__', | |
| 261 '__missing__', '__mod__', '__mul__', '__ne__', '__neg__', | |
| 262 '__new__', '__next__', '__or__', '__pos__', '__pow__', | |
| 263 '__prepare__', '__radd__', '__rand__', '__rdivmod__', '__repr__', | |
| 264 '__reversed__', '__rfloordiv__', '__rlshift__', '__rmatmul__', | |
| 265 '__rmod__', '__rmul__', '__ror__', '__round__', '__rpow__', | |
| 266 '__rrshift__', '__rshift__', '__rsub__', '__rtruediv__', | |
| 267 '__rxor__', '__set__', '__setattr__', '__setitem__', '__str__', | |
| 268 '__sub__', '__subclasscheck__', '__truediv__', | |
| 269 '__xor__'), suffix=r'\b'), | |
| 270 Name.Function.Magic), | |
| 271 ], | |
| 272 'magicvars': [ | |
| 273 (words(( | |
| 274 '__annotations__', '__bases__', '__class__', '__closure__', | |
| 275 '__code__', '__defaults__', '__dict__', '__doc__', '__file__', | |
| 276 '__func__', '__globals__', '__kwdefaults__', '__module__', | |
| 277 '__mro__', '__name__', '__objclass__', '__qualname__', | |
| 278 '__self__', '__slots__', '__weakref__'), suffix=r'\b'), | |
| 279 Name.Variable.Magic), | |
| 280 ], | |
| 281 'numbers': [ | |
| 282 (r'(\d(?:_?\d)*\.(?:\d(?:_?\d)*)?|(?:\d(?:_?\d)*)?\.\d(?:_?\d)*)' | |
| 283 r'([eE][+-]?\d(?:_?\d)*)?', Number.Float), | |
| 284 (r'\d(?:_?\d)*[eE][+-]?\d(?:_?\d)*j?', Number.Float), | |
| 285 (r'0[oO](?:_?[0-7])+', Number.Oct), | |
| 286 (r'0[bB](?:_?[01])+', Number.Bin), | |
| 287 (r'0[xX](?:_?[a-fA-F0-9])+', Number.Hex), | |
| 288 (r'\d(?:_?\d)*', Number.Integer), | |
| 289 ], | |
| 290 'name': [ | |
| 291 (r'@' + uni_name, Name.Decorator), | |
| 292 (r'@', Operator), # new matrix multiplication operator | |
| 293 (uni_name, Name), | |
| 294 ], | |
| 295 'funcname': [ | |
| 296 include('magicfuncs'), | |
| 297 (uni_name, Name.Function, '#pop'), | |
| 298 default('#pop'), | |
| 299 ], | |
| 300 'classname': [ | |
| 301 (uni_name, Name.Class, '#pop'), | |
| 302 ], | |
| 303 'import': [ | |
| 304 (r'(\s+)(as)(\s+)', bygroups(Text, Keyword, Text)), | |
| 305 (r'\.', Name.Namespace), | |
| 306 (uni_name, Name.Namespace), | |
| 307 (r'(\s*)(,)(\s*)', bygroups(Text, Operator, Text)), | |
| 308 default('#pop') # all else: go back | |
| 309 ], | |
| 310 'fromimport': [ | |
| 311 (r'(\s+)(import)\b', bygroups(Text, Keyword.Namespace), '#pop'), | |
| 312 (r'\.', Name.Namespace), | |
| 313 # if None occurs here, it's "raise x from None", since None can | |
| 314 # never be a module name | |
| 315 (r'None\b', Name.Builtin.Pseudo, '#pop'), | |
| 316 (uni_name, Name.Namespace), | |
| 317 default('#pop'), | |
| 318 ], | |
| 319 'fstringescape': [ | |
| 320 (r'\{\{', String.Escape), | |
| 321 (r'\}\}', String.Escape), | |
| 322 include('stringescape'), | |
| 323 ], | |
| 324 'stringescape': [ | |
| 325 (r'\\([\\abfnrtv"\']|\n|N\{.*?\}|u[a-fA-F0-9]{4}|' | |
| 326 r'U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape) | |
| 327 ], | |
| 328 'fstrings-single': fstring_rules(String.Single), | |
| 329 'fstrings-double': fstring_rules(String.Double), | |
| 330 'strings-single': innerstring_rules(String.Single), | |
| 331 'strings-double': innerstring_rules(String.Double), | |
| 332 'dqf': [ | |
| 333 (r'"', String.Double, '#pop'), | |
| 334 (r'\\\\|\\"|\\\n', String.Escape), # included here for raw strings | |
| 335 include('fstrings-double') | |
| 336 ], | |
| 337 'sqf': [ | |
| 338 (r"'", String.Single, '#pop'), | |
| 339 (r"\\\\|\\'|\\\n", String.Escape), # included here for raw strings | |
| 340 include('fstrings-single') | |
| 341 ], | |
| 342 'dqs': [ | |
| 343 (r'"', String.Double, '#pop'), | |
| 344 (r'\\\\|\\"|\\\n', String.Escape), # included here for raw strings | |
| 345 include('strings-double') | |
| 346 ], | |
| 347 'sqs': [ | |
| 348 (r"'", String.Single, '#pop'), | |
| 349 (r"\\\\|\\'|\\\n", String.Escape), # included here for raw strings | |
| 350 include('strings-single') | |
| 351 ], | |
| 352 'tdqf': [ | |
| 353 (r'"""', String.Double, '#pop'), | |
| 354 include('fstrings-double'), | |
| 355 (r'\n', String.Double) | |
| 356 ], | |
| 357 'tsqf': [ | |
| 358 (r"'''", String.Single, '#pop'), | |
| 359 include('fstrings-single'), | |
| 360 (r'\n', String.Single) | |
| 361 ], | |
| 362 'tdqs': [ | |
| 363 (r'"""', String.Double, '#pop'), | |
| 364 include('strings-double'), | |
| 365 (r'\n', String.Double) | |
| 366 ], | |
| 367 'tsqs': [ | |
| 368 (r"'''", String.Single, '#pop'), | |
| 369 include('strings-single'), | |
| 370 (r'\n', String.Single) | |
| 371 ], | |
| 372 } | |
| 373 | |
| 374 def analyse_text(text): | |
| 375 return shebang_matches(text, r'pythonw?(3(\.\d)?)?') | |
| 376 | |
| 377 | |
| 378 Python3Lexer = PythonLexer | |
| 379 | |
| 380 | |
| 381 class Python2Lexer(RegexLexer): | |
| 382 """ | |
| 383 For `Python 2.x <http://www.python.org>`_ source code. | |
| 384 | |
| 385 .. versionchanged:: 2.5 | |
| 386 This class has been renamed from ``PythonLexer``. ``PythonLexer`` now | |
| 387 refers to the Python 3 variant. File name patterns like ``*.py`` have | |
| 388 been moved to Python 3 as well. | |
| 389 """ | |
| 390 | |
| 391 name = 'Python 2.x' | |
| 392 aliases = ['python2', 'py2'] | |
| 393 filenames = [] # now taken over by PythonLexer (3.x) | |
| 394 mimetypes = ['text/x-python2', 'application/x-python2'] | |
| 395 | |
| 396 def innerstring_rules(ttype): | |
| 397 return [ | |
| 398 # the old style '%s' % (...) string formatting | |
| 399 (r'%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?' | |
| 400 '[hlL]?[E-GXc-giorsux%]', String.Interpol), | |
| 401 # backslashes, quotes and formatting signs must be parsed one at a time | |
| 402 (r'[^\\\'"%\n]+', ttype), | |
| 403 (r'[\'"\\]', ttype), | |
| 404 # unhandled string formatting sign | |
| 405 (r'%', ttype), | |
| 406 # newlines are an error (use "nl" state) | |
| 407 ] | |
| 408 | |
| 409 tokens = { | |
| 410 'root': [ | |
| 411 (r'\n', Text), | |
| 412 (r'^(\s*)([rRuUbB]{,2})("""(?:.|\n)*?""")', | |
| 413 bygroups(Text, String.Affix, String.Doc)), | |
| 414 (r"^(\s*)([rRuUbB]{,2})('''(?:.|\n)*?''')", | |
| 415 bygroups(Text, String.Affix, String.Doc)), | |
| 416 (r'[^\S\n]+', Text), | |
| 417 (r'\A#!.+$', Comment.Hashbang), | |
| 418 (r'#.*$', Comment.Single), | |
| 419 (r'[]{}:(),;[]', Punctuation), | |
| 420 (r'\\\n', Text), | |
| 421 (r'\\', Text), | |
| 422 (r'(in|is|and|or|not)\b', Operator.Word), | |
| 423 (r'!=|==|<<|>>|[-~+/*%=<>&^|.]', Operator), | |
| 424 include('keywords'), | |
| 425 (r'(def)((?:\s|\\\s)+)', bygroups(Keyword, Text), 'funcname'), | |
| 426 (r'(class)((?:\s|\\\s)+)', bygroups(Keyword, Text), 'classname'), | |
| 427 (r'(from)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Text), | |
| 428 'fromimport'), | |
| 429 (r'(import)((?:\s|\\\s)+)', bygroups(Keyword.Namespace, Text), | |
| 430 'import'), | |
| 431 include('builtins'), | |
| 432 include('magicfuncs'), | |
| 433 include('magicvars'), | |
| 434 include('backtick'), | |
| 435 ('([rR]|[uUbB][rR]|[rR][uUbB])(""")', | |
| 436 bygroups(String.Affix, String.Double), 'tdqs'), | |
| 437 ("([rR]|[uUbB][rR]|[rR][uUbB])(''')", | |
| 438 bygroups(String.Affix, String.Single), 'tsqs'), | |
| 439 ('([rR]|[uUbB][rR]|[rR][uUbB])(")', | |
| 440 bygroups(String.Affix, String.Double), 'dqs'), | |
| 441 ("([rR]|[uUbB][rR]|[rR][uUbB])(')", | |
| 442 bygroups(String.Affix, String.Single), 'sqs'), | |
| 443 ('([uUbB]?)(""")', bygroups(String.Affix, String.Double), | |
| 444 combined('stringescape', 'tdqs')), | |
| 445 ("([uUbB]?)(''')", bygroups(String.Affix, String.Single), | |
| 446 combined('stringescape', 'tsqs')), | |
| 447 ('([uUbB]?)(")', bygroups(String.Affix, String.Double), | |
| 448 combined('stringescape', 'dqs')), | |
| 449 ("([uUbB]?)(')", bygroups(String.Affix, String.Single), | |
| 450 combined('stringescape', 'sqs')), | |
| 451 include('name'), | |
| 452 include('numbers'), | |
| 453 ], | |
| 454 'keywords': [ | |
| 455 (words(( | |
| 456 'assert', 'break', 'continue', 'del', 'elif', 'else', 'except', | |
| 457 'exec', 'finally', 'for', 'global', 'if', 'lambda', 'pass', | |
| 458 'print', 'raise', 'return', 'try', 'while', 'yield', | |
| 459 'yield from', 'as', 'with'), suffix=r'\b'), | |
| 460 Keyword), | |
| 461 ], | |
| 462 'builtins': [ | |
| 463 (words(( | |
| 464 '__import__', 'abs', 'all', 'any', 'apply', 'basestring', 'bin', | |
| 465 'bool', 'buffer', 'bytearray', 'bytes', 'callable', 'chr', 'classmethod', | |
| 466 'cmp', 'coerce', 'compile', 'complex', 'delattr', 'dict', 'dir', 'divmod', | |
| 467 'enumerate', 'eval', 'execfile', 'exit', 'file', 'filter', 'float', | |
| 468 'frozenset', 'getattr', 'globals', 'hasattr', 'hash', 'hex', 'id', | |
| 469 'input', 'int', 'intern', 'isinstance', 'issubclass', 'iter', 'len', | |
| 470 'list', 'locals', 'long', 'map', 'max', 'min', 'next', 'object', | |
| 471 'oct', 'open', 'ord', 'pow', 'property', 'range', 'raw_input', 'reduce', | |
| 472 'reload', 'repr', 'reversed', 'round', 'set', 'setattr', 'slice', | |
| 473 'sorted', 'staticmethod', 'str', 'sum', 'super', 'tuple', 'type', | |
| 474 'unichr', 'unicode', 'vars', 'xrange', 'zip'), | |
| 475 prefix=r'(?<!\.)', suffix=r'\b'), | |
| 476 Name.Builtin), | |
| 477 (r'(?<!\.)(self|None|Ellipsis|NotImplemented|False|True|cls' | |
| 478 r')\b', Name.Builtin.Pseudo), | |
| 479 (words(( | |
| 480 'ArithmeticError', 'AssertionError', 'AttributeError', | |
| 481 'BaseException', 'DeprecationWarning', 'EOFError', 'EnvironmentError', | |
| 482 'Exception', 'FloatingPointError', 'FutureWarning', 'GeneratorExit', | |
| 483 'IOError', 'ImportError', 'ImportWarning', 'IndentationError', | |
| 484 'IndexError', 'KeyError', 'KeyboardInterrupt', 'LookupError', | |
| 485 'MemoryError', 'NameError', | |
| 486 'NotImplementedError', 'OSError', 'OverflowError', 'OverflowWarning', | |
| 487 'PendingDeprecationWarning', 'ReferenceError', | |
| 488 'RuntimeError', 'RuntimeWarning', 'StandardError', 'StopIteration', | |
| 489 'SyntaxError', 'SyntaxWarning', 'SystemError', 'SystemExit', | |
| 490 'TabError', 'TypeError', 'UnboundLocalError', 'UnicodeDecodeError', | |
| 491 'UnicodeEncodeError', 'UnicodeError', 'UnicodeTranslateError', | |
| 492 'UnicodeWarning', 'UserWarning', 'ValueError', 'VMSError', 'Warning', | |
| 493 'WindowsError', 'ZeroDivisionError'), prefix=r'(?<!\.)', suffix=r'\b'), | |
| 494 Name.Exception), | |
| 495 ], | |
| 496 'magicfuncs': [ | |
| 497 (words(( | |
| 498 '__abs__', '__add__', '__and__', '__call__', '__cmp__', '__coerce__', | |
| 499 '__complex__', '__contains__', '__del__', '__delattr__', '__delete__', | |
| 500 '__delitem__', '__delslice__', '__div__', '__divmod__', '__enter__', | |
| 501 '__eq__', '__exit__', '__float__', '__floordiv__', '__ge__', '__get__', | |
| 502 '__getattr__', '__getattribute__', '__getitem__', '__getslice__', '__gt__', | |
| 503 '__hash__', '__hex__', '__iadd__', '__iand__', '__idiv__', '__ifloordiv__', | |
| 504 '__ilshift__', '__imod__', '__imul__', '__index__', '__init__', | |
| 505 '__instancecheck__', '__int__', '__invert__', '__iop__', '__ior__', | |
| 506 '__ipow__', '__irshift__', '__isub__', '__iter__', '__itruediv__', | |
| 507 '__ixor__', '__le__', '__len__', '__long__', '__lshift__', '__lt__', | |
| 508 '__missing__', '__mod__', '__mul__', '__ne__', '__neg__', '__new__', | |
| 509 '__nonzero__', '__oct__', '__op__', '__or__', '__pos__', '__pow__', | |
| 510 '__radd__', '__rand__', '__rcmp__', '__rdiv__', '__rdivmod__', '__repr__', | |
| 511 '__reversed__', '__rfloordiv__', '__rlshift__', '__rmod__', '__rmul__', | |
| 512 '__rop__', '__ror__', '__rpow__', '__rrshift__', '__rshift__', '__rsub__', | |
| 513 '__rtruediv__', '__rxor__', '__set__', '__setattr__', '__setitem__', | |
| 514 '__setslice__', '__str__', '__sub__', '__subclasscheck__', '__truediv__', | |
| 515 '__unicode__', '__xor__'), suffix=r'\b'), | |
| 516 Name.Function.Magic), | |
| 517 ], | |
| 518 'magicvars': [ | |
| 519 (words(( | |
| 520 '__bases__', '__class__', '__closure__', '__code__', '__defaults__', | |
| 521 '__dict__', '__doc__', '__file__', '__func__', '__globals__', | |
| 522 '__metaclass__', '__module__', '__mro__', '__name__', '__self__', | |
| 523 '__slots__', '__weakref__'), | |
| 524 suffix=r'\b'), | |
| 525 Name.Variable.Magic), | |
| 526 ], | |
| 527 'numbers': [ | |
| 528 (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?j?', Number.Float), | |
| 529 (r'\d+[eE][+-]?[0-9]+j?', Number.Float), | |
| 530 (r'0[0-7]+j?', Number.Oct), | |
| 531 (r'0[bB][01]+', Number.Bin), | |
| 532 (r'0[xX][a-fA-F0-9]+', Number.Hex), | |
| 533 (r'\d+L', Number.Integer.Long), | |
| 534 (r'\d+j?', Number.Integer) | |
| 535 ], | |
| 536 'backtick': [ | |
| 537 ('`.*?`', String.Backtick), | |
| 538 ], | |
| 539 'name': [ | |
| 540 (r'@[\w.]+', Name.Decorator), | |
| 541 (r'[a-zA-Z_]\w*', Name), | |
| 542 ], | |
| 543 'funcname': [ | |
| 544 include('magicfuncs'), | |
| 545 (r'[a-zA-Z_]\w*', Name.Function, '#pop'), | |
| 546 default('#pop'), | |
| 547 ], | |
| 548 'classname': [ | |
| 549 (r'[a-zA-Z_]\w*', Name.Class, '#pop') | |
| 550 ], | |
| 551 'import': [ | |
| 552 (r'(?:[ \t]|\\\n)+', Text), | |
| 553 (r'as\b', Keyword.Namespace), | |
| 554 (r',', Operator), | |
| 555 (r'[a-zA-Z_][\w.]*', Name.Namespace), | |
| 556 default('#pop') # all else: go back | |
| 557 ], | |
| 558 'fromimport': [ | |
| 559 (r'(?:[ \t]|\\\n)+', Text), | |
| 560 (r'import\b', Keyword.Namespace, '#pop'), | |
| 561 # if None occurs here, it's "raise x from None", since None can | |
| 562 # never be a module name | |
| 563 (r'None\b', Name.Builtin.Pseudo, '#pop'), | |
| 564 # sadly, in "raise x from y" y will be highlighted as namespace too | |
| 565 (r'[a-zA-Z_.][\w.]*', Name.Namespace), | |
| 566 # anything else here also means "raise x from y" and is therefore | |
| 567 # not an error | |
| 568 default('#pop'), | |
| 569 ], | |
| 570 'stringescape': [ | |
| 571 (r'\\([\\abfnrtv"\']|\n|N\{.*?\}|u[a-fA-F0-9]{4}|' | |
| 572 r'U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape) | |
| 573 ], | |
| 574 'strings-single': innerstring_rules(String.Single), | |
| 575 'strings-double': innerstring_rules(String.Double), | |
| 576 'dqs': [ | |
| 577 (r'"', String.Double, '#pop'), | |
| 578 (r'\\\\|\\"|\\\n', String.Escape), # included here for raw strings | |
| 579 include('strings-double') | |
| 580 ], | |
| 581 'sqs': [ | |
| 582 (r"'", String.Single, '#pop'), | |
| 583 (r"\\\\|\\'|\\\n", String.Escape), # included here for raw strings | |
| 584 include('strings-single') | |
| 585 ], | |
| 586 'tdqs': [ | |
| 587 (r'"""', String.Double, '#pop'), | |
| 588 include('strings-double'), | |
| 589 (r'\n', String.Double) | |
| 590 ], | |
| 591 'tsqs': [ | |
| 592 (r"'''", String.Single, '#pop'), | |
| 593 include('strings-single'), | |
| 594 (r'\n', String.Single) | |
| 595 ], | |
| 596 } | |
| 597 | |
| 598 def analyse_text(text): | |
| 599 return shebang_matches(text, r'pythonw?2(\.\d)?') or \ | |
| 600 'import ' in text[:1000] | |
| 601 | |
| 602 | |
| 603 class PythonConsoleLexer(Lexer): | |
| 604 """ | |
| 605 For Python console output or doctests, such as: | |
| 606 | |
| 607 .. sourcecode:: pycon | |
| 608 | |
| 609 >>> a = 'foo' | |
| 610 >>> print a | |
| 611 foo | |
| 612 >>> 1 / 0 | |
| 613 Traceback (most recent call last): | |
| 614 File "<stdin>", line 1, in <module> | |
| 615 ZeroDivisionError: integer division or modulo by zero | |
| 616 | |
| 617 Additional options: | |
| 618 | |
| 619 `python3` | |
| 620 Use Python 3 lexer for code. Default is ``True``. | |
| 621 | |
| 622 .. versionadded:: 1.0 | |
| 623 .. versionchanged:: 2.5 | |
| 624 Now defaults to ``True``. | |
| 625 """ | |
| 626 name = 'Python console session' | |
| 627 aliases = ['pycon'] | |
| 628 mimetypes = ['text/x-python-doctest'] | |
| 629 | |
| 630 def __init__(self, **options): | |
| 631 self.python3 = get_bool_opt(options, 'python3', True) | |
| 632 Lexer.__init__(self, **options) | |
| 633 | |
| 634 def get_tokens_unprocessed(self, text): | |
| 635 if self.python3: | |
| 636 pylexer = PythonLexer(**self.options) | |
| 637 tblexer = PythonTracebackLexer(**self.options) | |
| 638 else: | |
| 639 pylexer = Python2Lexer(**self.options) | |
| 640 tblexer = Python2TracebackLexer(**self.options) | |
| 641 | |
| 642 curcode = '' | |
| 643 insertions = [] | |
| 644 curtb = '' | |
| 645 tbindex = 0 | |
| 646 tb = 0 | |
| 647 for match in line_re.finditer(text): | |
| 648 line = match.group() | |
| 649 if line.startswith('>>> ') or line.startswith('... '): | |
| 650 tb = 0 | |
| 651 insertions.append((len(curcode), | |
| 652 [(0, Generic.Prompt, line[:4])])) | |
| 653 curcode += line[4:] | |
| 654 elif line.rstrip() == '...' and not tb: | |
| 655 # only a new >>> prompt can end an exception block | |
| 656 # otherwise an ellipsis in place of the traceback frames | |
| 657 # will be mishandled | |
| 658 insertions.append((len(curcode), | |
| 659 [(0, Generic.Prompt, '...')])) | |
| 660 curcode += line[3:] | |
| 661 else: | |
| 662 if curcode: | |
| 663 yield from do_insertions( | |
| 664 insertions, pylexer.get_tokens_unprocessed(curcode)) | |
| 665 curcode = '' | |
| 666 insertions = [] | |
| 667 if (line.startswith('Traceback (most recent call last):') or | |
| 668 re.match(' File "[^"]+", line \\d+\\n$', line)): | |
| 669 tb = 1 | |
| 670 curtb = line | |
| 671 tbindex = match.start() | |
| 672 elif line == 'KeyboardInterrupt\n': | |
| 673 yield match.start(), Name.Class, line | |
| 674 elif tb: | |
| 675 curtb += line | |
| 676 if not (line.startswith(' ') or line.strip() == '...'): | |
| 677 tb = 0 | |
| 678 for i, t, v in tblexer.get_tokens_unprocessed(curtb): | |
| 679 yield tbindex+i, t, v | |
| 680 curtb = '' | |
| 681 else: | |
| 682 yield match.start(), Generic.Output, line | |
| 683 if curcode: | |
| 684 yield from do_insertions(insertions, | |
| 685 pylexer.get_tokens_unprocessed(curcode)) | |
| 686 if curtb: | |
| 687 for i, t, v in tblexer.get_tokens_unprocessed(curtb): | |
| 688 yield tbindex+i, t, v | |
| 689 | |
| 690 | |
| 691 class PythonTracebackLexer(RegexLexer): | |
| 692 """ | |
| 693 For Python 3.x tracebacks, with support for chained exceptions. | |
| 694 | |
| 695 .. versionadded:: 1.0 | |
| 696 | |
| 697 .. versionchanged:: 2.5 | |
| 698 This is now the default ``PythonTracebackLexer``. It is still available | |
| 699 as the alias ``Python3TracebackLexer``. | |
| 700 """ | |
| 701 | |
| 702 name = 'Python Traceback' | |
| 703 aliases = ['pytb', 'py3tb'] | |
| 704 filenames = ['*.pytb', '*.py3tb'] | |
| 705 mimetypes = ['text/x-python-traceback', 'text/x-python3-traceback'] | |
| 706 | |
| 707 tokens = { | |
| 708 'root': [ | |
| 709 (r'\n', Text), | |
| 710 (r'^Traceback \(most recent call last\):\n', Generic.Traceback, 'intb'), | |
| 711 (r'^During handling of the above exception, another ' | |
| 712 r'exception occurred:\n\n', Generic.Traceback), | |
| 713 (r'^The above exception was the direct cause of the ' | |
| 714 r'following exception:\n\n', Generic.Traceback), | |
| 715 (r'^(?= File "[^"]+", line \d+)', Generic.Traceback, 'intb'), | |
| 716 (r'^.*\n', Other), | |
| 717 ], | |
| 718 'intb': [ | |
| 719 (r'^( File )("[^"]+")(, line )(\d+)(, in )(.+)(\n)', | |
| 720 bygroups(Text, Name.Builtin, Text, Number, Text, Name, Text)), | |
| 721 (r'^( File )("[^"]+")(, line )(\d+)(\n)', | |
| 722 bygroups(Text, Name.Builtin, Text, Number, Text)), | |
| 723 (r'^( )(.+)(\n)', | |
| 724 bygroups(Text, using(PythonLexer), Text)), | |
| 725 (r'^([ \t]*)(\.\.\.)(\n)', | |
| 726 bygroups(Text, Comment, Text)), # for doctests... | |
| 727 (r'^([^:]+)(: )(.+)(\n)', | |
| 728 bygroups(Generic.Error, Text, Name, Text), '#pop'), | |
| 729 (r'^([a-zA-Z_][\w.]*)(:?\n)', | |
| 730 bygroups(Generic.Error, Text), '#pop') | |
| 731 ], | |
| 732 } | |
| 733 | |
| 734 | |
| 735 Python3TracebackLexer = PythonTracebackLexer | |
| 736 | |
| 737 | |
| 738 class Python2TracebackLexer(RegexLexer): | |
| 739 """ | |
| 740 For Python tracebacks. | |
| 741 | |
| 742 .. versionadded:: 0.7 | |
| 743 | |
| 744 .. versionchanged:: 2.5 | |
| 745 This class has been renamed from ``PythonTracebackLexer``. | |
| 746 ``PythonTracebackLexer`` now refers to the Python 3 variant. | |
| 747 """ | |
| 748 | |
| 749 name = 'Python 2.x Traceback' | |
| 750 aliases = ['py2tb'] | |
| 751 filenames = ['*.py2tb'] | |
| 752 mimetypes = ['text/x-python2-traceback'] | |
| 753 | |
| 754 tokens = { | |
| 755 'root': [ | |
| 756 # Cover both (most recent call last) and (innermost last) | |
| 757 # The optional ^C allows us to catch keyboard interrupt signals. | |
| 758 (r'^(\^C)?(Traceback.*\n)', | |
| 759 bygroups(Text, Generic.Traceback), 'intb'), | |
| 760 # SyntaxError starts with this. | |
| 761 (r'^(?= File "[^"]+", line \d+)', Generic.Traceback, 'intb'), | |
| 762 (r'^.*\n', Other), | |
| 763 ], | |
| 764 'intb': [ | |
| 765 (r'^( File )("[^"]+")(, line )(\d+)(, in )(.+)(\n)', | |
| 766 bygroups(Text, Name.Builtin, Text, Number, Text, Name, Text)), | |
| 767 (r'^( File )("[^"]+")(, line )(\d+)(\n)', | |
| 768 bygroups(Text, Name.Builtin, Text, Number, Text)), | |
| 769 (r'^( )(.+)(\n)', | |
| 770 bygroups(Text, using(Python2Lexer), Text)), | |
| 771 (r'^([ \t]*)(\.\.\.)(\n)', | |
| 772 bygroups(Text, Comment, Text)), # for doctests... | |
| 773 (r'^([^:]+)(: )(.+)(\n)', | |
| 774 bygroups(Generic.Error, Text, Name, Text), '#pop'), | |
| 775 (r'^([a-zA-Z_]\w*)(:?\n)', | |
| 776 bygroups(Generic.Error, Text), '#pop') | |
| 777 ], | |
| 778 } | |
| 779 | |
| 780 | |
| 781 class CythonLexer(RegexLexer): | |
| 782 """ | |
| 783 For Pyrex and `Cython <http://cython.org>`_ source code. | |
| 784 | |
| 785 .. versionadded:: 1.1 | |
| 786 """ | |
| 787 | |
| 788 name = 'Cython' | |
| 789 aliases = ['cython', 'pyx', 'pyrex'] | |
| 790 filenames = ['*.pyx', '*.pxd', '*.pxi'] | |
| 791 mimetypes = ['text/x-cython', 'application/x-cython'] | |
| 792 | |
| 793 tokens = { | |
| 794 'root': [ | |
| 795 (r'\n', Text), | |
| 796 (r'^(\s*)("""(?:.|\n)*?""")', bygroups(Text, String.Doc)), | |
| 797 (r"^(\s*)('''(?:.|\n)*?''')", bygroups(Text, String.Doc)), | |
| 798 (r'[^\S\n]+', Text), | |
| 799 (r'#.*$', Comment), | |
| 800 (r'[]{}:(),;[]', Punctuation), | |
| 801 (r'\\\n', Text), | |
| 802 (r'\\', Text), | |
| 803 (r'(in|is|and|or|not)\b', Operator.Word), | |
| 804 (r'(<)([a-zA-Z0-9.?]+)(>)', | |
| 805 bygroups(Punctuation, Keyword.Type, Punctuation)), | |
| 806 (r'!=|==|<<|>>|[-~+/*%=<>&^|.?]', Operator), | |
| 807 (r'(from)(\d+)(<=)(\s+)(<)(\d+)(:)', | |
| 808 bygroups(Keyword, Number.Integer, Operator, Name, Operator, | |
| 809 Name, Punctuation)), | |
| 810 include('keywords'), | |
| 811 (r'(def|property)(\s+)', bygroups(Keyword, Text), 'funcname'), | |
| 812 (r'(cp?def)(\s+)', bygroups(Keyword, Text), 'cdef'), | |
| 813 # (should actually start a block with only cdefs) | |
| 814 (r'(cdef)(:)', bygroups(Keyword, Punctuation)), | |
| 815 (r'(class|struct)(\s+)', bygroups(Keyword, Text), 'classname'), | |
| 816 (r'(from)(\s+)', bygroups(Keyword, Text), 'fromimport'), | |
| 817 (r'(c?import)(\s+)', bygroups(Keyword, Text), 'import'), | |
| 818 include('builtins'), | |
| 819 include('backtick'), | |
| 820 ('(?:[rR]|[uU][rR]|[rR][uU])"""', String, 'tdqs'), | |
| 821 ("(?:[rR]|[uU][rR]|[rR][uU])'''", String, 'tsqs'), | |
| 822 ('(?:[rR]|[uU][rR]|[rR][uU])"', String, 'dqs'), | |
| 823 ("(?:[rR]|[uU][rR]|[rR][uU])'", String, 'sqs'), | |
| 824 ('[uU]?"""', String, combined('stringescape', 'tdqs')), | |
| 825 ("[uU]?'''", String, combined('stringescape', 'tsqs')), | |
| 826 ('[uU]?"', String, combined('stringescape', 'dqs')), | |
| 827 ("[uU]?'", String, combined('stringescape', 'sqs')), | |
| 828 include('name'), | |
| 829 include('numbers'), | |
| 830 ], | |
| 831 'keywords': [ | |
| 832 (words(( | |
| 833 'assert', 'async', 'await', 'break', 'by', 'continue', 'ctypedef', 'del', 'elif', | |
| 834 'else', 'except', 'except?', 'exec', 'finally', 'for', 'fused', 'gil', | |
| 835 'global', 'if', 'include', 'lambda', 'nogil', 'pass', 'print', | |
| 836 'raise', 'return', 'try', 'while', 'yield', 'as', 'with'), suffix=r'\b'), | |
| 837 Keyword), | |
| 838 (r'(DEF|IF|ELIF|ELSE)\b', Comment.Preproc), | |
| 839 ], | |
| 840 'builtins': [ | |
| 841 (words(( | |
| 842 '__import__', 'abs', 'all', 'any', 'apply', 'basestring', 'bin', | |
| 843 'bool', 'buffer', 'bytearray', 'bytes', 'callable', 'chr', | |
| 844 'classmethod', 'cmp', 'coerce', 'compile', 'complex', 'delattr', | |
| 845 'dict', 'dir', 'divmod', 'enumerate', 'eval', 'execfile', 'exit', | |
| 846 'file', 'filter', 'float', 'frozenset', 'getattr', 'globals', | |
| 847 'hasattr', 'hash', 'hex', 'id', 'input', 'int', 'intern', 'isinstance', | |
| 848 'issubclass', 'iter', 'len', 'list', 'locals', 'long', 'map', 'max', | |
| 849 'min', 'next', 'object', 'oct', 'open', 'ord', 'pow', 'property', | |
| 850 'range', 'raw_input', 'reduce', 'reload', 'repr', 'reversed', | |
| 851 'round', 'set', 'setattr', 'slice', 'sorted', 'staticmethod', | |
| 852 'str', 'sum', 'super', 'tuple', 'type', 'unichr', 'unicode', 'unsigned', | |
| 853 'vars', 'xrange', 'zip'), prefix=r'(?<!\.)', suffix=r'\b'), | |
| 854 Name.Builtin), | |
| 855 (r'(?<!\.)(self|None|Ellipsis|NotImplemented|False|True|NULL' | |
| 856 r')\b', Name.Builtin.Pseudo), | |
| 857 (words(( | |
| 858 'ArithmeticError', 'AssertionError', 'AttributeError', | |
| 859 'BaseException', 'DeprecationWarning', 'EOFError', 'EnvironmentError', | |
| 860 'Exception', 'FloatingPointError', 'FutureWarning', 'GeneratorExit', | |
| 861 'IOError', 'ImportError', 'ImportWarning', 'IndentationError', | |
| 862 'IndexError', 'KeyError', 'KeyboardInterrupt', 'LookupError', | |
| 863 'MemoryError', 'NameError', 'NotImplemented', 'NotImplementedError', | |
| 864 'OSError', 'OverflowError', 'OverflowWarning', | |
| 865 'PendingDeprecationWarning', 'ReferenceError', 'RuntimeError', | |
| 866 'RuntimeWarning', 'StandardError', 'StopIteration', 'SyntaxError', | |
| 867 'SyntaxWarning', 'SystemError', 'SystemExit', 'TabError', | |
| 868 'TypeError', 'UnboundLocalError', 'UnicodeDecodeError', | |
| 869 'UnicodeEncodeError', 'UnicodeError', 'UnicodeTranslateError', | |
| 870 'UnicodeWarning', 'UserWarning', 'ValueError', 'Warning', | |
| 871 'ZeroDivisionError'), prefix=r'(?<!\.)', suffix=r'\b'), | |
| 872 Name.Exception), | |
| 873 ], | |
| 874 'numbers': [ | |
| 875 (r'(\d+\.?\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float), | |
| 876 (r'0\d+', Number.Oct), | |
| 877 (r'0[xX][a-fA-F0-9]+', Number.Hex), | |
| 878 (r'\d+L', Number.Integer.Long), | |
| 879 (r'\d+', Number.Integer) | |
| 880 ], | |
| 881 'backtick': [ | |
| 882 ('`.*?`', String.Backtick), | |
| 883 ], | |
| 884 'name': [ | |
| 885 (r'@\w+', Name.Decorator), | |
| 886 (r'[a-zA-Z_]\w*', Name), | |
| 887 ], | |
| 888 'funcname': [ | |
| 889 (r'[a-zA-Z_]\w*', Name.Function, '#pop') | |
| 890 ], | |
| 891 'cdef': [ | |
| 892 (r'(public|readonly|extern|api|inline)\b', Keyword.Reserved), | |
| 893 (r'(struct|enum|union|class)\b', Keyword), | |
| 894 (r'([a-zA-Z_]\w*)(\s*)(?=[(:#=]|$)', | |
| 895 bygroups(Name.Function, Text), '#pop'), | |
| 896 (r'([a-zA-Z_]\w*)(\s*)(,)', | |
| 897 bygroups(Name.Function, Text, Punctuation)), | |
| 898 (r'from\b', Keyword, '#pop'), | |
| 899 (r'as\b', Keyword), | |
| 900 (r':', Punctuation, '#pop'), | |
| 901 (r'(?=["\'])', Text, '#pop'), | |
| 902 (r'[a-zA-Z_]\w*', Keyword.Type), | |
| 903 (r'.', Text), | |
| 904 ], | |
| 905 'classname': [ | |
| 906 (r'[a-zA-Z_]\w*', Name.Class, '#pop') | |
| 907 ], | |
| 908 'import': [ | |
| 909 (r'(\s+)(as)(\s+)', bygroups(Text, Keyword, Text)), | |
| 910 (r'[a-zA-Z_][\w.]*', Name.Namespace), | |
| 911 (r'(\s*)(,)(\s*)', bygroups(Text, Operator, Text)), | |
| 912 default('#pop') # all else: go back | |
| 913 ], | |
| 914 'fromimport': [ | |
| 915 (r'(\s+)(c?import)\b', bygroups(Text, Keyword), '#pop'), | |
| 916 (r'[a-zA-Z_.][\w.]*', Name.Namespace), | |
| 917 # ``cdef foo from "header"``, or ``for foo from 0 < i < 10`` | |
| 918 default('#pop'), | |
| 919 ], | |
| 920 'stringescape': [ | |
| 921 (r'\\([\\abfnrtv"\']|\n|N\{.*?\}|u[a-fA-F0-9]{4}|' | |
| 922 r'U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape) | |
| 923 ], | |
| 924 'strings': [ | |
| 925 (r'%(\([a-zA-Z0-9]+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?' | |
| 926 '[hlL]?[E-GXc-giorsux%]', String.Interpol), | |
| 927 (r'[^\\\'"%\n]+', String), | |
| 928 # quotes, percents and backslashes must be parsed one at a time | |
| 929 (r'[\'"\\]', String), | |
| 930 # unhandled string formatting sign | |
| 931 (r'%', String) | |
| 932 # newlines are an error (use "nl" state) | |
| 933 ], | |
| 934 'nl': [ | |
| 935 (r'\n', String) | |
| 936 ], | |
| 937 'dqs': [ | |
| 938 (r'"', String, '#pop'), | |
| 939 (r'\\\\|\\"|\\\n', String.Escape), # included here again for raw strings | |
| 940 include('strings') | |
| 941 ], | |
| 942 'sqs': [ | |
| 943 (r"'", String, '#pop'), | |
| 944 (r"\\\\|\\'|\\\n", String.Escape), # included here again for raw strings | |
| 945 include('strings') | |
| 946 ], | |
| 947 'tdqs': [ | |
| 948 (r'"""', String, '#pop'), | |
| 949 include('strings'), | |
| 950 include('nl') | |
| 951 ], | |
| 952 'tsqs': [ | |
| 953 (r"'''", String, '#pop'), | |
| 954 include('strings'), | |
| 955 include('nl') | |
| 956 ], | |
| 957 } | |
| 958 | |
| 959 | |
| 960 class DgLexer(RegexLexer): | |
| 961 """ | |
| 962 Lexer for `dg <http://pyos.github.com/dg>`_, | |
| 963 a functional and object-oriented programming language | |
| 964 running on the CPython 3 VM. | |
| 965 | |
| 966 .. versionadded:: 1.6 | |
| 967 """ | |
| 968 name = 'dg' | |
| 969 aliases = ['dg'] | |
| 970 filenames = ['*.dg'] | |
| 971 mimetypes = ['text/x-dg'] | |
| 972 | |
| 973 tokens = { | |
| 974 'root': [ | |
| 975 (r'\s+', Text), | |
| 976 (r'#.*?$', Comment.Single), | |
| 977 | |
| 978 (r'(?i)0b[01]+', Number.Bin), | |
| 979 (r'(?i)0o[0-7]+', Number.Oct), | |
| 980 (r'(?i)0x[0-9a-f]+', Number.Hex), | |
| 981 (r'(?i)[+-]?[0-9]+\.[0-9]+(e[+-]?[0-9]+)?j?', Number.Float), | |
| 982 (r'(?i)[+-]?[0-9]+e[+-]?\d+j?', Number.Float), | |
| 983 (r'(?i)[+-]?[0-9]+j?', Number.Integer), | |
| 984 | |
| 985 (r"(?i)(br|r?b?)'''", String, combined('stringescape', 'tsqs', 'string')), | |
| 986 (r'(?i)(br|r?b?)"""', String, combined('stringescape', 'tdqs', 'string')), | |
| 987 (r"(?i)(br|r?b?)'", String, combined('stringescape', 'sqs', 'string')), | |
| 988 (r'(?i)(br|r?b?)"', String, combined('stringescape', 'dqs', 'string')), | |
| 989 | |
| 990 (r"`\w+'*`", Operator), | |
| 991 (r'\b(and|in|is|or|where)\b', Operator.Word), | |
| 992 (r'[!$%&*+\-./:<-@\\^|~;,]+', Operator), | |
| 993 | |
| 994 (words(( | |
| 995 'bool', 'bytearray', 'bytes', 'classmethod', 'complex', 'dict', 'dict\'', | |
| 996 'float', 'frozenset', 'int', 'list', 'list\'', 'memoryview', 'object', | |
| 997 'property', 'range', 'set', 'set\'', 'slice', 'staticmethod', 'str', | |
| 998 'super', 'tuple', 'tuple\'', 'type'), | |
| 999 prefix=r'(?<!\.)', suffix=r'(?![\'\w])'), | |
| 1000 Name.Builtin), | |
| 1001 (words(( | |
| 1002 '__import__', 'abs', 'all', 'any', 'bin', 'bind', 'chr', 'cmp', 'compile', | |
| 1003 'complex', 'delattr', 'dir', 'divmod', 'drop', 'dropwhile', 'enumerate', | |
| 1004 'eval', 'exhaust', 'filter', 'flip', 'foldl1?', 'format', 'fst', | |
| 1005 'getattr', 'globals', 'hasattr', 'hash', 'head', 'hex', 'id', 'init', | |
| 1006 'input', 'isinstance', 'issubclass', 'iter', 'iterate', 'last', 'len', | |
| 1007 'locals', 'map', 'max', 'min', 'next', 'oct', 'open', 'ord', 'pow', | |
| 1008 'print', 'repr', 'reversed', 'round', 'setattr', 'scanl1?', 'snd', | |
| 1009 'sorted', 'sum', 'tail', 'take', 'takewhile', 'vars', 'zip'), | |
| 1010 prefix=r'(?<!\.)', suffix=r'(?![\'\w])'), | |
| 1011 Name.Builtin), | |
| 1012 (r"(?<!\.)(self|Ellipsis|NotImplemented|None|True|False)(?!['\w])", | |
| 1013 Name.Builtin.Pseudo), | |
| 1014 | |
| 1015 (r"(?<!\.)[A-Z]\w*(Error|Exception|Warning)'*(?!['\w])", | |
| 1016 Name.Exception), | |
| 1017 (r"(?<!\.)(Exception|GeneratorExit|KeyboardInterrupt|StopIteration|" | |
| 1018 r"SystemExit)(?!['\w])", Name.Exception), | |
| 1019 | |
| 1020 (r"(?<![\w.])(except|finally|for|if|import|not|otherwise|raise|" | |
| 1021 r"subclass|while|with|yield)(?!['\w])", Keyword.Reserved), | |
| 1022 | |
| 1023 (r"[A-Z_]+'*(?!['\w])", Name), | |
| 1024 (r"[A-Z]\w+'*(?!['\w])", Keyword.Type), | |
| 1025 (r"\w+'*", Name), | |
| 1026 | |
| 1027 (r'[()]', Punctuation), | |
| 1028 (r'.', Error), | |
| 1029 ], | |
| 1030 'stringescape': [ | |
| 1031 (r'\\([\\abfnrtv"\']|\n|N\{.*?\}|u[a-fA-F0-9]{4}|' | |
| 1032 r'U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape) | |
| 1033 ], | |
| 1034 'string': [ | |
| 1035 (r'%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?' | |
| 1036 '[hlL]?[E-GXc-giorsux%]', String.Interpol), | |
| 1037 (r'[^\\\'"%\n]+', String), | |
| 1038 # quotes, percents and backslashes must be parsed one at a time | |
| 1039 (r'[\'"\\]', String), | |
| 1040 # unhandled string formatting sign | |
| 1041 (r'%', String), | |
| 1042 (r'\n', String) | |
| 1043 ], | |
| 1044 'dqs': [ | |
| 1045 (r'"', String, '#pop') | |
| 1046 ], | |
| 1047 'sqs': [ | |
| 1048 (r"'", String, '#pop') | |
| 1049 ], | |
| 1050 'tdqs': [ | |
| 1051 (r'"""', String, '#pop') | |
| 1052 ], | |
| 1053 'tsqs': [ | |
| 1054 (r"'''", String, '#pop') | |
| 1055 ], | |
| 1056 } | |
| 1057 | |
| 1058 | |
| 1059 class NumPyLexer(PythonLexer): | |
| 1060 """ | |
| 1061 A Python lexer recognizing Numerical Python builtins. | |
| 1062 | |
| 1063 .. versionadded:: 0.10 | |
| 1064 """ | |
| 1065 | |
| 1066 name = 'NumPy' | |
| 1067 aliases = ['numpy'] | |
| 1068 | |
| 1069 # override the mimetypes to not inherit them from python | |
| 1070 mimetypes = [] | |
| 1071 filenames = [] | |
| 1072 | |
| 1073 EXTRA_KEYWORDS = { | |
| 1074 'abs', 'absolute', 'accumulate', 'add', 'alen', 'all', 'allclose', | |
| 1075 'alltrue', 'alterdot', 'amax', 'amin', 'angle', 'any', 'append', | |
| 1076 'apply_along_axis', 'apply_over_axes', 'arange', 'arccos', 'arccosh', | |
| 1077 'arcsin', 'arcsinh', 'arctan', 'arctan2', 'arctanh', 'argmax', 'argmin', | |
| 1078 'argsort', 'argwhere', 'around', 'array', 'array2string', 'array_equal', | |
| 1079 'array_equiv', 'array_repr', 'array_split', 'array_str', 'arrayrange', | |
| 1080 'asanyarray', 'asarray', 'asarray_chkfinite', 'ascontiguousarray', | |
| 1081 'asfarray', 'asfortranarray', 'asmatrix', 'asscalar', 'astype', | |
| 1082 'atleast_1d', 'atleast_2d', 'atleast_3d', 'average', 'bartlett', | |
| 1083 'base_repr', 'beta', 'binary_repr', 'bincount', 'binomial', | |
| 1084 'bitwise_and', 'bitwise_not', 'bitwise_or', 'bitwise_xor', 'blackman', | |
| 1085 'bmat', 'broadcast', 'byte_bounds', 'bytes', 'byteswap', 'c_', | |
| 1086 'can_cast', 'ceil', 'choose', 'clip', 'column_stack', 'common_type', | |
| 1087 'compare_chararrays', 'compress', 'concatenate', 'conj', 'conjugate', | |
| 1088 'convolve', 'copy', 'corrcoef', 'correlate', 'cos', 'cosh', 'cov', | |
| 1089 'cross', 'cumprod', 'cumproduct', 'cumsum', 'delete', 'deprecate', | |
| 1090 'diag', 'diagflat', 'diagonal', 'diff', 'digitize', 'disp', 'divide', | |
| 1091 'dot', 'dsplit', 'dstack', 'dtype', 'dump', 'dumps', 'ediff1d', 'empty', | |
| 1092 'empty_like', 'equal', 'exp', 'expand_dims', 'expm1', 'extract', 'eye', | |
| 1093 'fabs', 'fastCopyAndTranspose', 'fft', 'fftfreq', 'fftshift', 'fill', | |
| 1094 'finfo', 'fix', 'flat', 'flatnonzero', 'flatten', 'fliplr', 'flipud', | |
| 1095 'floor', 'floor_divide', 'fmod', 'frexp', 'fromarrays', 'frombuffer', | |
| 1096 'fromfile', 'fromfunction', 'fromiter', 'frompyfunc', 'fromstring', | |
| 1097 'generic', 'get_array_wrap', 'get_include', 'get_numarray_include', | |
| 1098 'get_numpy_include', 'get_printoptions', 'getbuffer', 'getbufsize', | |
| 1099 'geterr', 'geterrcall', 'geterrobj', 'getfield', 'gradient', 'greater', | |
| 1100 'greater_equal', 'gumbel', 'hamming', 'hanning', 'histogram', | |
| 1101 'histogram2d', 'histogramdd', 'hsplit', 'hstack', 'hypot', 'i0', | |
| 1102 'identity', 'ifft', 'imag', 'index_exp', 'indices', 'inf', 'info', | |
| 1103 'inner', 'insert', 'int_asbuffer', 'interp', 'intersect1d', | |
| 1104 'intersect1d_nu', 'inv', 'invert', 'iscomplex', 'iscomplexobj', | |
| 1105 'isfinite', 'isfortran', 'isinf', 'isnan', 'isneginf', 'isposinf', | |
| 1106 'isreal', 'isrealobj', 'isscalar', 'issctype', 'issubclass_', | |
| 1107 'issubdtype', 'issubsctype', 'item', 'itemset', 'iterable', 'ix_', | |
| 1108 'kaiser', 'kron', 'ldexp', 'left_shift', 'less', 'less_equal', 'lexsort', | |
| 1109 'linspace', 'load', 'loads', 'loadtxt', 'log', 'log10', 'log1p', 'log2', | |
| 1110 'logical_and', 'logical_not', 'logical_or', 'logical_xor', 'logspace', | |
| 1111 'lstsq', 'mat', 'matrix', 'max', 'maximum', 'maximum_sctype', | |
| 1112 'may_share_memory', 'mean', 'median', 'meshgrid', 'mgrid', 'min', | |
| 1113 'minimum', 'mintypecode', 'mod', 'modf', 'msort', 'multiply', 'nan', | |
| 1114 'nan_to_num', 'nanargmax', 'nanargmin', 'nanmax', 'nanmin', 'nansum', | |
| 1115 'ndenumerate', 'ndim', 'ndindex', 'negative', 'newaxis', 'newbuffer', | |
| 1116 'newbyteorder', 'nonzero', 'not_equal', 'obj2sctype', 'ogrid', 'ones', | |
| 1117 'ones_like', 'outer', 'permutation', 'piecewise', 'pinv', 'pkgload', | |
| 1118 'place', 'poisson', 'poly', 'poly1d', 'polyadd', 'polyder', 'polydiv', | |
| 1119 'polyfit', 'polyint', 'polymul', 'polysub', 'polyval', 'power', 'prod', | |
| 1120 'product', 'ptp', 'put', 'putmask', 'r_', 'randint', 'random_integers', | |
| 1121 'random_sample', 'ranf', 'rank', 'ravel', 'real', 'real_if_close', | |
| 1122 'recarray', 'reciprocal', 'reduce', 'remainder', 'repeat', 'require', | |
| 1123 'reshape', 'resize', 'restoredot', 'right_shift', 'rint', 'roll', | |
| 1124 'rollaxis', 'roots', 'rot90', 'round', 'round_', 'row_stack', 's_', | |
| 1125 'sample', 'savetxt', 'sctype2char', 'searchsorted', 'seed', 'select', | |
| 1126 'set_numeric_ops', 'set_printoptions', 'set_string_function', | |
| 1127 'setbufsize', 'setdiff1d', 'seterr', 'seterrcall', 'seterrobj', | |
| 1128 'setfield', 'setflags', 'setmember1d', 'setxor1d', 'shape', | |
| 1129 'show_config', 'shuffle', 'sign', 'signbit', 'sin', 'sinc', 'sinh', | |
| 1130 'size', 'slice', 'solve', 'sometrue', 'sort', 'sort_complex', 'source', | |
| 1131 'split', 'sqrt', 'square', 'squeeze', 'standard_normal', 'std', | |
| 1132 'subtract', 'sum', 'svd', 'swapaxes', 'take', 'tan', 'tanh', 'tensordot', | |
| 1133 'test', 'tile', 'tofile', 'tolist', 'tostring', 'trace', 'transpose', | |
| 1134 'trapz', 'tri', 'tril', 'trim_zeros', 'triu', 'true_divide', 'typeDict', | |
| 1135 'typename', 'uniform', 'union1d', 'unique', 'unique1d', 'unravel_index', | |
| 1136 'unwrap', 'vander', 'var', 'vdot', 'vectorize', 'view', 'vonmises', | |
| 1137 'vsplit', 'vstack', 'weibull', 'where', 'who', 'zeros', 'zeros_like' | |
| 1138 } | |
| 1139 | |
| 1140 def get_tokens_unprocessed(self, text): | |
| 1141 for index, token, value in \ | |
| 1142 PythonLexer.get_tokens_unprocessed(self, text): | |
| 1143 if token is Name and value in self.EXTRA_KEYWORDS: | |
| 1144 yield index, Keyword.Pseudo, value | |
| 1145 else: | |
| 1146 yield index, token, value | |
| 1147 | |
| 1148 def analyse_text(text): | |
| 1149 return (shebang_matches(text, r'pythonw?(3(\.\d)?)?') or | |
| 1150 'import ' in text[:1000]) \ | |
| 1151 and ('import numpy' in text or 'from numpy import' in text) |
