Mercurial Repositories > eric / file comparison

comparison: ThirdParty/Pygments/pygments/lexers/julia.py

ThirdParty/Pygments/pygments/lexers/julia.py

changeset 5713
6762afd9f963
parent 5072
aab59042fefb
child 6651
e8f3b5568b21
equal deleted inserted replaced
5712:f0d08bdeacf4 5713:6762afd9f963
3 pygments.lexers.julia 3 pygments.lexers.julia
4 ~~~~~~~~~~~~~~~~~~~~~ 4 ~~~~~~~~~~~~~~~~~~~~~
5 5
6 Lexers for the Julia language. 6 Lexers for the Julia language.
7 7
8 :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. 8 :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
9 :license: BSD, see LICENSE for details. 9 :license: BSD, see LICENSE for details.
10 """ 10 """
11 11
12 import re 12 import re
13 13
14 from pygments.lexer import Lexer, RegexLexer, bygroups, combined, do_insertions 14 from pygments.lexer import Lexer, RegexLexer, bygroups, do_insertions, \
15 words, include
15 from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ 16 from pygments.token import Text, Comment, Operator, Keyword, Name, String, \
16 Number, Punctuation, Generic 17 Number, Punctuation, Generic
17 from pygments.util import shebang_matches, unirange 18 from pygments.util import shebang_matches, unirange
18 19
19 __all__ = ['JuliaLexer', 'JuliaConsoleLexer'] 20 __all__ = ['JuliaLexer', 'JuliaConsoleLexer']
20 21
22 allowed_variable = (
23 u'(?:[a-zA-Z_\u00A1-\uffff]|%s)(?:[a-zA-Z_0-9\u00A1-\uffff]|%s)*!*' %
24 ((unirange(0x10000, 0x10ffff),) * 2))
25
21 26
22 class JuliaLexer(RegexLexer): 27 class JuliaLexer(RegexLexer):
23 """ 28 """
24 For `Julia <http://julialang.org/>`_ source code. 29 For `Julia <http://julialang.org/>`_ source code.
25 30
26 .. versionadded:: 1.6 31 .. versionadded:: 1.6
27 """ 32 """
33
28 name = 'Julia' 34 name = 'Julia'
29 aliases = ['julia', 'jl'] 35 aliases = ['julia', 'jl']
30 filenames = ['*.jl'] 36 filenames = ['*.jl']
31 mimetypes = ['text/x-julia', 'application/x-julia'] 37 mimetypes = ['text/x-julia', 'application/x-julia']
32 38
33 flags = re.MULTILINE | re.UNICODE 39 flags = re.MULTILINE | re.UNICODE
34
35 builtins = [
36 'exit', 'whos', 'edit', 'load', 'is', 'isa', 'isequal', 'typeof', 'tuple',
37 'ntuple', 'uid', 'hash', 'finalizer', 'convert', 'promote', 'subtype',
38 'typemin', 'typemax', 'realmin', 'realmax', 'sizeof', 'eps', 'promote_type',
39 'method_exists', 'applicable', 'invoke', 'dlopen', 'dlsym', 'system',
40 'error', 'throw', 'assert', 'new', 'Inf', 'Nan', 'pi', 'im',
41 ]
42 40
43 tokens = { 41 tokens = {
44 'root': [ 42 'root': [
45 (r'\n', Text), 43 (r'\n', Text),
46 (r'[^\S\n]+', Text), 44 (r'[^\S\n]+', Text),
47 (r'#=', Comment.Multiline, "blockcomment"), 45 (r'#=', Comment.Multiline, "blockcomment"),
48 (r'#.*$', Comment), 46 (r'#.*$', Comment),
49 (r'[]{}:(),;[@]', Punctuation), 47 (r'[\[\]{}(),;]', Punctuation),
50 (r'\\\n', Text),
51 (r'\\', Text),
52 48
53 # keywords 49 # keywords
54 (r'(begin|while|for|in|return|break|continue|' 50 (r'in\b', Keyword.Pseudo),
55 r'macro|quote|let|if|elseif|else|try|catch|end|' 51 (r'(true|false)\b', Keyword.Constant),
56 r'bitstype|ccall|do|using|module|import|export|'
57 r'importall|baremodule|immutable)\b', Keyword),
58 (r'(local|global|const)\b', Keyword.Declaration), 52 (r'(local|global|const)\b', Keyword.Declaration),
59 (r'(Bool|Int|Int8|Int16|Int32|Int64|Uint|Uint8|Uint16|Uint32|Uint64' 53 (words([
60 r'|Float32|Float64|Complex64|Complex128|Any|Nothing|None)\b', 54 'function', 'type', 'typealias', 'abstract', 'immutable',
55 'baremodule', 'begin', 'bitstype', 'break', 'catch', 'ccall',
56 'continue', 'do', 'else', 'elseif', 'end', 'export', 'finally',
57 'for', 'if', 'import', 'importall', 'let', 'macro', 'module',
58 'quote', 'return', 'try', 'using', 'while'],
59 suffix=r'\b'), Keyword),
60
61 # NOTE
62 # Patterns below work only for definition sites and thus hardly reliable.
63 #
64 # functions
65 # (r'(function)(\s+)(' + allowed_variable + ')',
66 # bygroups(Keyword, Text, Name.Function)),
67 #
68 # types
69 # (r'(type|typealias|abstract|immutable)(\s+)(' + allowed_variable + ')',
70 # bygroups(Keyword, Text, Name.Class)),
71
72 # type names
73 (words([
74 'ANY', 'ASCIIString', 'AbstractArray', 'AbstractChannel',
75 'AbstractFloat', 'AbstractMatrix', 'AbstractRNG',
76 'AbstractSparseArray', 'AbstractSparseMatrix',
77 'AbstractSparseVector', 'AbstractString', 'AbstractVecOrMat',
78 'AbstractVector', 'Any', 'ArgumentError', 'Array',
79 'AssertionError', 'Associative', 'Base64DecodePipe',
80 'Base64EncodePipe', 'Bidiagonal', 'BigFloat', 'BigInt',
81 'BitArray', 'BitMatrix', 'BitVector', 'Bool', 'BoundsError',
82 'Box', 'BufferStream', 'CapturedException', 'CartesianIndex',
83 'CartesianRange', 'Cchar', 'Cdouble', 'Cfloat', 'Channel',
84 'Char', 'Cint', 'Cintmax_t', 'Clong', 'Clonglong',
85 'ClusterManager', 'Cmd', 'Coff_t', 'Colon', 'Complex',
86 'Complex128', 'Complex32', 'Complex64', 'CompositeException',
87 'Condition', 'Cptrdiff_t', 'Cshort', 'Csize_t', 'Cssize_t',
88 'Cstring', 'Cuchar', 'Cuint', 'Cuintmax_t', 'Culong',
89 'Culonglong', 'Cushort', 'Cwchar_t', 'Cwstring', 'DataType',
90 'Date', 'DateTime', 'DenseArray', 'DenseMatrix',
91 'DenseVecOrMat', 'DenseVector', 'Diagonal', 'Dict',
92 'DimensionMismatch', 'Dims', 'DirectIndexString', 'Display',
93 'DivideError', 'DomainError', 'EOFError', 'EachLine', 'Enum',
94 'Enumerate', 'ErrorException', 'Exception', 'Expr',
95 'Factorization', 'FileMonitor', 'FileOffset', 'Filter',
96 'Float16', 'Float32', 'Float64', 'FloatRange', 'Function',
97 'GenSym', 'GlobalRef', 'GotoNode', 'HTML', 'Hermitian', 'IO',
98 'IOBuffer', 'IOStream', 'IPv4', 'IPv6', 'InexactError',
99 'InitError', 'Int', 'Int128', 'Int16', 'Int32', 'Int64', 'Int8',
100 'IntSet', 'Integer', 'InterruptException', 'IntrinsicFunction',
101 'InvalidStateException', 'Irrational', 'KeyError', 'LabelNode',
102 'LambdaStaticData', 'LinSpace', 'LineNumberNode', 'LoadError',
103 'LocalProcess', 'LowerTriangular', 'MIME', 'Matrix',
104 'MersenneTwister', 'Method', 'MethodError', 'MethodTable',
105 'Module', 'NTuple', 'NewvarNode', 'NullException', 'Nullable',
106 'Number', 'ObjectIdDict', 'OrdinalRange', 'OutOfMemoryError',
107 'OverflowError', 'Pair', 'ParseError', 'PartialQuickSort',
108 'Pipe', 'PollingFileWatcher', 'ProcessExitedException',
109 'ProcessGroup', 'Ptr', 'QuoteNode', 'RandomDevice', 'Range',
110 'Rational', 'RawFD', 'ReadOnlyMemoryError', 'Real',
111 'ReentrantLock', 'Ref', 'Regex', 'RegexMatch',
112 'RemoteException', 'RemoteRef', 'RepString', 'RevString',
113 'RopeString', 'RoundingMode', 'SegmentationFault',
114 'SerializationState', 'Set', 'SharedArray', 'SharedMatrix',
115 'SharedVector', 'Signed', 'SimpleVector', 'SparseMatrixCSC',
116 'StackOverflowError', 'StatStruct', 'StepRange', 'StridedArray',
117 'StridedMatrix', 'StridedVecOrMat', 'StridedVector', 'SubArray',
118 'SubString', 'SymTridiagonal', 'Symbol', 'SymbolNode',
119 'Symmetric', 'SystemError', 'TCPSocket', 'Task', 'Text',
120 'TextDisplay', 'Timer', 'TopNode', 'Tridiagonal', 'Tuple',
121 'Type', 'TypeConstructor', 'TypeError', 'TypeName', 'TypeVar',
122 'UDPSocket', 'UInt', 'UInt128', 'UInt16', 'UInt32', 'UInt64',
123 'UInt8', 'UTF16String', 'UTF32String', 'UTF8String',
124 'UndefRefError', 'UndefVarError', 'UnicodeError', 'UniformScaling',
125 'Union', 'UnitRange', 'Unsigned', 'UpperTriangular', 'Val',
126 'Vararg', 'VecOrMat', 'Vector', 'VersionNumber', 'Void', 'WString',
127 'WeakKeyDict', 'WeakRef', 'WorkerConfig', 'Zip'], suffix=r'\b'),
61 Keyword.Type), 128 Keyword.Type),
62 129
63 # functions 130 # builtins
64 (r'(function)((?:\s|\\\s)+)', 131 (words([
65 bygroups(Keyword, Name.Function), 'funcname'), 132 u'ARGS', u'CPU_CORES', u'C_NULL', u'DevNull', u'ENDIAN_BOM',
66 133 u'ENV', u'I', u'Inf', u'Inf16', u'Inf32', u'Inf64',
67 # types 134 u'InsertionSort', u'JULIA_HOME', u'LOAD_PATH', u'MergeSort',
68 (r'(type|typealias|abstract|immutable)((?:\s|\\\s)+)', 135 u'NaN', u'NaN16', u'NaN32', u'NaN64', u'OS_NAME',
69 bygroups(Keyword, Name.Class), 'typename'), 136 u'QuickSort', u'RoundDown', u'RoundFromZero', u'RoundNearest',
137 u'RoundNearestTiesAway', u'RoundNearestTiesUp',
138 u'RoundToZero', u'RoundUp', u'STDERR', u'STDIN', u'STDOUT',
139 u'VERSION', u'WORD_SIZE', u'catalan', u'e', u'eu',
140 u'eulergamma', u'golden', u'im', u'nothing', u'pi', u'γ',
141 u'π', u'φ'],
142 suffix=r'\b'), Name.Builtin),
70 143
71 # operators 144 # operators
72 (r'==|!=|<=|>=|->|&&|\|\||::|<:|[-~+/*%=<>&^|.?!$]', Operator), 145 # see: https://github.com/JuliaLang/julia/blob/master/src/julia-parser.scm
73 (r'\.\*|\.\^|\.\\|\.\/|\\', Operator), 146 (words([
74 147 # prec-assignment
75 # builtins 148 u'=', u':=', u'+=', u'-=', u'*=', u'/=', u'//=', u'.//=', u'.*=', u'./=',
76 ('(' + '|'.join(builtins) + r')\b', Name.Builtin), 149 u'\=', u'.\=', u'^=', u'.^=', u'÷=', u'.÷=', u'%=', u'.%=', u'|=', u'&=',
77 150 u'$=', u'=>', u'<<=', u'>>=', u'>>>=', u'~', u'.+=', u'.-=',
78 # backticks 151 # prec-conditional
79 (r'`(?s).*?`', String.Backtick), 152 u'?',
153 # prec-arrow
154 u'--', u'-->',
155 # prec-lazy-or
156 u'||',
157 # prec-lazy-and
158 u'&&',
159 # prec-comparison
160 u'>', u'<', u'>=', u'≥', u'<=', u'≤', u'==', u'===', u'≡', u'!=', u'≠',
161 u'!==', u'≢', u'.>', u'.<', u'.>=', u'.≥', u'.<=', u'.≤', u'.==', u'.!=',
162 u'.≠', u'.=', u'.!', u'<:', u'>:', u'∈', u'∉', u'∋', u'∌', u'⊆',
163 u'⊈', u'⊂',
164 u'⊄', u'⊊',
165 # prec-pipe
166 u'|>', u'<|',
167 # prec-colon
168 u':',
169 # prec-plus
170 u'+', u'-', u'.+', u'.-', u'|', u'∪', u'$',
171 # prec-bitshift
172 u'<<', u'>>', u'>>>', u'.<<', u'.>>', u'.>>>',
173 # prec-times
174 u'*', u'/', u'./', u'÷', u'.÷', u'%', u'⋅', u'.%', u'.*', u'\\', u'.\\', u'&', u'∩',
175 # prec-rational
176 u'//', u'.//',
177 # prec-power
178 u'^', u'.^',
179 # prec-decl
180 u'::',
181 # prec-dot
182 u'.',
183 # unary op
184 u'+', u'-', u'!', u'~', u'√', u'∛', u'∜'
185 ]), Operator),
80 186
81 # chars 187 # chars
82 (r"'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,3}|\\u[a-fA-F0-9]{1,4}|" 188 (r"'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,3}|\\u[a-fA-F0-9]{1,4}|"
83 r"\\U[a-fA-F0-9]{1,6}|[^\\\'\n])'", String.Char), 189 r"\\U[a-fA-F0-9]{1,6}|[^\\\'\n])'", String.Char),
84 190
85 # try to match trailing transpose 191 # try to match trailing transpose
86 (r'(?<=[.\w)\]])\'+', Operator), 192 (r'(?<=[.\w)\]])\'+', Operator),
87 193
88 # strings 194 # strings
89 (r'(?:[IL])"', String, 'string'), 195 (r'"""', String, 'tqstring'),
90 (r'[E]?"', String, combined('stringescape', 'string')), 196 (r'"', String, 'string'),
197
198 # regular expressions
199 (r'r"""', String.Regex, 'tqregex'),
200 (r'r"', String.Regex, 'regex'),
201
202 # backticks
203 (r'`', String.Backtick, 'command'),
91 204
92 # names 205 # names
93 (r'@[\w.]+', Name.Decorator), 206 (allowed_variable, Name),
94 (u'(?:[a-zA-Z_\u00A1-\uffff]|%s)(?:[a-zA-Z_0-9\u00A1-\uffff]|%s)*!*' % 207 (r'@' + allowed_variable, Name.Decorator),
95 ((unirange(0x10000, 0x10ffff),)*2), Name),
96 208
97 # numbers 209 # numbers
98 (r'(\d+(_\d+)+\.\d*|\d*\.\d+(_\d+)+)([eEf][+-]?[0-9]+)?', Number.Float), 210 (r'(\d+(_\d+)+\.\d*|\d*\.\d+(_\d+)+)([eEf][+-]?[0-9]+)?', Number.Float),
99 (r'(\d+\.\d*|\d*\.\d+)([eEf][+-]?[0-9]+)?', Number.Float), 211 (r'(\d+\.\d*|\d*\.\d+)([eEf][+-]?[0-9]+)?', Number.Float),
100 (r'\d+(_\d+)+[eEf][+-]?[0-9]+', Number.Float), 212 (r'\d+(_\d+)+[eEf][+-]?[0-9]+', Number.Float),
107 (r'0x[a-fA-F0-9]+', Number.Hex), 219 (r'0x[a-fA-F0-9]+', Number.Hex),
108 (r'\d+(_\d+)+', Number.Integer), 220 (r'\d+(_\d+)+', Number.Integer),
109 (r'\d+', Number.Integer) 221 (r'\d+', Number.Integer)
110 ], 222 ],
111 223
112 'funcname': [
113 ('[a-zA-Z_]\w*', Name.Function, '#pop'),
114 ('\([^\s\w{]{1,2}\)', Operator, '#pop'),
115 ('[^\s\w{]{1,2}', Operator, '#pop'),
116 ],
117
118 'typename': [
119 ('[a-zA-Z_]\w*', Name.Class, '#pop')
120 ],
121
122 'stringescape': [
123 (r'\\([\\abfnrtv"\']|\n|N\{.*?\}|u[a-fA-F0-9]{4}|'
124 r'U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape)
125 ],
126 "blockcomment": [ 224 "blockcomment": [
127 (r'[^=#]', Comment.Multiline), 225 (r'[^=#]', Comment.Multiline),
128 (r'#=', Comment.Multiline, '#push'), 226 (r'#=', Comment.Multiline, '#push'),
129 (r'=#', Comment.Multiline, '#pop'), 227 (r'=#', Comment.Multiline, '#pop'),
130 (r'[=#]', Comment.Multiline), 228 (r'[=#]', Comment.Multiline),
131 ], 229 ],
230
132 'string': [ 231 'string': [
133 (r'"', String, '#pop'), 232 (r'"', String, '#pop'),
134 (r'\\\\|\\"|\\\n', String.Escape), # included here for raw strings 233 # FIXME: This escape pattern is not perfect.
234 (r'\\([\\"\'$nrbtfav]|(x|u|U)[a-fA-F0-9]+|\d+)', String.Escape),
135 # Interpolation is defined as "$" followed by the shortest full 235 # Interpolation is defined as "$" followed by the shortest full
136 # expression, which is something we can't parse. 236 # expression, which is something we can't parse.
137 # Include the most common cases here: $word, and $(paren'd expr). 237 # Include the most common cases here: $word, and $(paren'd expr).
138 (r'\$[a-zA-Z_]+', String.Interpol), 238 (r'\$' + allowed_variable, String.Interpol),
139 (r'\$\(', String.Interpol, 'in-intp'), 239 # (r'\$[a-zA-Z_]+', String.Interpol),
240 (r'(\$)(\()', bygroups(String.Interpol, Punctuation), 'in-intp'),
140 # @printf and @sprintf formats 241 # @printf and @sprintf formats
141 (r'%[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?[hlL]?[diouxXeEfFgGcrs%]', 242 (r'%[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?[hlL]?[E-GXc-giorsux%]',
142 String.Interpol), 243 String.Interpol),
143 (r'[^$%"\\]+', String), 244 (r'.|\s', String),
144 # unhandled special signs 245 ],
145 (r'[$%"\\]', String), 246
146 ], 247 'tqstring': [
248 (r'"""', String, '#pop'),
249 (r'\\([\\"\'$nrbtfav]|(x|u|U)[a-fA-F0-9]+|\d+)', String.Escape),
250 (r'\$' + allowed_variable, String.Interpol),
251 (r'(\$)(\()', bygroups(String.Interpol, Punctuation), 'in-intp'),
252 (r'.|\s', String),
253 ],
254
255 'regex': [
256 (r'"', String.Regex, '#pop'),
257 (r'\\"', String.Regex),
258 (r'.|\s', String.Regex),
259 ],
260
261 'tqregex': [
262 (r'"""', String.Regex, '#pop'),
263 (r'.|\s', String.Regex),
264 ],
265
266 'command': [
267 (r'`', String.Backtick, '#pop'),
268 (r'\$' + allowed_variable, String.Interpol),
269 (r'(\$)(\()', bygroups(String.Interpol, Punctuation), 'in-intp'),
270 (r'.|\s', String.Backtick)
271 ],
272
147 'in-intp': [ 273 'in-intp': [
148 (r'[^()]+', String.Interpol), 274 (r'\(', Punctuation, '#push'),
149 (r'\(', String.Interpol, '#push'), 275 (r'\)', Punctuation, '#pop'),
150 (r'\)', String.Interpol, '#pop'), 276 include('root'),
151 ] 277 ]
152 } 278 }
153 279
154 def analyse_text(text): 280 def analyse_text(text):
155 return shebang_matches(text, r'julia') 281 return shebang_matches(text, r'julia')
156
157
158 line_re = re.compile('.*?\n')
159 282
160 283
161 class JuliaConsoleLexer(Lexer): 284 class JuliaConsoleLexer(Lexer):
162 """ 285 """
163 For Julia console sessions. Modeled after MatlabSessionLexer. 286 For Julia console sessions. Modeled after MatlabSessionLexer.
167 name = 'Julia console' 290 name = 'Julia console'
168 aliases = ['jlcon'] 291 aliases = ['jlcon']
169 292
170 def get_tokens_unprocessed(self, text): 293 def get_tokens_unprocessed(self, text):
171 jllexer = JuliaLexer(**self.options) 294 jllexer = JuliaLexer(**self.options)
172 295 start = 0
173 curcode = '' 296 curcode = ''
174 insertions = [] 297 insertions = []
175 298 output = False
176 for match in line_re.finditer(text): 299 error = False
177 line = match.group() 300
178 301 for line in text.splitlines(True):
179 if line.startswith('julia>'): 302 if line.startswith('julia>'):
180 insertions.append((len(curcode), 303 insertions.append((len(curcode), [(0, Generic.Prompt, line[:6])]))
181 [(0, Generic.Prompt, line[:6])]))
182 curcode += line[6:] 304 curcode += line[6:]
183 305 output = False
184 elif line.startswith(' '): 306 error = False
185 307 elif line.startswith('help?>') or line.startswith('shell>'):
186 idx = len(curcode) 308 yield start, Generic.Prompt, line[:6]
187 309 yield start + 6, Text, line[6:]
188 # without is showing error on same line as before...? 310 output = False
189 line = "\n" + line 311 error = False
190 token = (0, Generic.Traceback, line) 312 elif line.startswith(' ') and not output:
191 insertions.append((idx, [token])) 313 insertions.append((len(curcode), [(0, Text, line[:6])]))
192 314 curcode += line[6:]
193 else: 315 else:
194 if curcode: 316 if curcode:
195 for item in do_insertions( 317 for item in do_insertions(
196 insertions, jllexer.get_tokens_unprocessed(curcode)): 318 insertions, jllexer.get_tokens_unprocessed(curcode)):
197 yield item 319 yield item
198 curcode = '' 320 curcode = ''
199 insertions = [] 321 insertions = []
200 322 if line.startswith('ERROR: ') or error:
201 yield match.start(), Generic.Output, line 323 yield start, Generic.Error, line
202 324 error = True
203 if curcode: # or item: 325 else:
326 yield start, Generic.Output, line
327 output = True
328 start += len(line)
329
330 if curcode:
204 for item in do_insertions( 331 for item in do_insertions(
205 insertions, jllexer.get_tokens_unprocessed(curcode)): 332 insertions, jllexer.get_tokens_unprocessed(curcode)):
206 yield item 333 yield item

Mercurial Repository: eric

eric ide

mercurial