Mercurial Repositories > eric / file comparison

comparison: ThirdParty/Pygments/pygments/lexers/clean.py

ThirdParty/Pygments/pygments/lexers/clean.py

changeset 6651
e8f3b5568b21
parent 5713
6762afd9f963
equal deleted inserted replaced
6650:1dd52aa8897c 6651:e8f3b5568b21
3 pygments.lexers.clean 3 pygments.lexers.clean
4 ~~~~~~~~~~~~~~~~~~~~~ 4 ~~~~~~~~~~~~~~~~~~~~~
5 5
6 Lexer for the Clean language. 6 Lexer for the Clean language.
7 7
8 :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. 8 :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS.
9 :license: BSD, see LICENSE for details. 9 :license: BSD, see LICENSE for details.
10 """ 10 """
11 11
12 from pygments.lexer import ExtendedRegexLexer, LexerContext, \ 12 from pygments.lexer import ExtendedRegexLexer, words, include, bygroups
13 bygroups, words, include, default 13 from pygments.token import Comment, Error, Keyword, Literal, Name, Number, \
14 from pygments.token import Comment, Keyword, Literal, Name, Number, Operator, \ 14 Operator, Punctuation, String, Whitespace
15 Punctuation, String, Text, Whitespace
16 15
17 __all__ = ['CleanLexer'] 16 __all__ = ['CleanLexer']
18 17
19 18
20 class CleanLexer(ExtendedRegexLexer): 19 class CleanLexer(ExtendedRegexLexer):
26 """ 25 """
27 name = 'Clean' 26 name = 'Clean'
28 aliases = ['clean'] 27 aliases = ['clean']
29 filenames = ['*.icl', '*.dcl'] 28 filenames = ['*.icl', '*.dcl']
30 29
31 def get_tokens_unprocessed(self, text=None, context=None): 30 keywords = (
32 ctx = LexerContext(text, 0) 31 'case', 'ccall', 'class', 'code', 'code inline', 'derive', 'export',
33 ctx.indent = 0 32 'foreign', 'generic', 'if', 'in', 'infix', 'infixl', 'infixr',
34 return ExtendedRegexLexer.get_tokens_unprocessed(self, text, context=ctx) 33 'instance', 'let', 'of', 'otherwise', 'special', 'stdcall', 'where',
34 'with')
35 35
36 def check_class_not_import(lexer, match, ctx): 36 modulewords = ('implementation', 'definition', 'system')
37 if match.group(0) == 'import':
38 yield match.start(), Keyword.Namespace, match.group(0)
39 ctx.stack = ctx.stack[:-1] + ['fromimportfunc']
40 else:
41 yield match.start(), Name.Class, match.group(0)
42 ctx.pos = match.end()
43 37
44 def check_instance_class(lexer, match, ctx): 38 lowerId = r'[a-z`][\w\d`]*'
45 if match.group(0) == 'instance' or match.group(0) == 'class': 39 upperId = r'[A-Z`][\w\d`]*'
46 yield match.start(), Keyword, match.group(0) 40 funnyId = r'[~@#\$%\^?!+\-*<>\\/|&=:]+'
47 else: 41 scoreUpperId = r'_' + upperId
48 yield match.start(), Name.Function, match.group(0) 42 scoreLowerId = r'_' + lowerId
49 ctx.stack = ctx.stack + ['fromimportfunctype'] 43 moduleId = r'[a-zA-Z_][a-zA-Z0-9_.`]+'
50 ctx.pos = match.end() 44 classId = '|'.join([lowerId, upperId, funnyId])
51
52 @staticmethod
53 def indent_len(text):
54 # Tabs are four spaces:
55 # https://svn.cs.ru.nl/repos/clean-platform/trunk/doc/STANDARDS.txt
56 text = text.replace('\n', '')
57 return len(text.replace('\t', ' ')), len(text)
58
59 def store_indent(lexer, match, ctx):
60 ctx.indent, _ = CleanLexer.indent_len(match.group(0))
61 ctx.pos = match.end()
62 yield match.start(), Text, match.group(0)
63
64 def check_indent1(lexer, match, ctx):
65 indent, reallen = CleanLexer.indent_len(match.group(0))
66 if indent > ctx.indent:
67 yield match.start(), Whitespace, match.group(0)
68 ctx.pos = match.start() + reallen + 1
69 else:
70 ctx.indent = 0
71 ctx.pos = match.start()
72 ctx.stack = ctx.stack[:-1]
73 yield match.start(), Whitespace, match.group(0)[1:]
74
75 def check_indent2(lexer, match, ctx):
76 indent, reallen = CleanLexer.indent_len(match.group(0))
77 if indent > ctx.indent:
78 yield match.start(), Whitespace, match.group(0)
79 ctx.pos = match.start() + reallen + 1
80 else:
81 ctx.indent = 0
82 ctx.pos = match.start()
83 ctx.stack = ctx.stack[:-2]
84
85 def check_indent3(lexer, match, ctx):
86 indent, reallen = CleanLexer.indent_len(match.group(0))
87 if indent > ctx.indent:
88 yield match.start(), Whitespace, match.group(0)
89 ctx.pos = match.start() + reallen + 1
90 else:
91 ctx.indent = 0
92 ctx.pos = match.start()
93 ctx.stack = ctx.stack[:-3]
94 yield match.start(), Whitespace, match.group(0)[1:]
95 if match.group(0) == '\n\n':
96 ctx.pos = ctx.pos + 1
97
98 def skip(lexer, match, ctx):
99 ctx.stack = ctx.stack[:-1]
100 ctx.pos = match.end()
101 yield match.start(), Comment, match.group(0)
102
103 keywords = ('class', 'instance', 'where', 'with', 'let', 'let!',
104 'in', 'case', 'of', 'infix', 'infixr', 'infixl', 'generic',
105 'derive', 'otherwise', 'code', 'inline')
106 45
107 tokens = { 46 tokens = {
108 'common': [ 47 'root': [
109 (r';', Punctuation, '#pop'), 48 include('comments'),
110 (r'//', Comment, 'singlecomment'), 49 include('keywords'),
50 include('module'),
51 include('import'),
52 include('whitespace'),
53 include('literals'),
54 include('operators'),
55 include('delimiters'),
56 include('names'),
111 ], 57 ],
112 'root': [ 58 'whitespace': [
113 # Comments 59 (r'\s+', Whitespace),
60 ],
61 'comments': [
114 (r'//.*\n', Comment.Single), 62 (r'//.*\n', Comment.Single),
115 (r'(?s)/\*\*.*?\*/', Comment.Special), 63 (r'/\*', Comment.Multi, 'comments.in'),
116 (r'(?s)/\*.*?\*/', Comment.Multi), 64 (r'/\*\*', Comment.Special, 'comments.in'),
117
118 # Modules, imports, etc.
119 (r'\b((?:implementation|definition|system)\s+)?(module)(\s+)([\w`.]+)',
120 bygroups(Keyword.Namespace, Keyword.Namespace, Text, Name.Class)),
121 (r'(?<=\n)import(?=\s)', Keyword.Namespace, 'import'),
122 (r'(?<=\n)from(?=\s)', Keyword.Namespace, 'fromimport'),
123
124 # Keywords
125 # We cannot use (?s)^|(?<=\s) as prefix, so need to repeat this
126 (words(keywords, prefix=r'(?<=\s)', suffix=r'(?=\s)'), Keyword),
127 (words(keywords, prefix=r'^', suffix=r'(?=\s)'), Keyword),
128
129 # Function definitions
130 (r'(?=\{\|)', Whitespace, 'genericfunction'),
131 (r'(?<=\n)([ \t]*)([\w`$()=\-<>~*\^|+&%]+)((?:\s+\w)*)(\s*)(::)',
132 bygroups(store_indent, Name.Function, Keyword.Type, Whitespace,
133 Punctuation),
134 'functiondefargs'),
135
136 # Type definitions
137 (r'(?<=\n)([ \t]*)(::)', bygroups(store_indent, Punctuation), 'typedef'),
138 (r'^([ \t]*)(::)', bygroups(store_indent, Punctuation), 'typedef'),
139
140 # Literals
141 (r'\'\\?.(?<!\\)\'', String.Char),
142 (r'\'\\\d+\'', String.Char),
143 (r'\'\\\\\'', String.Char), # (special case for '\\')
144 (r'[+\-~]?\s*\d+\.\d+(E[+\-~]?\d+)?\b', Number.Float),
145 (r'[+\-~]?\s*0[0-7]\b', Number.Oct),
146 (r'[+\-~]?\s*0x[0-9a-fA-F]\b', Number.Hex),
147 (r'[+\-~]?\s*\d+\b', Number.Integer),
148 (r'"', String.Double, 'doubleqstring'),
149 (words(('True', 'False'), prefix=r'(?<=\s)', suffix=r'(?=\s)'),
150 Literal),
151
152 # Qualified names
153 (r'(\')([\w.]+)(\'\.)',
154 bygroups(Punctuation, Name.Namespace, Punctuation)),
155
156 # Everything else is some name
157 (r'([\w`$%/?@]+\.?)*[\w`$%/?@]+', Name),
158
159 # Punctuation
160 (r'[{}()\[\],:;.#]', Punctuation),
161 (r'[+\-=!<>|&~*\^/]', Operator),
162 (r'\\\\', Operator),
163
164 # Lambda expressions
165 (r'\\.*?(->|\.|=)', Name.Function),
166
167 # Whitespace
168 (r'\s', Whitespace),
169
170 include('common'),
171 ], 65 ],
172 'fromimport': [ 66 'comments.in': [
173 include('common'), 67 (r'\*\/', Comment.Multi, '#pop'),
174 (r'([\w`.]+)', check_class_not_import), 68 (r'/\*', Comment.Multi, '#push'),
175 (r'\n', Whitespace, '#pop'), 69 (r'[^*/]+', Comment.Multi),
176 (r'\s', Whitespace), 70 (r'\*(?!/)', Comment.Multi),
71 (r'/', Comment.Multi),
177 ], 72 ],
178 'fromimportfunc': [ 73 'keywords': [
179 include('common'), 74 (words(keywords, prefix=r'\b', suffix=r'\b'), Keyword),
180 (r'(::)(\s+)([^,\s]+)', bygroups(Punctuation, Text, Keyword.Type)),
181 (r'([\w`$()=\-<>~*\^|+&%/]+)', check_instance_class),
182 (r',', Punctuation),
183 (r'\n', Whitespace, '#pop'),
184 (r'\s', Whitespace),
185 ], 75 ],
186 'fromimportfunctype': [ 76 'module': [
187 include('common'), 77 (words(modulewords, prefix=r'\b', suffix=r'\b'), Keyword.Namespace),
188 (r'[{(\[]', Punctuation, 'combtype'), 78 (r'\bmodule\b', Keyword.Namespace, 'module.name'),
189 (r',', Punctuation, '#pop'),
190 (r'[:;.#]', Punctuation),
191 (r'\n', Whitespace, '#pop:2'),
192 (r'[^\S\n]+', Whitespace),
193 (r'\S+', Keyword.Type),
194 ], 79 ],
195 'combtype': [ 80 'module.name': [
196 include('common'), 81 include('whitespace'),
197 (r'[})\]]', Punctuation, '#pop'), 82 (moduleId, Name.Class, '#pop'),
198 (r'[{(\[]', Punctuation, '#pop'),
199 (r'[,:;.#]', Punctuation),
200 (r'\s+', Whitespace),
201 (r'\S+', Keyword.Type),
202 ], 83 ],
203 'import': [ 84 'import': [
204 include('common'), 85 (r'\b(import)\b(\s*)', bygroups(Keyword, Whitespace), 'import.module'),
205 (words(('from', 'import', 'as', 'qualified'), 86 (r'\b(from)\b(\s*)\b(' + moduleId + r')\b(\s*)\b(import)\b',
206 prefix='(?<=\s)', suffix='(?=\s)'), Keyword.Namespace), 87 bygroups(Keyword, Whitespace, Name.Class, Whitespace, Keyword),
207 (r'[\w`.]+', Name.Class), 88 'import.what'),
208 (r'\n', Whitespace, '#pop'), 89 ],
90 'import.module': [
91 (r'\b(qualified)\b(\s*)', bygroups(Keyword, Whitespace)),
92 (r'(\s*)\b(as)\b', bygroups(Whitespace, Keyword), ('#pop', 'import.module.as')),
93 (moduleId, Name.Class),
94 (r'(\s*)(,)(\s*)', bygroups(Whitespace, Punctuation, Whitespace)),
95 (r'\s*', Whitespace, '#pop'),
96 ],
97 'import.module.as': [
98 include('whitespace'),
99 (lowerId, Name.Class, '#pop'),
100 (upperId, Name.Class, '#pop'),
101 ],
102 'import.what': [
103 (r'\b(class)\b(\s+)(' + classId + r')',
104 bygroups(Keyword, Whitespace, Name.Class), 'import.what.class'),
105 (r'\b(instance)(\s+)(' + classId + r')(\s+)',
106 bygroups(Keyword, Whitespace, Name.Class, Whitespace), 'import.what.instance'),
107 (r'(::)(\s*)\b(' + upperId + r')\b',
108 bygroups(Punctuation, Whitespace, Name.Class), 'import.what.type'),
109 (r'\b(generic)\b(\s+)\b(' + lowerId + '|' + upperId + r')\b',
110 bygroups(Keyword, Whitespace, Name)),
111 include('names'),
112 (r'(,)(\s+)', bygroups(Punctuation, Whitespace)),
113 (r'$', Whitespace, '#pop'),
114 include('whitespace'),
115 ],
116 'import.what.class': [
117 (r',', Punctuation, '#pop'),
118 (r'\(', Punctuation, 'import.what.class.members'),
119 (r'$', Whitespace, '#pop:2'),
120 include('whitespace'),
121 ],
122 'import.what.class.members': [
209 (r',', Punctuation), 123 (r',', Punctuation),
210 (r'[^\S\n]+', Whitespace), 124 (r'\.\.', Punctuation),
125 (r'\)', Punctuation, '#pop'),
126 include('names'),
211 ], 127 ],
212 'singlecomment': [ 128 'import.what.instance': [
213 (r'(.)(?=\n)', skip), 129 (r'[,)]', Punctuation, '#pop'),
214 (r'.+(?!\n)', Comment), 130 (r'\(', Punctuation, 'import.what.instance'),
131 (r'$', Whitespace, '#pop:2'),
132 include('whitespace'),
133 include('names'),
215 ], 134 ],
216 'doubleqstring': [ 135 'import.what.type': [
217 (r'[^\\"]+', String.Double), 136 (r',', Punctuation, '#pop'),
137 (r'[({]', Punctuation, 'import.what.type.consesandfields'),
138 (r'$', Whitespace, '#pop:2'),
139 include('whitespace'),
140 ],
141 'import.what.type.consesandfields': [
142 (r',', Punctuation),
143 (r'\.\.', Punctuation),
144 (r'[)}]', Punctuation, '#pop'),
145 include('names'),
146 ],
147 'literals': [
148 (r'\'([^\'\\]|\\(x[\da-fA-F]+|\d+|.))\'', Literal.Char),
149 (r'[+~-]?0[0-7]+\b', Number.Oct),
150 (r'[+~-]?\d+\.\d+(E[+-]?\d+)?', Number.Float),
151 (r'[+~-]?\d+\b', Number.Integer),
152 (r'[+~-]?0x[\da-fA-F]+\b', Number.Hex),
153 (r'True|False', Literal),
154 (r'"', String.Double, 'literals.stringd'),
155 ],
156 'literals.stringd': [
157 (r'[^\\"\n]+', String.Double),
218 (r'"', String.Double, '#pop'), 158 (r'"', String.Double, '#pop'),
219 (r'\\.', String.Double), 159 (r'\\.', String.Double),
160 (r'[$\n]', Error, '#pop'),
220 ], 161 ],
221 'typedef': [ 162 'operators': [
222 include('common'), 163 (r'[-~@#\$%\^?!+*<>\\/|&=:\.]+', Operator),
223 (r'[\w`]+', Keyword.Type), 164 (r'\b_+\b', Operator),
224 (r'[:=|(),\[\]{}!*]', Punctuation),
225 (r'->', Punctuation),
226 (r'\n(?=[^\s|])', Whitespace, '#pop'),
227 (r'\s', Whitespace),
228 (r'.', Keyword.Type),
229 ], 165 ],
230 'genericfunction': [ 166 'delimiters': [
231 include('common'), 167 (r'[,;(){}\[\]]', Punctuation),
232 (r'\{\|', Punctuation), 168 (r'(\')([\w`.]+)(\')',
233 (r'\|\}', Punctuation, '#pop'), 169 bygroups(Punctuation, Name.Class, Punctuation)),
234 (r',', Punctuation),
235 (r'->', Punctuation),
236 (r'(\s+of\s+)(\{)', bygroups(Keyword, Punctuation), 'genericftypes'),
237 (r'\s', Whitespace),
238 (r'[\w`\[\]{}!]+', Keyword.Type),
239 (r'[*()]', Punctuation),
240 ], 170 ],
241 'genericftypes': [ 171 'names': [
242 include('common'), 172 (lowerId, Name),
243 (r'[\w`]+', Keyword.Type), 173 (scoreLowerId, Name),
244 (r',', Punctuation), 174 (funnyId, Name.Function),
245 (r'\s', Whitespace), 175 (upperId, Name.Class),
246 (r'\}', Punctuation, '#pop'), 176 (scoreUpperId, Name.Class),
247 ],
248 'functiondefargs': [
249 include('common'),
250 (r'\n(\s*)', check_indent1),
251 (r'[!{}()\[\],:;.#]', Punctuation),
252 (r'->', Punctuation, 'functiondefres'),
253 (r'^(?=\S)', Whitespace, '#pop'),
254 (r'\S', Keyword.Type),
255 (r'\s', Whitespace),
256 ],
257 'functiondefres': [
258 include('common'),
259 (r'\n(\s*)', check_indent2),
260 (r'^(?=\S)', Whitespace, '#pop:2'),
261 (r'[!{}()\[\],:;.#]', Punctuation),
262 (r'\|', Punctuation, 'functiondefclasses'),
263 (r'\S', Keyword.Type),
264 (r'\s', Whitespace),
265 ],
266 'functiondefclasses': [
267 include('common'),
268 (r'\n(\s*)', check_indent3),
269 (r'^(?=\S)', Whitespace, '#pop:3'),
270 (r'[,&]', Punctuation),
271 (r'\[', Punctuation, 'functiondefuniquneq'),
272 (r'[\w`$()=\-<>~*\^|+&%/{}\[\]@]', Name.Function, 'functionname'),
273 (r'\s+', Whitespace),
274 ],
275 'functiondefuniquneq': [
276 include('common'),
277 (r'[a-z]+', Keyword.Type),
278 (r'\s+', Whitespace),
279 (r'<=|,', Punctuation),
280 (r'\]', Punctuation, '#pop')
281 ],
282 'functionname': [
283 include('common'),
284 (r'[\w`$()=\-<>~*\^|+&%/]+', Name.Function),
285 (r'(?=\{\|)', Punctuation, 'genericfunction'),
286 default('#pop'),
287 ] 177 ]
288 } 178 }

Mercurial Repository: eric

eric ide

mercurial