Mercurial Repositories > eric / file comparison
comparison: ThirdParty/Pygments/pygments/lexer.py
ThirdParty/Pygments/pygments/lexer.py
- changeset 4697
- c2e9bf425554
- parent 4172
- 4f20dba37ab6
- child 5713
- 6762afd9f963
equal
deleted
inserted
replaced
| 4696:bf4d19a7cade | 4697:c2e9bf425554 |
|---|---|
| 3 pygments.lexer | 3 pygments.lexer |
| 4 ~~~~~~~~~~~~~~ | 4 ~~~~~~~~~~~~~~ |
| 5 | 5 |
| 6 Base lexer classes. | 6 Base lexer classes. |
| 7 | 7 |
| 8 :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS. | 8 :copyright: Copyright 2006-2015 by the Pygments team, see AUTHORS. |
| 9 :license: BSD, see LICENSE for details. | 9 :license: BSD, see LICENSE for details. |
| 10 """ | 10 """ |
| 11 | 11 |
| 12 from __future__ import print_function | 12 from __future__ import print_function |
| 13 | 13 |
| 14 import re | 14 import re |
| 15 import sys | 15 import sys |
| 16 import time | 16 import time |
| 17 import itertools | |
| 18 | 17 |
| 19 from pygments.filter import apply_filters, Filter | 18 from pygments.filter import apply_filters, Filter |
| 20 from pygments.filters import get_filter_by_name | 19 from pygments.filters import get_filter_by_name |
| 21 from pygments.token import Error, Text, Other, _TokenType | 20 from pygments.token import Error, Text, Other, _TokenType |
| 22 from pygments.util import get_bool_opt, get_int_opt, get_list_opt, \ | 21 from pygments.util import get_bool_opt, get_int_opt, get_list_opt, \ |
| 41 """ | 40 """ |
| 42 This metaclass automagically converts ``analyse_text`` methods into | 41 This metaclass automagically converts ``analyse_text`` methods into |
| 43 static methods which always return float values. | 42 static methods which always return float values. |
| 44 """ | 43 """ |
| 45 | 44 |
| 46 def __new__(cls, name, bases, d): | 45 def __new__(mcs, name, bases, d): |
| 47 if 'analyse_text' in d: | 46 if 'analyse_text' in d: |
| 48 d['analyse_text'] = make_analysator(d['analyse_text']) | 47 d['analyse_text'] = make_analysator(d['analyse_text']) |
| 49 return type.__new__(cls, name, bases, d) | 48 return type.__new__(mcs, name, bases, d) |
| 50 | 49 |
| 51 | 50 |
| 52 @add_metaclass(LexerMeta) | 51 @add_metaclass(LexerMeta) |
| 53 class Lexer(object): | 52 class Lexer(object): |
| 54 """ | 53 """ |
| 187 text = text.expandtabs(self.tabsize) | 186 text = text.expandtabs(self.tabsize) |
| 188 if self.ensurenl and not text.endswith('\n'): | 187 if self.ensurenl and not text.endswith('\n'): |
| 189 text += '\n' | 188 text += '\n' |
| 190 | 189 |
| 191 def streamer(): | 190 def streamer(): |
| 192 for i, t, v in self.get_tokens_unprocessed(text): | 191 for _, t, v in self.get_tokens_unprocessed(text): |
| 193 yield t, v | 192 yield t, v |
| 194 stream = streamer() | 193 stream = streamer() |
| 195 if not unfiltered: | 194 if not unfiltered: |
| 196 stream = apply_filters(stream, self.filters, self) | 195 stream = apply_filters(stream, self.filters, self) |
| 197 return stream | 196 return stream |
| 244 # ------------------------------------------------------------------------------ | 243 # ------------------------------------------------------------------------------ |
| 245 # RegexLexer and ExtendedRegexLexer | 244 # RegexLexer and ExtendedRegexLexer |
| 246 # | 245 # |
| 247 | 246 |
| 248 | 247 |
| 249 class include(str): | 248 class include(str): # pylint: disable=invalid-name |
| 250 """ | 249 """ |
| 251 Indicates that a state should include rules from another state. | 250 Indicates that a state should include rules from another state. |
| 252 """ | 251 """ |
| 253 pass | 252 pass |
| 254 | 253 |
| 258 Indicates the a state should inherit from its superclass. | 257 Indicates the a state should inherit from its superclass. |
| 259 """ | 258 """ |
| 260 def __repr__(self): | 259 def __repr__(self): |
| 261 return 'inherit' | 260 return 'inherit' |
| 262 | 261 |
| 263 inherit = _inherit() | 262 inherit = _inherit() # pylint: disable=invalid-name |
| 264 | 263 |
| 265 | 264 |
| 266 class combined(tuple): | 265 class combined(tuple): # pylint: disable=invalid-name |
| 267 """ | 266 """ |
| 268 Indicates a state combined from multiple states. | 267 Indicates a state combined from multiple states. |
| 269 """ | 268 """ |
| 270 | 269 |
| 271 def __new__(cls, *args): | 270 def __new__(cls, *args): |
| 318 else: | 317 else: |
| 319 data = match.group(i + 1) | 318 data = match.group(i + 1) |
| 320 if data is not None: | 319 if data is not None: |
| 321 if ctx: | 320 if ctx: |
| 322 ctx.pos = match.start(i + 1) | 321 ctx.pos = match.start(i + 1) |
| 323 for item in action(lexer, _PseudoMatch(match.start(i + 1), | 322 for item in action( |
| 324 data), ctx): | 323 lexer, _PseudoMatch(match.start(i + 1), data), ctx): |
| 325 if item: | 324 if item: |
| 326 yield item | 325 yield item |
| 327 if ctx: | 326 if ctx: |
| 328 ctx.pos = match.end() | 327 ctx.pos = match.end() |
| 329 return callback | 328 return callback |
| 653 else: | 652 else: |
| 654 assert False, "wrong state def: %r" % new_state | 653 assert False, "wrong state def: %r" % new_state |
| 655 statetokens = tokendefs[statestack[-1]] | 654 statetokens = tokendefs[statestack[-1]] |
| 656 break | 655 break |
| 657 else: | 656 else: |
| 657 # We are here only if all state tokens have been considered | |
| 658 # and there was not a match on any of them. | |
| 658 try: | 659 try: |
| 659 if text[pos] == '\n': | 660 if text[pos] == '\n': |
| 660 # at EOL, reset state to "root" | 661 # at EOL, reset state to "root" |
| 661 statestack = ['root'] | 662 statestack = ['root'] |
| 662 statetokens = tokendefs['root'] | 663 statetokens = tokendefs['root'] |
