eric: comparison ThirdParty/Pygments/pygments/lexer.py

-:ace8a08028f3
+:da76c71624de
 pygments.lexer
 ~~~~~~~~~~~~~~
 Base lexer classes.
-:copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS.
+:copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
 :license: BSD, see LICENSE for details.
 """
-import re
+import re, itertools
 from pygments.filter import apply_filters, Filter
 from pygments.filters import get_filter_by_name
 from pygments.token import Error, Text, Other, _TokenType
 from pygments.util import get_bool_opt, get_int_opt, get_list_opt, \
 make_analysator
 import collections
 __all__ = ['Lexer', 'RegexLexer', 'ExtendedRegexLexer', 'DelegatingLexer',
-'LexerContext', 'include', 'bygroups', 'using', 'this']
+'LexerContext', 'include', 'inherit', 'bygroups', 'using', 'this']
 _encoding_map = [('\xef\xbb\xbf', 'utf-8'),
 ('\xff\xfe\0\0', 'utf-32'),
 ('\0\0\xfe\xff', 'utf-32be'),
 name = None
 #: Shortcuts for the lexer
 aliases = []
-#: fn match rules
+#: File name globs
 filenames = []
-#: fn alias filenames
+#: Secondary file name globs
 alias_filenames = []
-#: mime types
+#: MIME types
 mimetypes = []
+#: Priority, should multiple lexers match and no content is provided
+priority = 0
 def __init__(self, **options):
 self.options = options
 self.stripnl = get_bool_opt(options, 'stripnl', True)
 self.stripall = get_bool_opt(options, 'stripall', False)
 decoded = str(text, enc.get('encoding') or 'utf-8',
 errors='replace')
 text = decoded
 else:
 text = text.decode(self.encoding)
+else:
+if text.startswith('\ufeff'):
+text = text[len('\ufeff'):]
 # text now *is* a unicode string
 text = text.replace('\r\n', '\n')
 text = text.replace('\r', '\n')
 if self.stripall:
 text = text.strip()
 class include(str):
 """
 Indicates that a state should include rules from another state.
 """
 pass
+class _inherit(object):
+"""
+Indicates the a state should inherit from its superclass.
+"""
+def __repr__(self):
+return 'inherit'
+inherit = _inherit()
 class combined(tuple):
 """
 Indicates a state combined from multiple states.
 # it's a state reference
 assert tdef != state, "circular state reference %r" % state
 tokens.extend(cls._process_state(unprocessed, processed,
 str(tdef)))
 continue
+if isinstance(tdef, _inherit):
+# processed already
+continue
 assert type(tdef) is tuple, "wrong rule def %r" % tdef
 try:
 rex = cls._process_regex(tdef[0], rflags)
 processed = cls._all_tokens[name] = {}
 tokendefs = tokendefs or cls.tokens[name]
 for state in list(tokendefs.keys()):
 cls._process_state(tokendefs, processed, state)
 return processed
+def get_tokendefs(cls):
+"""
+Merge tokens from superclasses in MRO order, returning a single tokendef
+dictionary.
+Any state that is not defined by a subclass will be inherited
+automatically.  States that *are* defined by subclasses will, by
+default, override that state in the superclass.  If a subclass wishes to
+inherit definitions from a superclass, it can use the special value
+"inherit", which will cause the superclass' state definition to be
+included at that point in the state.
+"""
+tokens = {}
+inheritable = {}
+for c in itertools.chain((cls,), cls.__mro__):
+toks = c.__dict__.get('tokens', {})
+for state, items in toks.items():
+curitems = tokens.get(state)
+if curitems is None:
+tokens[state] = items
+try:
+inherit_ndx = items.index(inherit)
+except ValueError:
+continue
+inheritable[state] = inherit_ndx
+continue
+inherit_ndx = inheritable.pop(state, None)
+if inherit_ndx is None:
+continue
+# Replace the "inherit" value with the items
+curitems[inherit_ndx:inherit_ndx+1] = items
+try:
+new_inh_ndx = items.index(inherit)
+except ValueError:
+pass
+else:
+inheritable[state] = inherit_ndx + new_inh_ndx
+return tokens
 def __call__(cls, *args, **kwds):
 """Instantiate cls after preprocessing its token definitions."""
 if '_tokens' not in cls.__dict__:
 cls._all_tokens = {}
 cls._tmpname = 0
 if hasattr(cls, 'token_variants') and cls.token_variants:
 # don't process yet
 pass
 else:
-cls._tokens = cls.process_tokendef('', cls.tokens)
+cls._tokens = cls.process_tokendef('', cls.get_tokendefs())
 return type.__call__(cls, *args, **kwds)
 class RegexLexer(Lexer, metaclass=RegexLexerMeta):
 break
 else:
 try:
 if text[pos] == '\n':
 # at EOL, reset state to "root"
-pos += 1
 statestack = ['root']
 statetokens = tokendefs['root']
 yield pos, Text, '\n'
+pos += 1
 continue
 yield pos, Error, text[pos]
 pos += 1
 except IndexError:
 break
 statetokens = tokendefs[ctx.stack[-1]]
 # CAUTION: callback must set ctx.pos!
 if new_state is not None:
 # state transition
 if isinstance(new_state, tuple):
-ctx.stack.extend(new_state)
+for state in new_state:
+if state == '#pop':
+ctx.stack.pop()
+elif state == '#push':
+ctx.stack.append(statestack[-1])
+else:
+ctx.stack.append(state)
 elif isinstance(new_state, int):
 # pop
 del ctx.stack[new_state:]
 elif new_state == '#push':
 ctx.stack.append(ctx.stack[-1])
 try:
 if ctx.pos >= ctx.end:
 break
 if text[ctx.pos] == '\n':
 # at EOL, reset state to "root"
-ctx.pos += 1
 ctx.stack = ['root']
 statetokens = tokendefs['root']
 yield ctx.pos, Text, '\n'
+ctx.pos += 1
 continue
 yield ctx.pos, Error, text[ctx.pos]
 ctx.pos += 1
 except IndexError:
 break
 try:
 index, itokens = next(insertions)
 except StopIteration:
 insleft = False
 break  # not strictly necessary

Mercurial Repositories > eric / file comparison

comparison: ThirdParty/Pygments/pygments/lexer.py

ThirdParty/Pygments/pygments/lexer.py