--- a/ThirdParty/Pygments/pygments/lexer.py Sun Feb 17 19:05:40 2013 +0100
+++ b/ThirdParty/Pygments/pygments/lexer.py Sun Feb 17 19:07:15 2013 +0100
@@ -5,10 +5,10 @@
Base lexer classes.
- :copyright: Copyright 2006-2012 by the Pygments team, see AUTHORS.
+ :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
-import re
+import re, itertools
from pygments.filter import apply_filters, Filter
from pygments.filters import get_filter_by_name
@@ -19,7 +19,7 @@
__all__ = ['Lexer', 'RegexLexer', 'ExtendedRegexLexer', 'DelegatingLexer',
- 'LexerContext', 'include', 'bygroups', 'using', 'this']
+ 'LexerContext', 'include', 'inherit', 'bygroups', 'using', 'this']
_encoding_map = [('\xef\xbb\xbf', 'utf-8'),
@@ -73,15 +73,18 @@
#: Shortcuts for the lexer
aliases = []
- #: fn match rules
+ #: File name globs
filenames = []
- #: fn alias filenames
+ #: Secondary file name globs
alias_filenames = []
- #: mime types
+ #: MIME types
mimetypes = []
+ #: Priority, should multiple lexers match and no content is provided
+ priority = 0
+
def __init__(self, **options):
self.options = options
self.stripnl = get_bool_opt(options, 'stripnl', True)
@@ -162,6 +165,10 @@
text = decoded
else:
text = text.decode(self.encoding)
+ else:
+ if text.startswith('\ufeff'):
+ text = text[len('\ufeff'):]
+
# text now *is* a unicode string
text = text.replace('\r\n', '\n')
text = text.replace('\r', '\n')
@@ -237,6 +244,16 @@
pass
+class _inherit(object):
+ """
+ Indicates the a state should inherit from its superclass.
+ """
+ def __repr__(self):
+ return 'inherit'
+
+inherit = _inherit()
+
+
class combined(tuple):
"""
Indicates a state combined from multiple states.
@@ -427,6 +444,9 @@
tokens.extend(cls._process_state(unprocessed, processed,
str(tdef)))
continue
+ if isinstance(tdef, _inherit):
+ # processed already
+ continue
assert type(tdef) is tuple, "wrong rule def %r" % tdef
@@ -455,6 +475,49 @@
cls._process_state(tokendefs, processed, state)
return processed
+ def get_tokendefs(cls):
+ """
+ Merge tokens from superclasses in MRO order, returning a single tokendef
+ dictionary.
+
+ Any state that is not defined by a subclass will be inherited
+ automatically. States that *are* defined by subclasses will, by
+ default, override that state in the superclass. If a subclass wishes to
+ inherit definitions from a superclass, it can use the special value
+ "inherit", which will cause the superclass' state definition to be
+ included at that point in the state.
+ """
+ tokens = {}
+ inheritable = {}
+ for c in itertools.chain((cls,), cls.__mro__):
+ toks = c.__dict__.get('tokens', {})
+
+ for state, items in toks.items():
+ curitems = tokens.get(state)
+ if curitems is None:
+ tokens[state] = items
+ try:
+ inherit_ndx = items.index(inherit)
+ except ValueError:
+ continue
+ inheritable[state] = inherit_ndx
+ continue
+
+ inherit_ndx = inheritable.pop(state, None)
+ if inherit_ndx is None:
+ continue
+
+ # Replace the "inherit" value with the items
+ curitems[inherit_ndx:inherit_ndx+1] = items
+ try:
+ new_inh_ndx = items.index(inherit)
+ except ValueError:
+ pass
+ else:
+ inheritable[state] = inherit_ndx + new_inh_ndx
+
+ return tokens
+
def __call__(cls, *args, **kwds):
"""Instantiate cls after preprocessing its token definitions."""
if '_tokens' not in cls.__dict__:
@@ -464,7 +527,7 @@
# don't process yet
pass
else:
- cls._tokens = cls.process_tokendef('', cls.tokens)
+ cls._tokens = cls.process_tokendef('', cls.get_tokendefs())
return type.__call__(cls, *args, **kwds)
@@ -542,10 +605,10 @@
try:
if text[pos] == '\n':
# at EOL, reset state to "root"
- pos += 1
statestack = ['root']
statetokens = tokendefs['root']
yield pos, Text, '\n'
+ pos += 1
continue
yield pos, Error, text[pos]
pos += 1
@@ -604,7 +667,13 @@
if new_state is not None:
# state transition
if isinstance(new_state, tuple):
- ctx.stack.extend(new_state)
+ for state in new_state:
+ if state == '#pop':
+ ctx.stack.pop()
+ elif state == '#push':
+ ctx.stack.append(statestack[-1])
+ else:
+ ctx.stack.append(state)
elif isinstance(new_state, int):
# pop
del ctx.stack[new_state:]
@@ -620,10 +689,10 @@
break
if text[ctx.pos] == '\n':
# at EOL, reset state to "root"
- ctx.pos += 1
ctx.stack = ['root']
statetokens = tokendefs['root']
yield ctx.pos, Text, '\n'
+ ctx.pos += 1
continue
yield ctx.pos, Error, text[ctx.pos]
ctx.pos += 1
@@ -692,4 +761,3 @@
except StopIteration:
insleft = False
break # not strictly necessary
-
