eric: comparison eric6/ThirdParty/Pygments/pygments/lexer.py

-:bf5f777260a6
+:21b0534faebc
 pygments.lexer
 ~~~~~~~~~~~~~~
 Base lexer classes.
-:copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS.
+:copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS.
 :license: BSD, see LICENSE for details.
 """
-from __future__ import print_function
 import re
 import sys
 import time
 from pygments.filter import apply_filters, Filter
 from pygments.filters import get_filter_by_name
 from pygments.token import Error, Text, Other, _TokenType
 from pygments.util import get_bool_opt, get_int_opt, get_list_opt, \
-make_analysator, text_type, add_metaclass, iteritems, Future, guess_decode
+make_analysator, Future, guess_decode
 from pygments.regexopt import regex_opt
 __all__ = ['Lexer', 'RegexLexer', 'ExtendedRegexLexer', 'DelegatingLexer',
 'LexerContext', 'include', 'inherit', 'bygroups', 'using', 'this',
 'default', 'words']
 if 'analyse_text' in d:
 d['analyse_text'] = make_analysator(d['analyse_text'])
 return type.__new__(mcs, name, bases, d)
-@add_metaclass(LexerMeta)
+class Lexer(metaclass=LexerMeta):
-class Lexer(object):
 """
 Lexer for a specific language.
 Basic options recognized:
 ``stripnl``
 is bypassed even if filters are defined.
 Also preprocess the text, i.e. expand tabs and strip it if
 wanted and applies registered filters.
 """
-if not isinstance(text, text_type):
+if not isinstance(text, str):
 if self.encoding == 'guess':
 text, _ = guess_decode(text)
 elif self.encoding == 'chardet':
 try:
 import chardet
 Indicates that a state should include rules from another state.
 """
 pass
-class _inherit(object):
+class _inherit:
 """
 Indicates the a state should inherit from its superclass.
 """
 def __repr__(self):
 return 'inherit'
 def __init__(self, *args):
 # tuple.__init__ doesn't do anything
 pass
-class _PseudoMatch(object):
+class _PseudoMatch:
 """
 A pseudo match object constructed from a string.
 """
 def __init__(self, start, text):
 if ctx:
 ctx.pos = match.end()
 return callback
-class _This(object):
+class _This:
 """
 Special singleton used for indicating the caller class.
 Used by ``using``.
 """
 this = _This()
 def using(_other, **kwargs):
 """
 tokens = {}
 inheritable = {}
 for c in cls.__mro__:
 toks = c.__dict__.get('tokens', {})
-for state, items in iteritems(toks):
+for state, items in toks.items():
 curitems = tokens.get(state)
 if curitems is None:
 # N.b. because this is assigned by reference, sufficiently
 # deep hierarchies are processed incrementally (e.g. for
 # A(B), B(C), C(RegexLexer), B will be premodified so X(B)
 cls._tokens = cls.process_tokendef('', cls.get_tokendefs())
 return type.__call__(cls, *args, **kwds)
-@add_metaclass(RegexLexerMeta)
+class RegexLexer(Lexer, metaclass=RegexLexerMeta):
-class RegexLexer(Lexer):
 """
 Base for simple stateful regular expression-based lexers.
 Simplifies the lexing process so that you need only
 provide a list of states and regular expressions.
 """
 if new_state is not None:
 # state transition
 if isinstance(new_state, tuple):
 for state in new_state:
 if state == '#pop':
-statestack.pop()
+if len(statestack) > 1:
+statestack.pop()
 elif state == '#push':
 statestack.append(statestack[-1])
 else:
 statestack.append(state)
 elif isinstance(new_state, int):
-# pop
+# pop, but keep at least one state on the stack
-del statestack[new_state:]
+# (random code leading to unexpected pops should
+# not allow exceptions)
+if abs(new_state) >= len(statestack):
+del statestack[1:]
+else:
+del statestack[new_state:]
 elif new_state == '#push':
 statestack.append(statestack[-1])
 else:
 assert False, "wrong state def: %r" % new_state
 statetokens = tokendefs[statestack[-1]]
 pos += 1
 except IndexError:
 break
-class LexerContext(object):
+class LexerContext:
 """
 A helper object that holds lexer position data.
 """
 def __init__(self, text, pos, stack=None, end=None):
 if new_state is not None:
 # state transition
 if isinstance(new_state, tuple):
 for state in new_state:
 if state == '#pop':
-ctx.stack.pop()
+if len(ctx.stack) > 1:
+ctx.stack.pop()
 elif state == '#push':
 ctx.stack.append(ctx.stack[-1])
 else:
 ctx.stack.append(state)
 elif isinstance(new_state, int):
-# pop
+# see RegexLexer for why this check is made
-del ctx.stack[new_state:]
+if abs(new_state) >= len(ctx.stack):
+del ctx.state[1:]
+else:
+del ctx.stack[new_state:]
 elif new_state == '#push':
 ctx.stack.append(ctx.stack[-1])
 else:
 assert False, "wrong state def: %r" % new_state
 statetokens = tokendefs[ctx.stack[-1]]
 info[1] += t1 - t0
 return res
 return match_func
-@add_metaclass(ProfilingRegexLexerMeta)
+class ProfilingRegexLexer(RegexLexer, metaclass=ProfilingRegexLexerMeta):
-class ProfilingRegexLexer(RegexLexer):
 """Drop-in replacement for RegexLexer that does profiling of its regexes."""
 _prof_data = []
 _prof_sort_index = 4  # defaults to time per call

Mercurial Repositories > eric / file comparison

comparison: eric6/ThirdParty/Pygments/pygments/lexer.py

eric6/ThirdParty/Pygments/pygments/lexer.py