diff -r 8bc578136279 -r 4f20dba37ab6 ThirdParty/Pygments/pygments/util.py
--- a/ThirdParty/Pygments/pygments/util.py Wed Mar 11 18:25:37 2015 +0100
+++ b/ThirdParty/Pygments/pygments/util.py Wed Mar 11 18:32:27 2015 +0100
@@ -5,37 +5,31 @@
Utility functions.
- :copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+ :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
:license: BSD, see LICENSE for details.
"""
-from __future__ import unicode_literals
-try:
- chr = unichr
-except NameError:
- pass
-
import re
import sys
-import codecs
split_path_re = re.compile(r'[/\\ ]')
doctype_lookup_re = re.compile(r'''(?smx)
(<\?.*?\?>)?\s*
<!DOCTYPE\s+(
- [a-zA-Z_][a-zA-Z0-9]*\s+
+ [a-zA-Z_][a-zA-Z0-9]*
+ (?: \s+ # optional in HTML5
[a-zA-Z_][a-zA-Z0-9]*\s+
- "[^"]*")
+ "[^"]*")?
+ )
[^>]*>
''')
tag_re = re.compile(r'<(.+?)(\s.*?)?>.*?</.+?>(?uism)')
+xml_decl_re = re.compile(r'\s*<\?xml[^>]*\?>', re.I)
class ClassNotFound(ValueError):
- """
- If one of the get_*_by_* functions didn't find a matching class.
- """
+ """Raised if one of the lookup functions didn't find a matching class."""
class OptionError(Exception):
@@ -58,10 +52,10 @@
return string
elif isinstance(string, int):
return bool(string)
- elif not isinstance(string, str):
+ elif not isinstance(string, string_types):
raise OptionError('Invalid type %r for option %s; use '
'1/0, yes/no, true/false, on/off' % (
- string, optname))
+ string, optname))
elif string.lower() in ('1', 'yes', 'true', 'on'):
return True
elif string.lower() in ('0', 'no', 'false', 'off'):
@@ -69,7 +63,7 @@
else:
raise OptionError('Invalid value %r for option %s; use '
'1/0, yes/no, true/false, on/off' % (
- string, optname))
+ string, optname))
def get_int_opt(options, optname, default=None):
@@ -79,23 +73,23 @@
except TypeError:
raise OptionError('Invalid type %r for option %s; you '
'must give an integer value' % (
- string, optname))
+ string, optname))
except ValueError:
raise OptionError('Invalid value %r for option %s; you '
'must give an integer value' % (
- string, optname))
+ string, optname))
def get_list_opt(options, optname, default=None):
val = options.get(optname, default)
- if isinstance(val, str):
+ if isinstance(val, string_types):
return val.split()
elif isinstance(val, (list, tuple)):
return list(val)
else:
raise OptionError('Invalid type %r for option %s; you '
'must give a list value' % (
- val, optname))
+ val, optname))
def docstring_headline(obj):
@@ -111,10 +105,7 @@
def make_analysator(f):
- """
- Return a static text analysation function that
- returns float values.
- """
+ """Return a static text analyser function that returns float values."""
def text_analyse(text):
try:
rv = f(text)
@@ -131,8 +122,7 @@
def shebang_matches(text, regex):
- """
- Check if the given regular expression matches the last part of the
+ """Check if the given regular expression matches the last part of the
shebang if one exists.
>>> from pygments.util import shebang_matches
@@ -177,8 +167,8 @@
def doctype_matches(text, regex):
- """
- Check if the doctype matches a regular expression (if present).
+ """Check if the doctype matches a regular expression (if present).
+
Note that this method only checks the first part of a DOCTYPE.
eg: 'html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"'
"""
@@ -186,21 +176,21 @@
if m is None:
return False
doctype = m.group(2)
- return re.compile(regex).match(doctype.strip()) is not None
+ return re.compile(regex, re.I).match(doctype.strip()) is not None
def html_doctype_matches(text):
- """
- Check if the file looks like it has a html doctype.
- """
- return doctype_matches(text, r'html\s+PUBLIC\s+"-//W3C//DTD X?HTML.*')
+ """Check if the file looks like it has a html doctype."""
+ return doctype_matches(text, r'html')
_looks_like_xml_cache = {}
+
+
def looks_like_xml(text):
- """
- Check if a doctype exists or if we have some tags.
- """
+ """Check if a doctype exists or if we have some tags."""
+ if xml_decl_re.match(text):
+ return True
key = hash(text)
try:
return _looks_like_xml_cache[key]
@@ -212,15 +202,20 @@
_looks_like_xml_cache[key] = rv
return rv
+
# Python narrow build compatibility
def _surrogatepair(c):
+ # Given a unicode character code
+ # with length greater than 16 bits,
+ # return the two 16 bit surrogate pair.
+ # From example D28 of:
+ # http://www.unicode.org/book/ch03.pdf
return (0xd7c0 + (c >> 10), (0xdc00 + (c & 0x3ff)))
+
def unirange(a, b):
- """
- Returns a regular expression string to match the given non-BMP range.
- """
+ """Returns a regular expression string to match the given non-BMP range."""
if b < a:
raise ValueError("Bad character range")
if a < 0x10000 or b < 0x10000:
@@ -228,7 +223,7 @@
if sys.maxunicode > 0xffff:
# wide build
- return '[%s-%s]' % (chr(a), chr(b))
+ return u'[%s-%s]' % (unichr(a), unichr(b))
else:
# narrow build stores surrogates, and the 're' module handles them
# (incorrectly) as characters. Since there is still ordering among
@@ -242,42 +237,151 @@
ah, al = _surrogatepair(a)
bh, bl = _surrogatepair(b)
if ah == bh:
- return '(?:%s[%s-%s])' % (chr(ah), chr(al), chr(bl))
+ return u'(?:%s[%s-%s])' % (unichr(ah), unichr(al), unichr(bl))
else:
buf = []
- buf.append('%s[%s-%s]' %
- (chr(ah), chr(al),
- ah == bh and chr(bl) or chr(0xdfff)))
+ buf.append(u'%s[%s-%s]' %
+ (unichr(ah), unichr(al),
+ ah == bh and unichr(bl) or unichr(0xdfff)))
if ah - bh > 1:
- buf.append('[%s-%s][%s-%s]' %
- chr(ah+1), chr(bh-1), chr(0xdc00), chr(0xdfff))
+ buf.append(u'[%s-%s][%s-%s]' %
+ unichr(ah+1), unichr(bh-1), unichr(0xdc00), unichr(0xdfff))
if ah != bh:
- buf.append('%s[%s-%s]' %
- (chr(bh), chr(0xdc00), chr(bl)))
+ buf.append(u'%s[%s-%s]' %
+ (unichr(bh), unichr(0xdc00), unichr(bl)))
+
+ return u'(?:' + u'|'.join(buf) + u')'
+
+
+def format_lines(var_name, seq, raw=False, indent_level=0):
+ """Formats a sequence of strings for output."""
+ lines = []
+ base_indent = ' ' * indent_level * 4
+ inner_indent = ' ' * (indent_level + 1) * 4
+ lines.append(base_indent + var_name + ' = (')
+ if raw:
+ # These should be preformatted reprs of, say, tuples.
+ for i in seq:
+ lines.append(inner_indent + i + ',')
+ else:
+ for i in seq:
+ # Force use of single quotes
+ r = repr(i + '"')
+ lines.append(inner_indent + r[:-2] + r[-1] + ',')
+ lines.append(base_indent + ')')
+ return '\n'.join(lines)
+
+
+def duplicates_removed(it, already_seen=()):
+ """
+ Returns a list with duplicates removed from the iterable `it`.
+
+ Order is preserved.
+ """
+ lst = []
+ seen = set()
+ for i in it:
+ if i in seen or i in already_seen:
+ continue
+ lst.append(i)
+ seen.add(i)
+ return lst
+
+
+class Future(object):
+ """Generic class to defer some work.
- return '(?:' + '|'.join(buf) + ')'
+ Handled specially in RegexLexerMeta, to support regex string construction at
+ first use.
+ """
+ def get(self):
+ raise NotImplementedError
+
+
+def guess_decode(text):
+ """Decode *text* with guessed encoding.
+
+ First try UTF-8; this should fail for non-UTF-8 encodings.
+ Then try the preferred locale encoding.
+ Fall back to latin-1, which always works.
+ """
+ try:
+ text = text.decode('utf-8')
+ return text, 'utf-8'
+ except UnicodeDecodeError:
+ try:
+ import locale
+ prefencoding = locale.getpreferredencoding()
+ text = text.decode()
+ return text, prefencoding
+ except (UnicodeDecodeError, LookupError):
+ text = text.decode('latin1')
+ return text, 'latin1'
+
+
+def guess_decode_from_terminal(text, term):
+ """Decode *text* coming from terminal *term*.
+
+ First try the terminal encoding, if given.
+ Then try UTF-8. Then try the preferred locale encoding.
+ Fall back to latin-1, which always works.
+ """
+ if getattr(term, 'encoding', None):
+ try:
+ text = text.decode(term.encoding)
+ except UnicodeDecodeError:
+ pass
+ else:
+ return text, term.encoding
+ return guess_decode(text)
+
+
+def terminal_encoding(term):
+ """Return our best guess of encoding for the given *term*."""
+ if getattr(term, 'encoding', None):
+ return term.encoding
+ import locale
+ return locale.getpreferredencoding()
+
# Python 2/3 compatibility
-if sys.version_info < (3,0):
- b = bytes = str
+if sys.version_info < (3, 0):
+ unichr = unichr
+ xrange = xrange
+ string_types = (str, unicode)
+ text_type = unicode
u_prefix = 'u'
- import StringIO, cStringIO
+ iteritems = dict.iteritems
+ itervalues = dict.itervalues
+ import StringIO
+ import cStringIO
+ # unfortunately, io.StringIO in Python 2 doesn't accept str at all
+ StringIO = StringIO.StringIO
BytesIO = cStringIO.StringIO
- StringIO = StringIO.StringIO
- uni_open = codecs.open
else:
- import builtins
- bytes = builtins.bytes
+ unichr = chr
+ xrange = range
+ string_types = (str,)
+ text_type = str
u_prefix = ''
- def b(s):
- if isinstance(s, str):
- return bytes(list(map(ord, s)))
- elif isinstance(s, bytes):
- return s
- else:
- raise TypeError("Invalid argument %r for b()" % (s,))
- import io
- BytesIO = io.BytesIO
- StringIO = io.StringIO
- uni_open = builtins.open
+ iteritems = dict.items
+ itervalues = dict.values
+ from io import StringIO, BytesIO, TextIOWrapper
+
+ class UnclosingTextIOWrapper(TextIOWrapper):
+ # Don't close underlying buffer on destruction.
+ def close(self):
+ pass
+
+
+def add_metaclass(metaclass):
+ """Class decorator for creating a class with a metaclass."""
+ def wrapper(cls):
+ orig_vars = cls.__dict__.copy()
+ orig_vars.pop('__dict__', None)
+ orig_vars.pop('__weakref__', None)
+ for slots_var in orig_vars.get('__slots__', ()):
+ orig_vars.pop(slots_var)
+ return metaclass(cls.__name__, cls.__bases__, orig_vars)
+ return wrapper
