eric: comparison ThirdParty/Pygments/pygments/util.py

-:8bc578136279
+:4f20dba37ab6
 pygments.util
 ~~~~~~~~~~~~~
 Utility functions.
-:copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+:copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
 :license: BSD, see LICENSE for details.
 """
-from __future__ import unicode_literals
-try:
-chr = unichr
-except NameError:
-pass
 import re
 import sys
-import codecs
 split_path_re = re.compile(r'[/\\ ]')
 doctype_lookup_re = re.compile(r'''(?smx)
 (<\?.*?\?>)?\s*
 <!DOCTYPE\s+(
+[a-zA-Z_][a-zA-Z0-9]*
+(?: \s+      # optional in HTML5
 [a-zA-Z_][a-zA-Z0-9]*\s+
-[a-zA-Z_][a-zA-Z0-9]*\s+
+"[^"]*")?
-"[^"]*")
+)
 [^>]*>
 ''')
 tag_re = re.compile(r'<(.+?)(\s.*?)?>.*?</.+?>(?uism)')
+xml_decl_re = re.compile(r'\s*<\?xml[^>]*\?>', re.I)
 class ClassNotFound(ValueError):
-"""
+"""Raised if one of the lookup functions didn't find a matching class."""
-If one of the get_*_by_* functions didn't find a matching class.
-"""
 class OptionError(Exception):
 pass
 string = options.get(optname, default)
 if isinstance(string, bool):
 return string
 elif isinstance(string, int):
 return bool(string)
-elif not isinstance(string, str):
+elif not isinstance(string, string_types):
 raise OptionError('Invalid type %r for option %s; use '
 '1/0, yes/no, true/false, on/off' % (
 string, optname))
 elif string.lower() in ('1', 'yes', 'true', 'on'):
 return True
 elif string.lower() in ('0', 'no', 'false', 'off'):
 return False
 else:
 raise OptionError('Invalid value %r for option %s; use '
 '1/0, yes/no, true/false, on/off' % (
 string, optname))
 def get_int_opt(options, optname, default=None):
 string = options.get(optname, default)
 try:
 return int(string)
 except TypeError:
 raise OptionError('Invalid type %r for option %s; you '
 'must give an integer value' % (
 string, optname))
 except ValueError:
 raise OptionError('Invalid value %r for option %s; you '
 'must give an integer value' % (
 string, optname))
 def get_list_opt(options, optname, default=None):
 val = options.get(optname, default)
-if isinstance(val, str):
+if isinstance(val, string_types):
 return val.split()
 elif isinstance(val, (list, tuple)):
 return list(val)
 else:
 raise OptionError('Invalid type %r for option %s; you '
 'must give a list value' % (
 val, optname))
 def docstring_headline(obj):
 if not obj.__doc__:
 return ''
 break
 return ''.join(res).lstrip()
 def make_analysator(f):
-"""
+"""Return a static text analyser function that returns float values."""
-Return a static text analysation function that
-returns float values.
-"""
 def text_analyse(text):
 try:
 rv = f(text)
 except Exception:
 return 0.0
 text_analyse.__doc__ = f.__doc__
 return staticmethod(text_analyse)
 def shebang_matches(text, regex):
-"""
+"""Check if the given regular expression matches the last part of the
-Check if the given regular expression matches the last part of the
 shebang if one exists.
 >>> from pygments.util import shebang_matches
 >>> shebang_matches('#!/usr/bin/env python', r'python(2\.\d)?')
 True
 return True
 return False
 def doctype_matches(text, regex):
-"""
+"""Check if the doctype matches a regular expression (if present).
-Check if the doctype matches a regular expression (if present).
 Note that this method only checks the first part of a DOCTYPE.
 eg: 'html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"'
 """
 m = doctype_lookup_re.match(text)
 if m is None:
 return False
 doctype = m.group(2)
-return re.compile(regex).match(doctype.strip()) is not None
+return re.compile(regex, re.I).match(doctype.strip()) is not None
 def html_doctype_matches(text):
-"""
+"""Check if the file looks like it has a html doctype."""
-Check if the file looks like it has a html doctype.
+return doctype_matches(text, r'html')
-"""
-return doctype_matches(text, r'html\s+PUBLIC\s+"-//W3C//DTD X?HTML.*')
 _looks_like_xml_cache = {}
 def looks_like_xml(text):
-"""
+"""Check if a doctype exists or if we have some tags."""
-Check if a doctype exists or if we have some tags.
+if xml_decl_re.match(text):
-"""
+return True
 key = hash(text)
 try:
 return _looks_like_xml_cache[key]
 except KeyError:
 m = doctype_lookup_re.match(text)
 return True
 rv = tag_re.search(text[:1000]) is not None
 _looks_like_xml_cache[key] = rv
 return rv
 # Python narrow build compatibility
 def _surrogatepair(c):
+# Given a unicode character code
+# with length greater than 16 bits,
+# return the two 16 bit surrogate pair.
+# From example D28 of:
+# http://www.unicode.org/book/ch03.pdf
 return (0xd7c0 + (c >> 10), (0xdc00 + (c & 0x3ff)))
 def unirange(a, b):
-"""
+"""Returns a regular expression string to match the given non-BMP range."""
-Returns a regular expression string to match the given non-BMP range.
-"""
 if b < a:
 raise ValueError("Bad character range")
 if a < 0x10000 or b < 0x10000:
 raise ValueError("unirange is only defined for non-BMP ranges")
 if sys.maxunicode > 0xffff:
 # wide build
-return '[%s-%s]' % (chr(a), chr(b))
+return u'[%s-%s]' % (unichr(a), unichr(b))
 else:
 # narrow build stores surrogates, and the 're' module handles them
 # (incorrectly) as characters.  Since there is still ordering among
 # these characters, expand the range to one that it understands.  Some
 # background in http://bugs.python.org/issue3665 and
 # literals because jython [which uses the wide path] can't load this
 # file if they are literals.
 ah, al = _surrogatepair(a)
 bh, bl = _surrogatepair(b)
 if ah == bh:
-return '(?:%s[%s-%s])' % (chr(ah), chr(al), chr(bl))
+return u'(?:%s[%s-%s])' % (unichr(ah), unichr(al), unichr(bl))
 else:
 buf = []
-buf.append('%s[%s-%s]' %
+buf.append(u'%s[%s-%s]' %
-(chr(ah), chr(al),
+(unichr(ah), unichr(al),
-ah == bh and chr(bl) or chr(0xdfff)))
+ah == bh and unichr(bl) or unichr(0xdfff)))
 if ah - bh > 1:
-buf.append('[%s-%s][%s-%s]' %
+buf.append(u'[%s-%s][%s-%s]' %
-chr(ah+1), chr(bh-1), chr(0xdc00), chr(0xdfff))
+unichr(ah+1), unichr(bh-1), unichr(0xdc00), unichr(0xdfff))
 if ah != bh:
-buf.append('%s[%s-%s]' %
+buf.append(u'%s[%s-%s]' %
-(chr(bh), chr(0xdc00), chr(bl)))
+(unichr(bh), unichr(0xdc00), unichr(bl)))
-return '(?:' + '|'.join(buf) + ')'
+return u'(?:' + u'|'.join(buf) + u')'
+def format_lines(var_name, seq, raw=False, indent_level=0):
+"""Formats a sequence of strings for output."""
+lines = []
+base_indent = ' ' * indent_level * 4
+inner_indent = ' ' * (indent_level + 1) * 4
+lines.append(base_indent + var_name + ' = (')
+if raw:
+# These should be preformatted reprs of, say, tuples.
+for i in seq:
+lines.append(inner_indent + i + ',')
+else:
+for i in seq:
+# Force use of single quotes
+r = repr(i + '"')
+lines.append(inner_indent + r[:-2] + r[-1] + ',')
+lines.append(base_indent + ')')
+return '\n'.join(lines)
+def duplicates_removed(it, already_seen=()):
+"""
+Returns a list with duplicates removed from the iterable `it`.
+Order is preserved.
+"""
+lst = []
+seen = set()
+for i in it:
+if i in seen or i in already_seen:
+continue
+lst.append(i)
+seen.add(i)
+return lst
+class Future(object):
+"""Generic class to defer some work.
+Handled specially in RegexLexerMeta, to support regex string construction at
+first use.
+"""
+def get(self):
+raise NotImplementedError
+def guess_decode(text):
+"""Decode *text* with guessed encoding.
+First try UTF-8; this should fail for non-UTF-8 encodings.
+Then try the preferred locale encoding.
+Fall back to latin-1, which always works.
+"""
+try:
+text = text.decode('utf-8')
+return text, 'utf-8'
+except UnicodeDecodeError:
+try:
+import locale
+prefencoding = locale.getpreferredencoding()
+text = text.decode()
+return text, prefencoding
+except (UnicodeDecodeError, LookupError):
+text = text.decode('latin1')
+return text, 'latin1'
+def guess_decode_from_terminal(text, term):
+"""Decode *text* coming from terminal *term*.
+First try the terminal encoding, if given.
+Then try UTF-8.  Then try the preferred locale encoding.
+Fall back to latin-1, which always works.
+"""
+if getattr(term, 'encoding', None):
+try:
+text = text.decode(term.encoding)
+except UnicodeDecodeError:
+pass
+else:
+return text, term.encoding
+return guess_decode(text)
+def terminal_encoding(term):
+"""Return our best guess of encoding for the given *term*."""
+if getattr(term, 'encoding', None):
+return term.encoding
+import locale
+return locale.getpreferredencoding()
 # Python 2/3 compatibility
-if sys.version_info < (3,0):
+if sys.version_info < (3, 0):
-b = bytes = str
+unichr = unichr
+xrange = xrange
+string_types = (str, unicode)
+text_type = unicode
 u_prefix = 'u'
-import StringIO, cStringIO
+iteritems = dict.iteritems
+itervalues = dict.itervalues
+import StringIO
+import cStringIO
+# unfortunately, io.StringIO in Python 2 doesn't accept str at all
+StringIO = StringIO.StringIO
 BytesIO = cStringIO.StringIO
-StringIO = StringIO.StringIO
-uni_open = codecs.open
 else:
-import builtins
+unichr = chr
-bytes = builtins.bytes
+xrange = range
+string_types = (str,)
+text_type = str
 u_prefix = ''
-def b(s):
+iteritems = dict.items
-if isinstance(s, str):
+itervalues = dict.values
-return bytes(list(map(ord, s)))
+from io import StringIO, BytesIO, TextIOWrapper
-elif isinstance(s, bytes):
-return s
+class UnclosingTextIOWrapper(TextIOWrapper):
-else:
+# Don't close underlying buffer on destruction.
-raise TypeError("Invalid argument %r for b()" % (s,))
+def close(self):
-import io
+pass
-BytesIO = io.BytesIO
-StringIO = io.StringIO
-uni_open = builtins.open
+def add_metaclass(metaclass):
+"""Class decorator for creating a class with a metaclass."""
+def wrapper(cls):
+orig_vars = cls.__dict__.copy()
+orig_vars.pop('__dict__', None)
+orig_vars.pop('__weakref__', None)
+for slots_var in orig_vars.get('__slots__', ()):
+orig_vars.pop(slots_var)
+return metaclass(cls.__name__, cls.__bases__, orig_vars)
+return wrapper

Mercurial Repositories > eric / file comparison

comparison: ThirdParty/Pygments/pygments/util.py

ThirdParty/Pygments/pygments/util.py