eric: comparison ThirdParty/Pygments/pygments/cmdline.py

-:8bc578136279
+:4f20dba37ab6
 pygments.cmdline
 ~~~~~~~~~~~~~~~~
 Command line interface.
-:copyright: Copyright 2006-2013 by the Pygments team, see AUTHORS.
+:copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS.
 :license: BSD, see LICENSE for details.
 """
-from __future__ import unicode_literals
+from __future__ import print_function
 import sys
 import getopt
 from textwrap import dedent
 from pygments import __version__, highlight
-from pygments.util import ClassNotFound, OptionError, docstring_headline
+from pygments.util import ClassNotFound, OptionError, docstring_headline, \
-from pygments.lexers import get_all_lexers, get_lexer_by_name, get_lexer_for_filename, \
+guess_decode, guess_decode_from_terminal, terminal_encoding
-find_lexer_class, guess_lexer, TextLexer
+from pygments.lexers import get_all_lexers, get_lexer_by_name, guess_lexer, \
+get_lexer_for_filename, find_lexer_class, TextLexer
+from pygments.formatters.latex import LatexEmbeddedLexer, LatexFormatter
 from pygments.formatters import get_all_formatters, get_formatter_by_name, \
 get_formatter_for_filename, find_formatter_class, \
 TerminalFormatter  # pylint:disable-msg=E0611
 from pygments.filters import get_all_filters, find_filter_class
 from pygments.styles import get_all_styles, get_style_by_name
 USAGE = """\
 Usage: %s [-l <lexer> | -g] [-F <filter>[:<options>]] [-f <formatter>]
-[-O <options>] [-P <option=value>] [-o <outfile>] [<infile>]
+[-O <options>] [-P <option=value>] [-s] [-o <outfile>] [<infile>]
 %s -S <style> -f <formatter> [-a <arg>] [-O <options>] [-P <option=value>]
 %s -L [<which> ...]
 %s -N <filename>
 %s -H <type> <name>
 Highlight the input file and write the result to <outfile>.
 If no input file is given, use stdin, if -o is not given, use stdout.
+If -s is passed, lexing will be done in "streaming" mode, reading and
+highlighting one line at a time.  This will only work properly with
+lexers that have no constructs spanning multiple lines!
 <lexer> is a lexer name (query all lexer names with -L). If -l is not
 given, the lexer is guessed from the extension of the input file name
 (this obviously doesn't work if the input is stdin).  If -g is passed,
 attempt to guess the lexer from the file contents, or pass through as
 plain text if this fails (this can work for stdin).
 the given filename. It does not take input or highlight anything.
 If no specific lexer can be determined "text" is returned.
 The -H option prints detailed help for the object <name> of type <type>,
 where <type> is one of "lexer", "formatter" or "filter".
+The -s option processes lines one at a time until EOF, rather than
+waiting to process the entire file.  This only works for stdin, and
+is intended for streaming input such as you get from 'tail -f'.
+Example usage: "tail -f sql.log | pygmentize -s -l sql"
 The -h option prints this help.
 The -V option prints the package version.
 """
 continue
 o_args = o_str.split(',')
 for o_arg in o_args:
 o_arg = o_arg.strip()
 try:
-o_key, o_val = o_arg.split('=')
+o_key, o_val = o_arg.split('=', 1)
 o_key = o_key.strip()
 o_val = o_val.strip()
 except ValueError:
 opts[o_arg] = True
 else:
 cls = get_style_by_name(name)
 print("* " + name + ':')
 print("    %s" % docstring_headline(cls))
-def main(args=sys.argv):
+def main_inner(popts, args, usage):
-"""
-Main command line entry point.
-"""
-# pylint: disable-msg=R0911,R0912,R0915
-usage = USAGE % ((args[0],) * 6)
-if sys.platform in ['win32', 'cygwin']:
-try:
-# Provide coloring under Windows, if possible
-import colorama
-colorama.init()
-except ImportError:
-pass
-try:
-popts, args = getopt.getopt(args[1:], "l:f:F:o:O:P:LS:a:N:hVHg")
-except getopt.GetoptError as err:
-print(usage, file=sys.stderr)
-return 2
 opts = {}
 O_opts = []
 P_opts = []
 F_opts = []
 for opt, arg in popts:
 P_opts.append(arg)
 elif opt == '-F':
 F_opts.append(arg)
 opts[opt] = arg
-if not opts and not args:
-print(usage)
-return 0
 if opts.pop('-h', None) is not None:
 print(usage)
 return 0
 if opts.pop('-V', None) is not None:
-print('Pygments version %s, (c) 2006-2013 by Georg Brandl.' % __version__)
+print('Pygments version %s, (c) 2006-2014 by Georg Brandl.' % __version__)
 return 0
 # handle ``pygmentize -L``
 L_opt = opts.pop('-L', None)
 if L_opt is not None:
 except ValueError:
 parsed_opts[p_opt] = True
 else:
 parsed_opts[name] = value
 opts.pop('-P', None)
+# encodings
+inencoding  = parsed_opts.get('inencoding', parsed_opts.get('encoding'))
+outencoding = parsed_opts.get('outencoding', parsed_opts.get('encoding'))
 # handle ``pygmentize -N``
 infn = opts.pop('-N', None)
 if infn is not None:
 try:
 # parse -F options
 F_opts = _parse_filters(F_opts)
 opts.pop('-F', None)
-# select formatter
+# select lexer
-outfn = opts.pop('-o', None)
+lexer = None
-fmter = opts.pop('-f', None)
-if fmter:
+# given by name?
-try:
+lexername = opts.pop('-l', None)
-fmter = get_formatter_by_name(fmter, **parsed_opts)
+if lexername:
+try:
+lexer = get_lexer_by_name(lexername, **parsed_opts)
 except (OptionError, ClassNotFound) as err:
 print('Error:', err, file=sys.stderr)
 return 1
-if outfn:
+# read input code
-if not fmter:
+code = None
-try:
-fmter = get_formatter_for_filename(outfn, **parsed_opts)
-except (OptionError, ClassNotFound) as err:
-print('Error:', err, file=sys.stderr)
-return 1
-try:
-outfile = open(outfn, 'wb')
-except Exception as err:
-print('Error: cannot open outfile:', err, file=sys.stderr)
-return 1
-else:
-if not fmter:
-fmter = TerminalFormatter(**parsed_opts)
-outfile = sys.stdout
-# select lexer
-lexer = opts.pop('-l', None)
-if lexer:
-try:
-lexer = get_lexer_by_name(lexer, **parsed_opts)
-except (OptionError, ClassNotFound) as err:
-print('Error:', err, file=sys.stderr)
-return 1
 if args:
 if len(args) > 1:
 print(usage, file=sys.stderr)
 return 2
+if '-s' in opts:
+print('Error: -s option not usable when input file specified',
+file=sys.stderr)
+return 1
 infn = args[0]
 try:
-code = open(infn, 'rb').read()
+with open(infn, 'rb') as infp:
+code = infp.read()
 except Exception as err:
 print('Error: cannot read infile:', err, file=sys.stderr)
 return 1
+if not inencoding:
+code, inencoding = guess_decode(code)
+# do we have to guess the lexer?
 if not lexer:
 try:
 lexer = get_lexer_for_filename(infn, code, **parsed_opts)
 except ClassNotFound as err:
 if '-g' in opts:
 return 1
 except OptionError as err:
 print('Error:', err, file=sys.stderr)
 return 1
-else:
+elif '-s' not in opts:  # treat stdin as full file (-s support is later)
-if '-g' in opts:
+# read code from terminal, always in binary mode since we want to
+# decode ourselves and be tolerant with it
+if sys.version_info > (3,):
+# Python 3: we have to use .buffer to get a binary stream
+code = sys.stdin.buffer.read()
+else:
 code = sys.stdin.read()
+if not inencoding:
+code, inencoding = guess_decode_from_terminal(code, sys.stdin)
+# else the lexer will do the decoding
+if not lexer:
 try:
 lexer = guess_lexer(code, **parsed_opts)
 except ClassNotFound:
 lexer = TextLexer(**parsed_opts)
-elif not lexer:
-print('Error: no lexer name given and reading ' + \
+# select formatter
-'from stdin (try using -g or -l <lexer>)', file=sys.stderr)
+outfn = opts.pop('-o', None)
-return 2
+fmter = opts.pop('-f', None)
+if fmter:
+try:
+fmter = get_formatter_by_name(fmter, **parsed_opts)
+except (OptionError, ClassNotFound) as err:
+print('Error:', err, file=sys.stderr)
+return 1
+if outfn:
+if not fmter:
+try:
+fmter = get_formatter_for_filename(outfn, **parsed_opts)
+except (OptionError, ClassNotFound) as err:
+print('Error:', err, file=sys.stderr)
+return 1
+try:
+outfile = open(outfn, 'wb')
+except Exception as err:
+print('Error: cannot open outfile:', err, file=sys.stderr)
+return 1
+else:
+if not fmter:
+fmter = TerminalFormatter(**parsed_opts)
+if sys.version_info > (3,):
+# Python 3: we have to use .buffer to get a binary stream
+outfile = sys.stdout.buffer
 else:
-code = sys.stdin.read()
+outfile = sys.stdout
-# No encoding given? Use latin1 if output file given,
+# determine output encoding if not explicitly selected
-# stdin/stdout encoding otherwise.
+if not outencoding:
-# (This is a compromise, I'm not too happy with it...)
-if 'encoding' not in parsed_opts and 'outencoding' not in parsed_opts:
 if outfn:
-# encoding pass-through
+# output file? use lexer encoding for now (can still be None)
-fmter.encoding = 'latin1'
+fmter.encoding = inencoding
 else:
-if sys.version_info < (3,):
+# else use terminal encoding
-# use terminal encoding; Python 3's terminals already do that
+fmter.encoding = terminal_encoding(sys.stdout)
-lexer.encoding = getattr(sys.stdin, 'encoding',
-None) or 'ascii'
+# provide coloring under Windows, if possible
-fmter.encoding = getattr(sys.stdout, 'encoding',
+if not outfn and sys.platform in ('win32', 'cygwin') and \
-None) or 'ascii'
+fmter.name in ('Terminal', 'Terminal256'):
-elif not outfn and sys.version_info > (3,):
+# unfortunately colorama doesn't support binary streams on Py3
-# output to terminal with encoding -> use .buffer
+if sys.version_info > (3,):
-outfile = sys.stdout.buffer
+from pygments.util import UnclosingTextIOWrapper
+outfile = UnclosingTextIOWrapper(outfile, encoding=fmter.encoding)
+fmter.encoding = None
+try:
+import colorama.initialise
+except ImportError:
+pass
+else:
+outfile = colorama.initialise.wrap_stream(
+outfile, convert=None, strip=None, autoreset=False, wrap=True)
+# When using the LaTeX formatter and the option `escapeinside` is
+# specified, we need a special lexer which collects escaped text
+# before running the chosen language lexer.
+escapeinside = parsed_opts.get('escapeinside', '')
+if len(escapeinside) == 2 and isinstance(fmter, LatexFormatter):
+left = escapeinside[0]
+right = escapeinside[1]
+lexer = LatexEmbeddedLexer(left, right, lexer)
+# process filters
+for fname, fopts in F_opts:
+try:
+lexer.add_filter(fname, **fopts)
+except ClassNotFound as err:
+print('Error:', err, file=sys.stderr)
+return 1
 # ... and do it!
+if '-s' not in opts:
+# process whole input as per normal...
+highlight(code, lexer, fmter, outfile)
+return 0
+else:
+if not lexer:
+print('Error: when using -s a lexer has to be selected with -l',
+file=sys.stderr)
+return 1
+# line by line processing of stdin (eg: for 'tail -f')...
+try:
+while 1:
+if sys.version_info > (3,):
+# Python 3: we have to use .buffer to get a binary stream
+line = sys.stdin.buffer.readline()
+else:
+line = sys.stdin.readline()
+if not line:
+break
+if not inencoding:
+line = guess_decode_from_terminal(line, sys.stdin)[0]
+highlight(line, lexer, fmter, outfile)
+if hasattr(outfile, 'flush'):
+outfile.flush()
+except KeyboardInterrupt:
+return 0
+def main(args=sys.argv):
+"""
+Main command line entry point.
+"""
+usage = USAGE % ((args[0],) * 6)
 try:
-# process filters
+popts, args = getopt.getopt(args[1:], "l:f:F:o:O:P:LS:a:N:hVHgs")
-for fname, fopts in F_opts:
+except getopt.GetoptError:
-lexer.add_filter(fname, **fopts)
+print(usage, file=sys.stderr)
-highlight(code, lexer, fmter, outfile)
+return 2
-except Exception as err:
+try:
+return main_inner(popts, args, usage)
+except Exception:
 import traceback
 info = traceback.format_exception(*sys.exc_info())
 msg = info[-1].strip()
 if len(info) >= 3:
 # extract relevant file and position info
 msg += '\n   (f%s)' % info[-2].split('\n')[0].strip()[1:]
 print(file=sys.stderr)
 print('*** Error while highlighting:', file=sys.stderr)
 print(msg, file=sys.stderr)
 return 1
-return 0

Mercurial Repositories > eric / file comparison

comparison: ThirdParty/Pygments/pygments/cmdline.py

ThirdParty/Pygments/pygments/cmdline.py