eric: comparison eric7/DebugClients/Python/coverage/phystokens.py

-:d728227e8ebb
+:0802ae193343
 # Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0
 # For details: https://github.com/nedbat/coveragepy/blob/master/NOTICE.txt
 """Better tokenizing for coverage.py."""
-import codecs
+import ast
 import keyword
 import re
-import sys
 import token
 import tokenize
 from coverage import env
-from coverage.backward import iternext, unicode_class
 from coverage.misc import contract
 def phys_tokens(toks):
 """Return all physical tokens, even line continuations.
 last_ttext = ttext
 yield ttype, ttext, (slineno, scol), (elineno, ecol), ltext
 last_lineno = elineno
+class MatchCaseFinder(ast.NodeVisitor):
+"""Helper for finding match/case lines."""
+def __init__(self, source):
+# This will be the set of line numbers that start match or case statements.
+self.match_case_lines = set()
+self.visit(ast.parse(source))
+def visit_Match(self, node):
+"""Invoked by ast.NodeVisitor.visit"""
+self.match_case_lines.add(node.lineno)
+for case in node.cases:
+self.match_case_lines.add(case.pattern.lineno)
+self.generic_visit(node)
 @contract(source='unicode')
 def source_token_lines(source):
 """Generate a series of lines, one for each line in `source`.
 Each line is a list of pairs, each pair is a token::
 col = 0
 source = source.expandtabs(8).replace('\r\n', '\n')
 tokgen = generate_tokens(source)
-for ttype, ttext, (_, scol), (_, ecol), _ in phys_tokens(tokgen):
+if env.PYBEHAVIOR.soft_keywords:
+match_case_lines = MatchCaseFinder(source).match_case_lines
+for ttype, ttext, (sline, scol), (_, ecol), _ in phys_tokens(tokgen):
 mark_start = True
 for part in re.split('(\n)', ttext):
 if part == '\n':
 yield line
 line = []
 mark_end = False
 elif ttype in ws_tokens:
 mark_end = False
 else:
 if mark_start and scol > col:
-line.append(("ws", u" " * (scol - col)))
+line.append(("ws", " " * (scol - col)))
 mark_start = False
 tok_class = tokenize.tok_name.get(ttype, 'xx').lower()[:3]
-if ttype == token.NAME and keyword.iskeyword(ttext):
+if ttype == token.NAME:
-tok_class = "key"
+if keyword.iskeyword(ttext):
+# Hard keywords are always keywords.
+tok_class = "key"
+elif env.PYBEHAVIOR.soft_keywords and keyword.issoftkeyword(ttext):
+# Soft keywords appear at the start of the line, on lines that start
+# match or case statements.
+if len(line) == 0:
+is_start_of_line = True
+elif (len(line) == 1) and line[0][0] == "ws":
+is_start_of_line = True
+else:
+is_start_of_line = False
+if is_start_of_line and sline in match_case_lines:
+tok_class = "key"
 line.append((tok_class, part))
 mark_end = True
 scol = 0
 if mark_end:
 col = ecol
 if line:
 yield line
-class CachedTokenizer(object):
+class CachedTokenizer:
 """A one-element cache around tokenize.generate_tokens.
 When reporting, coverage.py tokenizes files twice, once to find the
 structure of the file, and once to syntax-color it.  Tokenizing is
 expensive, and easily cached.
 @contract(text='unicode')
 def generate_tokens(self, text):
 """A stand-in for `tokenize.generate_tokens`."""
 if text != self.last_text:
 self.last_text = text
-readline = iternext(text.splitlines(True))
+readline = iter(text.splitlines(True)).__next__
 self.last_tokens = list(tokenize.generate_tokens(readline))
 return self.last_tokens
 # Create our generate_tokens cache as a callable replacement function.
 generate_tokens = CachedTokenizer().generate_tokens
 COOKIE_RE = re.compile(r"^[ \t]*#.*coding[:=][ \t]*([-\w.]+)", flags=re.MULTILINE)
 @contract(source='bytes')
-def _source_encoding_py2(source):
+def source_encoding(source):
 """Determine the encoding for `source`, according to PEP 263.
-`source` is a byte string, the text of the program.
+`source` is a byte string: the text of the program.
 Returns a string, the name of the encoding.
 """
-assert isinstance(source, bytes)
+readline = iter(source.splitlines(True)).__next__
-# Do this so the detect_encode code we copied will work.
-readline = iternext(source.splitlines(True))
-# This is mostly code adapted from Py3.2's tokenize module.
-def _get_normal_name(orig_enc):
-"""Imitates get_normal_name in tokenizer.c."""
-# Only care about the first 12 characters.
-enc = orig_enc[:12].lower().replace("_", "-")
-if re.match(r"^utf-8($|-)", enc):
-return "utf-8"
-if re.match(r"^(latin-1|iso-8859-1|iso-latin-1)($|-)", enc):
-return "iso-8859-1"
-return orig_enc
-# From detect_encode():
-# It detects the encoding from the presence of a UTF-8 BOM or an encoding
-# cookie as specified in PEP-0263.  If both a BOM and a cookie are present,
-# but disagree, a SyntaxError will be raised.  If the encoding cookie is an
-# invalid charset, raise a SyntaxError.  Note that if a UTF-8 BOM is found,
-# 'utf-8-sig' is returned.
-# If no encoding is specified, then the default will be returned.
-default = 'ascii'
-bom_found = False
-encoding = None
-def read_or_stop():
-"""Get the next source line, or ''."""
-try:
-return readline()
-except StopIteration:
-return ''
-def find_cookie(line):
-"""Find an encoding cookie in `line`."""
-try:
-line_string = line.decode('ascii')
-except UnicodeDecodeError:
-return None
-matches = COOKIE_RE.findall(line_string)
-if not matches:
-return None
-encoding = _get_normal_name(matches[0])
-try:
-codec = codecs.lookup(encoding)
-except LookupError:
-# This behavior mimics the Python interpreter
-raise SyntaxError("unknown encoding: " + encoding)
-if bom_found:
-# codecs in 2.3 were raw tuples of functions, assume the best.
-codec_name = getattr(codec, 'name', encoding)
-if codec_name != 'utf-8':
-# This behavior mimics the Python interpreter
-raise SyntaxError('encoding problem: utf-8')
-encoding += '-sig'
-return encoding
-first = read_or_stop()
-if first.startswith(codecs.BOM_UTF8):
-bom_found = True
-first = first[3:]
-default = 'utf-8-sig'
-if not first:
-return default
-encoding = find_cookie(first)
-if encoding:
-return encoding
-second = read_or_stop()
-if not second:
-return default
-encoding = find_cookie(second)
-if encoding:
-return encoding
-return default
-@contract(source='bytes')
-def _source_encoding_py3(source):
-"""Determine the encoding for `source`, according to PEP 263.
-`source` is a byte string: the text of the program.
-Returns a string, the name of the encoding.
-"""
-readline = iternext(source.splitlines(True))
 return tokenize.detect_encoding(readline)[0]
-if env.PY3:
-source_encoding = _source_encoding_py3
-else:
-source_encoding = _source_encoding_py2
 @contract(source='unicode')
 def compile_unicode(source, filename, mode):
 """Just like the `compile` builtin, but works on any Unicode string.
 Python 2's compile() builtin has a stupid restriction: if the source string
 is Unicode, then it may not have a encoding declaration in it.  Why not?
-Who knows!  It also decodes to utf8, and then tries to interpret those utf8
+Who knows!  It also decodes to utf-8, and then tries to interpret those
-bytes according to the encoding declaration.  Why? Who knows!
+utf-8 bytes according to the encoding declaration.  Why? Who knows!
 This function neuters the coding declaration, and compiles it.
 """
 source = neuter_encoding_declaration(source)
-if env.PY2 and isinstance(filename, unicode_class):
-filename = filename.encode(sys.getfilesystemencoding(), "replace")
 code = compile(source, filename, mode)
 return code
 @contract(source='unicode', returns='unicode')

Mercurial Repositories > eric / file comparison

comparison: eric7/DebugClients/Python/coverage/phystokens.py

eric7/DebugClients/Python/coverage/phystokens.py