eric: comparison DebugClients/Python3/coverage/parser.py

-:04e5dfbd3f3d
+:3586ebd9fac8
 # Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0
 # For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt
 """Code parsing for coverage.py."""
+import ast
 import collections
-import dis
+import os
 import re
 import token
 import tokenize
+from coverage import env
 from coverage.backward import range    # pylint: disable=redefined-builtin
-from coverage.backward import bytes_to_ints
+from coverage.backward import bytes_to_ints, string_class
-from coverage.bytecode import ByteCodes, CodeObjects
+from coverage.bytecode import CodeObjects
-from coverage.misc import contract, nice_pair, expensive, join_regex
+from coverage.debug import short_stack
+from coverage.misc import contract, new_contract, nice_pair, join_regex
 from coverage.misc import CoverageException, NoSource, NotPython
-from coverage.phystokens import compile_unicode, generate_tokens
+from coverage.phystokens import compile_unicode, generate_tokens, neuter_encoding_declaration
 class PythonParser(object):
-"""Parse code to find executable lines, excluded lines, etc."""
+"""Parse code to find executable lines, excluded lines, etc.
+This information is all based on static analysis: no code execution is
+involved.
+"""
 @contract(text='unicode|None')
 def __init__(self, text=None, filename=None, exclude=None):
 """
 Source can be provided as `text`, the text itself, or `filename`, from
 which the text will be read.  Excluded lines are those that match
 "No source for code: '%s': %s" % (self.filename, err)
 )
 self.exclude = exclude
-self.show_tokens = False
 # The text lines of the parsed code.
 self.lines = self.text.split('\n')
-# The line numbers of excluded lines of code.
+# The normalized line numbers of the statements in the code. Exclusions
+# are taken into account, and statements are adjusted to their first
+# lines.
+self.statements = set()
+# The normalized line numbers of the excluded lines in the code,
+# adjusted to their first lines.
 self.excluded = set()
+# The raw_* attributes are only used in this class, and in
+# lab/parser.py to show how this class is working.
+# The line numbers that start statements, as reported by the line
+# number table in the bytecode.
+self.raw_statements = set()
+# The raw line numbers of excluded lines of code, as marked by pragmas.
+self.raw_excluded = set()
+# The line numbers of class and function definitions.
+self.raw_classdefs = set()
 # The line numbers of docstring lines.
-self.docstrings = set()
+self.raw_docstrings = set()
-# The line numbers of class definitions.
+# Internal detail, used by lab/parser.py.
-self.classdefs = set()
+self.show_tokens = False
-# A dict mapping line numbers to (lo,hi) for multi-line statements.
+# A dict mapping line numbers to lexical statement starts for
-self.multiline = {}
+# multi-line statements.
+self._multiline = {}
-# The line numbers that start statements.
-self.statement_starts = set()
+# Lazily-created ByteParser, arc data, and missing arc descriptions.
-# Lazily-created ByteParser and arc data.
 self._byte_parser = None
 self._all_arcs = None
+self._missing_arc_fragments = None
 @property
 def byte_parser(self):
 """Create a ByteParser on demand."""
 if not self._byte_parser:
 Returns a set of line numbers, the lines that contain a match for one
 of the regexes in `regexes`.  The entire line needn't match, just a
 part of it.
 """
-regex_c = re.compile(join_regex(regexes))
+combined = join_regex(regexes)
+if env.PY2:
+combined = combined.decode("utf8")
+regex_c = re.compile(combined)
 matches = set()
 for i, ltext in enumerate(self.lines, start=1):
 if regex_c.search(ltext):
 matches.add(i)
 return matches
 def _raw_parse(self):
 """Parse the source to find the interesting facts about its lines.
-A handful of member fields are updated.
+A handful of attributes are updated.
 """
 # Find lines which match an exclusion pattern.
 if self.exclude:
-self.excluded = self.lines_matching(self.exclude)
+self.raw_excluded = self.lines_matching(self.exclude)
 # Tokenize, to find excluded suites, to find docstrings, and to find
 # multi-line statements.
 indent = 0
 exclude_indent = 0
 excluding = False
+excluding_decorators = False
 prev_toktype = token.INDENT
 first_line = None
 empty = True
+first_on_line = True
 tokgen = generate_tokens(self.text)
 for toktype, ttext, (slineno, _), (elineno, _), ltext in tokgen:
 if self.show_tokens:                # pragma: not covered
 print("%10s %5s %-20r %r" % (
 ))
 if toktype == token.INDENT:
 indent += 1
 elif toktype == token.DEDENT:
 indent -= 1
-elif toktype == token.NAME and ttext == 'class':
+elif toktype == token.NAME:
-# Class definitions look like branches in the byte code, so
+if ttext == 'class':
-# we need to exclude them.  The simplest way is to note the
+# Class definitions look like branches in the bytecode, so
-# lines with the 'class' keyword.
+# we need to exclude them.  The simplest way is to note the
-self.classdefs.add(slineno)
+# lines with the 'class' keyword.
-elif toktype == token.OP and ttext == ':':
+self.raw_classdefs.add(slineno)
-if not excluding and elineno in self.excluded:
+elif toktype == token.OP:
-# Start excluding a suite.  We trigger off of the colon
+if ttext == ':':
-# token so that the #pragma comment will be recognized on
+should_exclude = (elineno in self.raw_excluded) or excluding_decorators
-# the same line as the colon.
+if not excluding and should_exclude:
-exclude_indent = indent
+# Start excluding a suite.  We trigger off of the colon
-excluding = True
+# token so that the #pragma comment will be recognized on
+# the same line as the colon.
+self.raw_excluded.add(elineno)
+exclude_indent = indent
+excluding = True
+excluding_decorators = False
+elif ttext == '@' and first_on_line:
+# A decorator.
+if elineno in self.raw_excluded:
+excluding_decorators = True
+if excluding_decorators:
+self.raw_excluded.add(elineno)
 elif toktype == token.STRING and prev_toktype == token.INDENT:
 # Strings that are first on an indented line are docstrings.
 # (a trick from trace.py in the stdlib.) This works for
 # 99.9999% of cases.  For the rest (!) see:
 # http://stackoverflow.com/questions/1769332/x/1769794#1769794
-self.docstrings.update(range(slineno, elineno+1))
+self.raw_docstrings.update(range(slineno, elineno+1))
 elif toktype == token.NEWLINE:
 if first_line is not None and elineno != first_line:
 # We're at the end of a line, and we've ended on a
 # different line than the first line of the statement,
 # so record a multi-line range.
 for l in range(first_line, elineno+1):
-self.multiline[l] = first_line
+self._multiline[l] = first_line
 first_line = None
+first_on_line = True
 if ttext.strip() and toktype != tokenize.COMMENT:
 # A non-whitespace token.
 empty = False
 if first_line is None:
 first_line = slineno
 # Check whether to end an excluded suite.
 if excluding and indent <= exclude_indent:
 excluding = False
 if excluding:
-self.excluded.add(elineno)
+self.raw_excluded.add(elineno)
+first_on_line = False
 prev_toktype = toktype
 # Find the starts of the executable statements.
 if not empty:
-self.statement_starts.update(self.byte_parser._find_statements())
+self.raw_statements.update(self.byte_parser._find_statements())
 def first_line(self, line):
 """Return the first line number of the statement including `line`."""
-first_line = self.multiline.get(line)
+return self._multiline.get(line, line)
-if first_line:
-return first_line
-else:
-return line
 def first_lines(self, lines):
 """Map the line numbers in `lines` to the correct first line of the
 statement.
 """Implement `FileReporter.translate_lines`."""
 return self.first_lines(lines)
 def translate_arcs(self, arcs):
 """Implement `FileReporter.translate_arcs`."""
-return [
+return [(self.first_line(a), self.first_line(b)) for (a, b) in arcs]
-(self.first_line(a), self.first_line(b))
-for (a, b) in arcs
-]
-@expensive
 def parse_source(self):
 """Parse source text to find executable lines, excluded lines, etc.
-Return values are 1) a set of executable line numbers, and 2) a set of
+Sets the .excluded and .statements attributes, normalized to the first
-excluded line numbers.
+line of multi-line statements.
-Reported line numbers are normalized to the first line of multi-line
-statements.
 """
 try:
 self._raw_parse()
 except (tokenize.TokenError, IndentationError) as err:
 if hasattr(err, "lineno"):
 lineno = err.lineno         # IndentationError
 else:
 lineno = err.args[1][0]     # TokenError
 raise NotPython(
-"Couldn't parse '%s' as Python source: '%s' at line %d" % (
+u"Couldn't parse '%s' as Python source: '%s' at line %d" % (
 self.filename, err.args[0], lineno
 )
 )
-excluded_lines = self.first_lines(self.excluded)
+self.excluded = self.first_lines(self.raw_excluded)
-ignore = set()
-ignore.update(excluded_lines)
+ignore = self.excluded | self.raw_docstrings
-ignore.update(self.docstrings)
+starts = self.raw_statements - ignore
-starts = self.statement_starts - ignore
+self.statements = self.first_lines(starts) - ignore
-lines = self.first_lines(starts)
-lines -= ignore
-return lines, excluded_lines
 def arcs(self):
 """Get information about the arcs available in the code.
 Returns a set of line number pairs.  Line numbers have been normalized
 to the first line of multi-line statements.
 """
 if self._all_arcs is None:
-self._all_arcs = set()
+self._analyze_ast()
-for l1, l2 in self.byte_parser._all_arcs():
-fl1 = self.first_line(l1)
-fl2 = self.first_line(l2)
-if fl1 != fl2:
-self._all_arcs.add((fl1, fl2))
 return self._all_arcs
+def _analyze_ast(self):
+"""Run the AstArcAnalyzer and save its results.
+`_all_arcs` is the set of arcs in the code.
+"""
+aaa = AstArcAnalyzer(self.text, self.raw_statements, self._multiline)
+aaa.analyze()
+self._all_arcs = set()
+for l1, l2 in aaa.arcs:
+fl1 = self.first_line(l1)
+fl2 = self.first_line(l2)
+if fl1 != fl2:
+self._all_arcs.add((fl1, fl2))
+self._missing_arc_fragments = aaa.missing_arc_fragments
 def exit_counts(self):
 """Get a count of exits from that each line.
 Excluded lines are excluded.
 """
-excluded_lines = self.first_lines(self.excluded)
 exit_counts = collections.defaultdict(int)
 for l1, l2 in self.arcs():
 if l1 < 0:
 # Don't ever report -1 as a line number
 continue
-if l1 in excluded_lines:
+if l1 in self.excluded:
 # Don't report excluded lines as line numbers.
 continue
-if l2 in excluded_lines:
+if l2 in self.excluded:
 # Arcs to excluded lines shouldn't count.
 continue
 exit_counts[l1] += 1
 # Class definitions have one extra exit, so remove one for each:
-for l in self.classdefs:
+for l in self.raw_classdefs:
 # Ensure key is there: class definitions can include excluded lines.
 if l in exit_counts:
 exit_counts[l] -= 1
 return exit_counts
+def missing_arc_description(self, start, end, executed_arcs=None):
-## Opcodes that guide the ByteParser.
+"""Provide an English sentence describing a missing arc."""
+if self._missing_arc_fragments is None:
-def _opcode(name):
+self._analyze_ast()
-"""Return the opcode by name from the dis module."""
-return dis.opmap[name]
+actual_start = start
+if (
-def _opcode_set(*names):
+executed_arcs and
-"""Return a set of opcodes by the names in `names`."""
+end < 0 and end == -start and
-s = set()
+(end, start) not in executed_arcs and
-for name in names:
+(end, start) in self._missing_arc_fragments
-try:
+):
-s.add(_opcode(name))
+# It's a one-line callable, and we never even started it,
-except KeyError:
+# and we have a message about not starting it.
-pass
+start, end = end, start
-return s
+fragment_pairs = self._missing_arc_fragments.get((start, end), [(None, None)])
-# Opcodes that leave the code object.
-OPS_CODE_END = _opcode_set('RETURN_VALUE')
+msgs = []
+for fragment_pair in fragment_pairs:
-# Opcodes that unconditionally end the code chunk.
+smsg, emsg = fragment_pair
-OPS_CHUNK_END = _opcode_set(
-'JUMP_ABSOLUTE', 'JUMP_FORWARD', 'RETURN_VALUE', 'RAISE_VARARGS',
+if emsg is None:
-'BREAK_LOOP', 'CONTINUE_LOOP',
+if end < 0:
-)
+# Hmm, maybe we have a one-line callable, let's check.
+if (-end, end) in self._missing_arc_fragments:
-# Opcodes that unconditionally begin a new code chunk.  By starting new chunks
+return self.missing_arc_description(-end, end)
-# with unconditional jump instructions, we neatly deal with jumps to jumps
+emsg = "didn't jump to the function exit"
-# properly.
+else:
-OPS_CHUNK_BEGIN = _opcode_set('JUMP_ABSOLUTE', 'JUMP_FORWARD')
+emsg = "didn't jump to line {lineno}"
+emsg = emsg.format(lineno=end)
-# Opcodes that push a block on the block stack.
-OPS_PUSH_BLOCK = _opcode_set(
+msg = "line {start} {emsg}".format(start=actual_start, emsg=emsg)
-'SETUP_LOOP', 'SETUP_EXCEPT', 'SETUP_FINALLY', 'SETUP_WITH'
+if smsg is not None:
-)
+msg += ", because {smsg}".format(smsg=smsg.format(lineno=actual_start))
-# Block types for exception handling.
+msgs.append(msg)
-OPS_EXCEPT_BLOCKS = _opcode_set('SETUP_EXCEPT', 'SETUP_FINALLY')
+return " or ".join(msgs)
-# Opcodes that pop a block from the block stack.
-OPS_POP_BLOCK = _opcode_set('POP_BLOCK')
-# Opcodes that have a jump destination, but aren't really a jump.
-OPS_NO_JUMP = OPS_PUSH_BLOCK
-# Individual opcodes we need below.
-OP_BREAK_LOOP = _opcode('BREAK_LOOP')
-OP_END_FINALLY = _opcode('END_FINALLY')
-OP_COMPARE_OP = _opcode('COMPARE_OP')
-COMPARE_EXCEPTION = 10  # just have to get this constant from the code.
-OP_LOAD_CONST = _opcode('LOAD_CONST')
-OP_RETURN_VALUE = _opcode('RETURN_VALUE')
 class ByteParser(object):
-"""Parse byte codes to understand the structure of code."""
+"""Parse bytecode to understand the structure of code."""
 @contract(text='unicode')
 def __init__(self, text, code=None, filename=None):
 self.text = text
 if code:
 else:
 try:
 self.code = compile_unicode(text, filename, "exec")
 except SyntaxError as synerr:
 raise NotPython(
-"Couldn't parse '%s' as Python source: '%s' at line %d" % (
+u"Couldn't parse '%s' as Python source: '%s' at line %d" % (
 filename, synerr.msg, synerr.lineno
 )
 )
 # Alternative Python implementations don't always provide all the
 # attributes on code objects that we need to do the analysis.
-for attr in ['co_lnotab', 'co_firstlineno', 'co_consts', 'co_code']:
+for attr in ['co_lnotab', 'co_firstlineno', 'co_consts']:
 if not hasattr(self.code, attr):
 raise CoverageException(
 "This implementation of Python doesn't support code analysis.\n"
 "Run coverage.py under CPython for this command."
 )
 for bp in self.child_parsers():
 # Get all of the lineno information from this code.
 for _, l in bp._bytes_lines():
 yield l
-def _block_stack_repr(self, block_stack):               # pragma: debugging
-"""Get a string version of `block_stack`, for debugging."""
+#
-blocks = ", ".join(
+# AST analysis
-"(%s, %r)" % (dis.opname[b[0]], b[1]) for b in block_stack
+#
-)
-return "[" + blocks + "]"
+class LoopBlock(object):
+"""A block on the block stack representing a `for` or `while` loop."""
-def _split_into_chunks(self):
+def __init__(self, start):
-"""Split the code object into a list of `Chunk` objects.
+self.start = start
+self.break_exits = set()
-Each chunk is only entered at its first instruction, though there can
-be many exits from a chunk.
+class FunctionBlock(object):
-Returns a list of `Chunk` objects.
+"""A block on the block stack representing a function definition."""
+def __init__(self, start, name):
-"""
+self.start = start
-# The list of chunks so far, and the one we're working on.
+self.name = name
-chunks = []
-chunk = None
+class TryBlock(object):
-# A dict mapping byte offsets of line starts to the line numbers.
+"""A block on the block stack representing a `try` block."""
-bytes_lines_map = dict(self._bytes_lines())
+def __init__(self, handler_start=None, final_start=None):
+self.handler_start = handler_start
-# The block stack: loops and try blocks get pushed here for the
+self.final_start = final_start
-# implicit jumps that can occur.
+self.break_from = set()
-# Each entry is a tuple: (block type, destination)
+self.continue_from = set()
-block_stack = []
+self.return_from = set()
+self.raise_from = set()
-# Some op codes are followed by branches that should be ignored.  This
-# is a count of how many ignores are left.
-ignore_branch = 0
+class ArcStart(collections.namedtuple("Arc", "lineno, cause")):
+"""The information needed to start an arc.
-# We have to handle the last two bytecodes specially.
-ult = penult = None
+`lineno` is the line number the arc starts from.  `cause` is a fragment
+used as the startmsg for AstArcAnalyzer.missing_arc_fragments.
-# Get a set of all of the jump-to points.
-jump_to = set()
+"""
-bytecodes = list(ByteCodes(self.code.co_code))
+def __new__(cls, lineno, cause=None):
-for bc in bytecodes:
+return super(ArcStart, cls).__new__(cls, lineno, cause)
-if bc.jump_to >= 0:
-jump_to.add(bc.jump_to)
+# Define contract words that PyContract doesn't have.
-chunk_lineno = 0
+# ArcStarts is for a list or set of ArcStart's.
+new_contract('ArcStarts', lambda seq: all(isinstance(x, ArcStart) for x in seq))
-# Walk the byte codes building chunks.
-for bc in bytecodes:
-# Maybe have to start a new chunk.
+class AstArcAnalyzer(object):
-start_new_chunk = False
+"""Analyze source text with an AST to find executable code paths."""
-first_chunk = False
-if bc.offset in bytes_lines_map:
+@contract(text='unicode', statements=set)
-# Start a new chunk for each source line number.
+def __init__(self, text, statements, multiline):
-start_new_chunk = True
+self.root_node = ast.parse(neuter_encoding_declaration(text))
-chunk_lineno = bytes_lines_map[bc.offset]
+# TODO: I think this is happening in too many places.
-first_chunk = True
+self.statements = set(multiline.get(l, l) for l in statements)
-elif bc.offset in jump_to:
+self.multiline = multiline
-# To make chunks have a single entrance, we have to make a new
-# chunk when we get to a place some bytecode jumps to.
+if int(os.environ.get("COVERAGE_ASTDUMP", 0)):      # pragma: debugging
-start_new_chunk = True
+# Dump the AST so that failing tests have helpful output.
-elif bc.op in OPS_CHUNK_BEGIN:
+print("Statements: {}".format(self.statements))
-# Jumps deserve their own unnumbered chunk.  This fixes
+print("Multiline map: {}".format(self.multiline))
-# problems with jumps to jumps getting confused.
+ast_dump(self.root_node)
-start_new_chunk = True
+self.arcs = set()
-if not chunk or start_new_chunk:
-if chunk:
+# A map from arc pairs to a pair of sentence fragments: (startmsg, endmsg).
-chunk.exits.add(bc.offset)
+# For an arc from line 17, they should be usable like:
-chunk = Chunk(bc.offset, chunk_lineno, first_chunk)
+#    "Line 17 {endmsg}, because {startmsg}"
-if not chunks:
+self.missing_arc_fragments = collections.defaultdict(list)
-# The very first chunk of a code object is always an
+self.block_stack = []
-# entrance.
-chunk.entrance = True
+self.debug = bool(int(os.environ.get("COVERAGE_TRACK_ARCS", 0)))
-chunks.append(chunk)
+def analyze(self):
-# Look at the opcode.
+"""Examine the AST tree from `root_node` to determine possible arcs.
-if bc.jump_to >= 0 and bc.op not in OPS_NO_JUMP:
-if ignore_branch:
+This sets the `arcs` attribute to be a set of (from, to) line number
-# Someone earlier wanted us to ignore this branch.
+pairs.
-ignore_branch -= 1
-else:
+"""
-# The opcode has a jump, it's an exit for this chunk.
+for node in ast.walk(self.root_node):
-chunk.exits.add(bc.jump_to)
+node_name = node.__class__.__name__
+code_object_handler = getattr(self, "_code_object__" + node_name, None)
-if bc.op in OPS_CODE_END:
+if code_object_handler is not None:
-# The opcode can exit the code object.
+code_object_handler(node)
-chunk.exits.add(-self.code.co_firstlineno)
-if bc.op in OPS_PUSH_BLOCK:
+def add_arc(self, start, end, smsg=None, emsg=None):
-# The opcode adds a block to the block_stack.
+"""Add an arc, including message fragments to use if it is missing."""
-block_stack.append((bc.op, bc.jump_to))
+if self.debug:
-if bc.op in OPS_POP_BLOCK:
+print("\nAdding arc: ({}, {}): {!r}, {!r}".format(start, end, smsg, emsg))
-# The opcode pops a block from the block stack.
+print(short_stack(limit=6))
-block_stack.pop()
+self.arcs.add((start, end))
-if bc.op in OPS_CHUNK_END:
-# This opcode forces the end of the chunk.
+if smsg is not None or emsg is not None:
-if bc.op == OP_BREAK_LOOP:
+self.missing_arc_fragments[(start, end)].append((smsg, emsg))
-# A break is implicit: jump where the top of the
-# block_stack points.
+def nearest_blocks(self):
-chunk.exits.add(block_stack[-1][1])
+"""Yield the blocks in nearest-to-farthest order."""
-chunk = None
+return reversed(self.block_stack)
-if bc.op == OP_END_FINALLY:
-# For the finally clause we need to find the closest exception
+@contract(returns=int)
-# block, and use its jump target as an exit.
+def line_for_node(self, node):
-for block in reversed(block_stack):
+"""What is the right line number to use for this node?
-if block[0] in OPS_EXCEPT_BLOCKS:
-chunk.exits.add(block[1])
+This dispatches to _line__Node functions where needed.
-break
-if bc.op == OP_COMPARE_OP and bc.arg == COMPARE_EXCEPTION:
+"""
-# This is an except clause.  We want to overlook the next
+node_name = node.__class__.__name__
-# branch, so that except's don't count as branches.
+handler = getattr(self, "_line__" + node_name, None)
-ignore_branch += 1
+if handler is not None:
+return handler(node)
-penult = ult
+else:
-ult = bc
+return node.lineno
-if chunks:
+def _line__Assign(self, node):
-# The last two bytecodes could be a dummy "return None" that
+return self.line_for_node(node.value)
-# shouldn't be counted as real code. Every Python code object seems
-# to end with a return, and a "return None" is inserted if there
+def _line__Dict(self, node):
-# isn't an explicit return in the source.
+# Python 3.5 changed how dict literals are made.
-if ult and penult:
+if env.PYVERSION >= (3, 5) and node.keys:
-if penult.op == OP_LOAD_CONST and ult.op == OP_RETURN_VALUE:
+if node.keys[0] is not None:
-if self.code.co_consts[penult.arg] is None:
+return node.keys[0].lineno
-# This is "return None", but is it dummy?  A real line
+else:
-# would be a last chunk all by itself.
+# Unpacked dict literals `{**{'a':1}}` have None as the key,
-if chunks[-1].byte != penult.offset:
+# use the value in that case.
-ex = -self.code.co_firstlineno
+return node.values[0].lineno
-# Split the last chunk
+else:
-last_chunk = chunks[-1]
+return node.lineno
-last_chunk.exits.remove(ex)
-last_chunk.exits.add(penult.offset)
+def _line__List(self, node):
-chunk = Chunk(
+if node.elts:
-penult.offset, last_chunk.line, False
+return self.line_for_node(node.elts[0])
-)
+else:
-chunk.exits.add(ex)
+return node.lineno
-chunks.append(chunk)
+def _line__Module(self, node):
-# Give all the chunks a length.
+if node.body:
-chunks[-1].length = bc.next_offset - chunks[-1].byte
+return self.line_for_node(node.body[0])
-for i in range(len(chunks)-1):
+else:
-chunks[i].length = chunks[i+1].byte - chunks[i].byte
+# Modules have no line number, they always start at 1.
+return 1
-#self.validate_chunks(chunks)
-return chunks
+OK_TO_DEFAULT = set([
+"Assign", "Assert", "AugAssign", "Delete", "Exec", "Expr", "Global",
-def validate_chunks(self, chunks):                      # pragma: debugging
+"Import", "ImportFrom", "Nonlocal", "Pass", "Print",
-"""Validate the rule that chunks have a single entrance."""
+])
-# starts is the entrances to the chunks
-starts = set(ch.byte for ch in chunks)
+@contract(returns='ArcStarts')
-for ch in chunks:
+def add_arcs(self, node):
-assert all((ex in starts or ex < 0) for ex in ch.exits)
+"""Add the arcs for `node`.
-def _arcs(self):
+Return a set of ArcStarts, exits from this node to the next.
-"""Find the executable arcs in the code.
+"""
-Yields pairs: (from,to).  From and to are integer line numbers.  If
+node_name = node.__class__.__name__
-from is < 0, then the arc is an entrance into the code object.  If to
+handler = getattr(self, "_handle__" + node_name, None)
-is < 0, the arc is an exit from the code object.
+if handler is not None:
+return handler(node)
-"""
-chunks = self._split_into_chunks()
+if 0:
+node_name = node.__class__.__name__
-# A map from byte offsets to the chunk starting at that offset.
+if node_name not in self.OK_TO_DEFAULT:
-byte_chunks = dict((c.byte, c) for c in chunks)
+print("*** Unhandled: {0}".format(node))
+return set([ArcStart(self.line_for_node(node), cause=None)])
-# Traverse from the first chunk in each line, and yield arcs where
-# the trace function will be invoked.
+@contract(returns='ArcStarts')
-for chunk in chunks:
+def add_body_arcs(self, body, from_start=None, prev_starts=None):
-if chunk.entrance:
+"""Add arcs for the body of a compound statement.
-yield (-1, chunk.line)
+`body` is the body node.  `from_start` is a single `ArcStart` that can
-if not chunk.first:
+be the previous line in flow before this body.  `prev_starts` is a set
+of ArcStarts that can be the previous line.  Only one of them should be
+given.
+Returns a set of ArcStarts, the exits from this body.
+"""
+if prev_starts is None:
+prev_starts = set([from_start])
+for body_node in body:
+lineno = self.line_for_node(body_node)
+first_line = self.multiline.get(lineno, lineno)
+if first_line not in self.statements:
 continue
+for prev_start in prev_starts:
-chunks_considered = set()
+self.add_arc(prev_start.lineno, lineno, prev_start.cause)
-chunks_to_consider = [chunk]
+prev_starts = self.add_arcs(body_node)
-while chunks_to_consider:
+return prev_starts
-# Get the chunk we're considering, and make sure we don't
-# consider it again.
+def is_constant_expr(self, node):
-this_chunk = chunks_to_consider.pop()
+"""Is this a compile-time constant?"""
-chunks_considered.add(this_chunk)
+node_name = node.__class__.__name__
+if node_name in ["NameConstant", "Num"]:
-# For each exit, add the line number if the trace function
+return True
-# would be triggered, or add the chunk to those being
+elif node_name == "Name":
-# considered if not.
+if env.PY3 and node.id in ["True", "False", "None"]:
-for ex in this_chunk.exits:
+return True
-if ex < 0:
+return False
-yield (chunk.line, ex)
-else:
+# tests to write:
-next_chunk = byte_chunks[ex]
+# TODO: while EXPR:
-if next_chunk in chunks_considered:
+# TODO: while False:
-continue
+# TODO: listcomps hidden deep in other expressions
+# TODO: listcomps hidden in lists: x = [[i for i in range(10)]]
-# The trace function is invoked if visiting the first
+# TODO: nested function definitions
-# bytecode in a line, or if the transition is a
-# backward jump.
+@contract(exits='ArcStarts')
-backward_jump = next_chunk.byte < this_chunk.byte
+def process_break_exits(self, exits):
-if next_chunk.first or backward_jump:
+"""Add arcs due to jumps from `exits` being breaks."""
-if next_chunk.line != chunk.line:
+for block in self.nearest_blocks():
-yield (chunk.line, next_chunk.line)
+if isinstance(block, LoopBlock):
-else:
+block.break_exits.update(exits)
-chunks_to_consider.append(next_chunk)
+break
+elif isinstance(block, TryBlock) and block.final_start is not None:
-def _all_chunks(self):
+block.break_from.update(exits)
-"""Returns a list of `Chunk` objects for this code and its children.
+break
-See `_split_into_chunks` for details.
+@contract(exits='ArcStarts')
+def process_continue_exits(self, exits):
-"""
+"""Add arcs due to jumps from `exits` being continues."""
-chunks = []
+for block in self.nearest_blocks():
-for bp in self.child_parsers():
+if isinstance(block, LoopBlock):
-chunks.extend(bp._split_into_chunks())
+for xit in exits:
+self.add_arc(xit.lineno, block.start, xit.cause)
-return chunks
+break
+elif isinstance(block, TryBlock) and block.final_start is not None:
-def _all_arcs(self):
+block.continue_from.update(exits)
-"""Get the set of all arcs in this code object and its children.
+break
-See `_arcs` for details.
+@contract(exits='ArcStarts')
+def process_raise_exits(self, exits):
-"""
+"""Add arcs due to jumps from `exits` being raises."""
-arcs = set()
+for block in self.nearest_blocks():
-for bp in self.child_parsers():
+if isinstance(block, TryBlock):
-arcs.update(bp._arcs())
+if block.handler_start is not None:
+for xit in exits:
-return arcs
+self.add_arc(xit.lineno, block.handler_start, xit.cause)
+break
+elif block.final_start is not None:
-class Chunk(object):
+block.raise_from.update(exits)
-"""A sequence of byte codes with a single entrance.
+break
+elif isinstance(block, FunctionBlock):
-To analyze byte code, we have to divide it into chunks, sequences of byte
+for xit in exits:
-codes such that each chunk has only one entrance, the first instruction in
+self.add_arc(
-the block.
+xit.lineno, -block.start, xit.cause,
+"didn't except from function '{0}'".format(block.name),
-This is almost the CS concept of `basic block`_, except that we're willing
+)
-to have many exits from a chunk, and "basic block" is a more cumbersome
+break
-term.
+@contract(exits='ArcStarts')
-.. _basic block: http://en.wikipedia.org/wiki/Basic_block
+def process_return_exits(self, exits):
+"""Add arcs due to jumps from `exits` being returns."""
-`byte` is the offset to the bytecode starting this chunk.
+for block in self.nearest_blocks():
+if isinstance(block, TryBlock) and block.final_start is not None:
-`line` is the source line number containing this chunk.
+block.return_from.update(exits)
+break
-`first` is true if this is the first chunk in the source line.
+elif isinstance(block, FunctionBlock):
+for xit in exits:
-An exit < 0 means the chunk can leave the code (return).  The exit is
+self.add_arc(
-the negative of the starting line number of the code block.
+xit.lineno, -block.start, xit.cause,
+"didn't return from function '{0}'".format(block.name),
-The `entrance` attribute is a boolean indicating whether the code object
+)
-can be entered at this chunk.
+break
+## Handlers
+@contract(returns='ArcStarts')
+def _handle__Break(self, node):
+here = self.line_for_node(node)
+break_start = ArcStart(here, cause="the break on line {lineno} wasn't executed")
+self.process_break_exits([break_start])
+return set()
+@contract(returns='ArcStarts')
+def _handle_decorated(self, node):
+"""Add arcs for things that can be decorated (classes and functions)."""
+last = self.line_for_node(node)
+if node.decorator_list:
+for dec_node in node.decorator_list:
+dec_start = self.line_for_node(dec_node)
+if dec_start != last:
+self.add_arc(last, dec_start)
+last = dec_start
+# The definition line may have been missed, but we should have it
+# in `self.statements`.  For some constructs, `line_for_node` is
+# not what we'd think of as the first line in the statement, so map
+# it to the first one.
+body_start = self.line_for_node(node.body[0])
+body_start = self.multiline.get(body_start, body_start)
+for lineno in range(last+1, body_start):
+if lineno in self.statements:
+self.add_arc(last, lineno)
+last = lineno
+# The body is handled in collect_arcs.
+return set([ArcStart(last, cause=None)])
+_handle__ClassDef = _handle_decorated
+@contract(returns='ArcStarts')
+def _handle__Continue(self, node):
+here = self.line_for_node(node)
+continue_start = ArcStart(here, cause="the continue on line {lineno} wasn't executed")
+self.process_continue_exits([continue_start])
+return set()
+@contract(returns='ArcStarts')
+def _handle__For(self, node):
+start = self.line_for_node(node.iter)
+self.block_stack.append(LoopBlock(start=start))
+from_start = ArcStart(start, cause="the loop on line {lineno} never started")
+exits = self.add_body_arcs(node.body, from_start=from_start)
+# Any exit from the body will go back to the top of the loop.
+for xit in exits:
+self.add_arc(xit.lineno, start, xit.cause)
+my_block = self.block_stack.pop()
+exits = my_block.break_exits
+from_start = ArcStart(start, cause="the loop on line {lineno} didn't complete")
+if node.orelse:
+else_exits = self.add_body_arcs(node.orelse, from_start=from_start)
+exits |= else_exits
+else:
+# no else clause: exit from the for line.
+exits.add(from_start)
+return exits
+_handle__AsyncFor = _handle__For
+_handle__FunctionDef = _handle_decorated
+_handle__AsyncFunctionDef = _handle_decorated
+@contract(returns='ArcStarts')
+def _handle__If(self, node):
+start = self.line_for_node(node.test)
+from_start = ArcStart(start, cause="the condition on line {lineno} was never true")
+exits = self.add_body_arcs(node.body, from_start=from_start)
+from_start = ArcStart(start, cause="the condition on line {lineno} was never false")
+exits |= self.add_body_arcs(node.orelse, from_start=from_start)
+return exits
+@contract(returns='ArcStarts')
+def _handle__Raise(self, node):
+here = self.line_for_node(node)
+raise_start = ArcStart(here, cause="the raise on line {lineno} wasn't executed")
+self.process_raise_exits([raise_start])
+# `raise` statement jumps away, no exits from here.
+return set()
+@contract(returns='ArcStarts')
+def _handle__Return(self, node):
+here = self.line_for_node(node)
+return_start = ArcStart(here, cause="the return on line {lineno} wasn't executed")
+self.process_return_exits([return_start])
+# `return` statement jumps away, no exits from here.
+return set()
+@contract(returns='ArcStarts')
+def _handle__Try(self, node):
+if node.handlers:
+handler_start = self.line_for_node(node.handlers[0])
+else:
+handler_start = None
+if node.finalbody:
+final_start = self.line_for_node(node.finalbody[0])
+else:
+final_start = None
+try_block = TryBlock(handler_start=handler_start, final_start=final_start)
+self.block_stack.append(try_block)
+start = self.line_for_node(node)
+exits = self.add_body_arcs(node.body, from_start=ArcStart(start, cause=None))
+# We're done with the `try` body, so this block no longer handles
+# exceptions. We keep the block so the `finally` clause can pick up
+# flows from the handlers and `else` clause.
+if node.finalbody:
+try_block.handler_start = None
+if node.handlers:
+# If there are `except` clauses, then raises in the try body
+# will already jump to them.  Start this set over for raises in
+# `except` and `else`.
+try_block.raise_from = set([])
+else:
+self.block_stack.pop()
+handler_exits = set()
+if node.handlers:
+last_handler_start = None
+for handler_node in node.handlers:
+handler_start = self.line_for_node(handler_node)
+if last_handler_start is not None:
+self.add_arc(last_handler_start, handler_start)
+last_handler_start = handler_start
+from_cause = "the exception caught by line {lineno} didn't happen"
+from_start = ArcStart(handler_start, cause=from_cause)
+handler_exits |= self.add_body_arcs(handler_node.body, from_start=from_start)
+if node.orelse:
+exits = self.add_body_arcs(node.orelse, prev_starts=exits)
+exits |= handler_exits
+if node.finalbody:
+self.block_stack.pop()
+final_from = (                  # You can get to the `finally` clause from:
+exits |                         # the exits of the body or `else` clause,
+try_block.break_from |          # or a `break`,
+try_block.continue_from |       # or a `continue`,
+try_block.raise_from |          # or a `raise`,
+try_block.return_from           # or a `return`.
+)
+exits = self.add_body_arcs(node.finalbody, prev_starts=final_from)
+if try_block.break_from:
+break_exits = self._combine_finally_starts(try_block.break_from, exits)
+self.process_break_exits(break_exits)
+if try_block.continue_from:
+continue_exits = self._combine_finally_starts(try_block.continue_from, exits)
+self.process_continue_exits(continue_exits)
+if try_block.raise_from:
+raise_exits = self._combine_finally_starts(try_block.raise_from, exits)
+self.process_raise_exits(raise_exits)
+if try_block.return_from:
+return_exits = self._combine_finally_starts(try_block.return_from, exits)
+self.process_return_exits(return_exits)
+return exits
+def _combine_finally_starts(self, starts, exits):
+"""Helper for building the cause of `finally` branches."""
+causes = []
+for lineno, cause in sorted(starts):
+if cause is not None:
+causes.append(cause.format(lineno=lineno))
+cause = " or ".join(causes)
+exits = set(ArcStart(ex.lineno, cause) for ex in exits)
+return exits
+@contract(returns='ArcStarts')
+def _handle__TryExcept(self, node):
+# Python 2.7 uses separate TryExcept and TryFinally nodes. If we get
+# TryExcept, it means there was no finally, so fake it, and treat as
+# a general Try node.
+node.finalbody = []
+return self._handle__Try(node)
+@contract(returns='ArcStarts')
+def _handle__TryFinally(self, node):
+# Python 2.7 uses separate TryExcept and TryFinally nodes. If we get
+# TryFinally, see if there's a TryExcept nested inside. If so, merge
+# them. Otherwise, fake fields to complete a Try node.
+node.handlers = []
+node.orelse = []
+first = node.body[0]
+if first.__class__.__name__ == "TryExcept" and node.lineno == first.lineno:
+assert len(node.body) == 1
+node.body = first.body
+node.handlers = first.handlers
+node.orelse = first.orelse
+return self._handle__Try(node)
+@contract(returns='ArcStarts')
+def _handle__While(self, node):
+constant_test = self.is_constant_expr(node.test)
+start = to_top = self.line_for_node(node.test)
+if constant_test:
+to_top = self.line_for_node(node.body[0])
+self.block_stack.append(LoopBlock(start=start))
+from_start = ArcStart(start, cause="the condition on line {lineno} was never true")
+exits = self.add_body_arcs(node.body, from_start=from_start)
+for xit in exits:
+self.add_arc(xit.lineno, to_top, xit.cause)
+exits = set()
+my_block = self.block_stack.pop()
+exits.update(my_block.break_exits)
+from_start = ArcStart(start, cause="the condition on line {lineno} was never false")
+if node.orelse:
+else_exits = self.add_body_arcs(node.orelse, from_start=from_start)
+exits |= else_exits
+else:
+# No `else` clause: you can exit from the start.
+if not constant_test:
+exits.add(from_start)
+return exits
+@contract(returns='ArcStarts')
+def _handle__With(self, node):
+start = self.line_for_node(node)
+exits = self.add_body_arcs(node.body, from_start=ArcStart(start))
+return exits
+_handle__AsyncWith = _handle__With
+def _code_object__Module(self, node):
+start = self.line_for_node(node)
+if node.body:
+exits = self.add_body_arcs(node.body, from_start=ArcStart(-start))
+for xit in exits:
+self.add_arc(xit.lineno, -start, xit.cause, "didn't exit the module")
+else:
+# Empty module.
+self.add_arc(-start, start)
+self.add_arc(start, -start)
+def _code_object__FunctionDef(self, node):
+start = self.line_for_node(node)
+self.block_stack.append(FunctionBlock(start=start, name=node.name))
+exits = self.add_body_arcs(node.body, from_start=ArcStart(-start))
+self.process_return_exits(exits)
+self.block_stack.pop()
+_code_object__AsyncFunctionDef = _code_object__FunctionDef
+def _code_object__ClassDef(self, node):
+start = self.line_for_node(node)
+self.add_arc(-start, start)
+exits = self.add_body_arcs(node.body, from_start=ArcStart(start))
+for xit in exits:
+self.add_arc(
+xit.lineno, -start, xit.cause,
+"didn't exit the body of class '{0}'".format(node.name),
+)
+def _make_oneline_code_method(noun):     # pylint: disable=no-self-argument
+"""A function to make methods for online callable _code_object__ methods."""
+def _code_object__oneline_callable(self, node):
+start = self.line_for_node(node)
+self.add_arc(-start, start, None, "didn't run the {0} on line {1}".format(noun, start))
+self.add_arc(
+start, -start, None,
+"didn't finish the {0} on line {1}".format(noun, start),
+)
+return _code_object__oneline_callable
+_code_object__Lambda = _make_oneline_code_method("lambda")
+_code_object__GeneratorExp = _make_oneline_code_method("generator expression")
+_code_object__DictComp = _make_oneline_code_method("dictionary comprehension")
+_code_object__SetComp = _make_oneline_code_method("set comprehension")
+if env.PY3:
+_code_object__ListComp = _make_oneline_code_method("list comprehension")
+SKIP_DUMP_FIELDS = ["ctx"]
+def _is_simple_value(value):
+"""Is `value` simple enough to be displayed on a single line?"""
+return (
+value in [None, [], (), {}, set()] or
+isinstance(value, (string_class, int, float))
+)
+# TODO: a test of ast_dump?
+def ast_dump(node, depth=0):
+"""Dump the AST for `node`.
+This recursively walks the AST, printing a readable version.
 """
-def __init__(self, byte, line, first):
+indent = " " * depth
-self.byte = byte
+if not isinstance(node, ast.AST):
-self.line = line
+print("{0}<{1} {2!r}>".format(indent, node.__class__.__name__, node))
-self.first = first
+return
-self.length = 0
-self.entrance = False
+lineno = getattr(node, "lineno", None)
-self.exits = set()
+if lineno is not None:
+linemark = " @ {0}".format(node.lineno)
-def __repr__(self):
+else:
-return "<%d+%d @%d%s%s %r>" % (
+linemark = ""
-self.byte,
+head = "{0}<{1}{2}".format(indent, node.__class__.__name__, linemark)
-self.length,
-self.line,
+named_fields = [
-"!" if self.first else "",
+(name, value)
-"v" if self.entrance else "",
+for name, value in ast.iter_fields(node)
-list(self.exits),
+if name not in SKIP_DUMP_FIELDS
-)
+]
+if not named_fields:
+print("{0}>".format(head))
+elif len(named_fields) == 1 and _is_simple_value(named_fields[0][1]):
+field_name, value = named_fields[0]
+print("{0} {1}: {2!r}>".format(head, field_name, value))
+else:
+print(head)
+if 0:
+print("{0}# mro: {1}".format(
+indent, ", ".join(c.__name__ for c in node.__class__.__mro__[1:]),
+))
+next_indent = indent + "    "
+for field_name, value in named_fields:
+prefix = "{0}{1}:".format(next_indent, field_name)
+if _is_simple_value(value):
+print("{0} {1!r}".format(prefix, value))
+elif isinstance(value, list):
+print("{0} [".format(prefix))
+for n in value:
+ast_dump(n, depth + 8)
+print("{0}]".format(next_indent))
+else:
+print(prefix)
+ast_dump(value, depth + 8)
+print("{0}>".format(indent))

Mercurial Repositories > eric / file comparison

comparison: DebugClients/Python3/coverage/parser.py

DebugClients/Python3/coverage/parser.py