eric: comparison DebugClients/Python/coverage/parser.py

-:bedab77d0fa3
+:d6c795b5ce33
 from coverage import env
 from coverage.backward import range    # pylint: disable=redefined-builtin
 from coverage.backward import bytes_to_ints, string_class
 from coverage.bytecode import CodeObjects
 from coverage.debug import short_stack
-from coverage.misc import contract, new_contract, nice_pair, join_regex
+from coverage.misc import contract, join_regex, new_contract, nice_pair, one_of
-from coverage.misc import CoverageException, NoSource, NotPython
+from coverage.misc import NoSource, NotPython, StopEverything
 from coverage.phystokens import compile_unicode, generate_tokens, neuter_encoding_declaration
 class PythonParser(object):
 """Parse code to find executable lines, excluded lines, etc.
 empty = True
 first_on_line = True
 tokgen = generate_tokens(self.text)
 for toktype, ttext, (slineno, _), (elineno, _), ltext in tokgen:
-if self.show_tokens:                # pragma: not covered
+if self.show_tokens:                # pragma: debugging
 print("%10s %5s %-20r %r" % (
 tokenize.tok_name.get(toktype, toktype),
 nice_pair((slineno, elineno)), ttext, ltext
 ))
 if toktype == token.INDENT:
 )
 )
 # Alternative Python implementations don't always provide all the
 # attributes on code objects that we need to do the analysis.
-for attr in ['co_lnotab', 'co_firstlineno', 'co_consts']:
+for attr in ['co_lnotab', 'co_firstlineno']:
 if not hasattr(self.code, attr):
-raise CoverageException(
+raise StopEverything(                   # pragma: only jython
 "This implementation of Python doesn't support code analysis.\n"
-"Run coverage.py under CPython for this command."
+"Run coverage.py under another Python for this command."
 )
 def child_parsers(self):
 """Iterate over all the code objects nested within this one.
 # AST analysis
 #
 class LoopBlock(object):
 """A block on the block stack representing a `for` or `while` loop."""
+@contract(start=int)
 def __init__(self, start):
+# The line number where the loop starts.
 self.start = start
+# A set of ArcStarts, the arcs from break statements exiting this loop.
 self.break_exits = set()
 class FunctionBlock(object):
 """A block on the block stack representing a function definition."""
+@contract(start=int, name=str)
 def __init__(self, start, name):
+# The line number where the function starts.
 self.start = start
+# The name of the function.
 self.name = name
 class TryBlock(object):
 """A block on the block stack representing a `try` block."""
-def __init__(self, handler_start=None, final_start=None):
+@contract(handler_start='int|None', final_start='int|None')
+def __init__(self, handler_start, final_start):
+# The line number of the first "except" handler, if any.
 self.handler_start = handler_start
+# The line number of the "finally:" clause, if any.
 self.final_start = final_start
+# The ArcStarts for breaks/continues/returns/raises inside the "try:"
+# that need to route through the "finally:" clause.
 self.break_from = set()
 self.continue_from = set()
 self.return_from = set()
 self.raise_from = set()
 class ArcStart(collections.namedtuple("Arc", "lineno, cause")):
 """The information needed to start an arc.
-`lineno` is the line number the arc starts from.  `cause` is a fragment
+`lineno` is the line number the arc starts from.
-used as the startmsg for AstArcAnalyzer.missing_arc_fragments.
+`cause` is an English text fragment used as the `startmsg` for
+AstArcAnalyzer.missing_arc_fragments.  It will be used to describe why an
+arc wasn't executed, so should fit well into a sentence of the form,
+"Line 17 didn't run because {cause}."  The fragment can include "{lineno}"
+to have `lineno` interpolated into it.
 """
 def __new__(cls, lineno, cause=None):
 return super(ArcStart, cls).__new__(cls, lineno, cause)
 # Define contract words that PyContract doesn't have.
 # ArcStarts is for a list or set of ArcStart's.
 new_contract('ArcStarts', lambda seq: all(isinstance(x, ArcStart) for x in seq))
+# Turn on AST dumps with an environment variable.
+AST_DUMP = bool(int(os.environ.get("COVERAGE_AST_DUMP", 0)))
+class NodeList(object):
+"""A synthetic fictitious node, containing a sequence of nodes.
+This is used when collapsing optimized if-statements, to represent the
+unconditional execution of one of the clauses.
+"""
+def __init__(self, body):
+self.body = body
+self.lineno = body[0].lineno
 class AstArcAnalyzer(object):
 """Analyze source text with an AST to find executable code paths."""
 self.root_node = ast.parse(neuter_encoding_declaration(text))
 # TODO: I think this is happening in too many places.
 self.statements = set(multiline.get(l, l) for l in statements)
 self.multiline = multiline
-if int(os.environ.get("COVERAGE_ASTDUMP", 0)):      # pragma: debugging
+if AST_DUMP:                                # pragma: debugging
 # Dump the AST so that failing tests have helpful output.
-print("Statements: {}".format(self.statements))
+print("Statements: {0}".format(self.statements))
-print("Multiline map: {}".format(self.multiline))
+print("Multiline map: {0}".format(self.multiline))
 ast_dump(self.root_node)
 self.arcs = set()
-# A map from arc pairs to a pair of sentence fragments: (startmsg, endmsg).
+# A map from arc pairs to a list of pairs of sentence fragments:
+#   { (start, end): [(startmsg, endmsg), ...], }
+#
 # For an arc from line 17, they should be usable like:
 #    "Line 17 {endmsg}, because {startmsg}"
 self.missing_arc_fragments = collections.defaultdict(list)
 self.block_stack = []
 if code_object_handler is not None:
 code_object_handler(node)
 def add_arc(self, start, end, smsg=None, emsg=None):
 """Add an arc, including message fragments to use if it is missing."""
-if self.debug:
+if self.debug:                      # pragma: debugging
 print("\nAdding arc: ({}, {}): {!r}, {!r}".format(start, end, smsg, emsg))
 print(short_stack(limit=6))
 self.arcs.add((start, end))
 if smsg is not None or emsg is not None:
 def _line__Module(self, node):
 if node.body:
 return self.line_for_node(node.body[0])
 else:
-# Modules have no line number, they always start at 1.
+# Empty modules have no line number, they always start at 1.
 return 1
+# The node types that just flow to the next node with no complications.
 OK_TO_DEFAULT = set([
 "Assign", "Assert", "AugAssign", "Delete", "Exec", "Expr", "Global",
 "Import", "ImportFrom", "Nonlocal", "Pass", "Print",
 ])
 @contract(returns='ArcStarts')
 def add_arcs(self, node):
 """Add the arcs for `node`.
-Return a set of ArcStarts, exits from this node to the next.
+Return a set of ArcStarts, exits from this node to the next. Because a
+node represents an entire sub-tree (including its children), the exits
+from a node can be arbitrarily complex::
+if something(1):
+if other(2):
+doit(3)
+else:
+doit(5)
+There are two exits from line 1: they start at line 3 and line 5.
 """
 node_name = node.__class__.__name__
 handler = getattr(self, "_handle__" + node_name, None)
 if handler is not None:
 return handler(node)
+else:
-if 0:
+# No handler: either it's something that's ok to default (a simple
-node_name = node.__class__.__name__
+# statement), or it's something we overlooked. Change this 0 to 1
-if node_name not in self.OK_TO_DEFAULT:
+# to see if it's overlooked.
-print("*** Unhandled: {0}".format(node))
+if 0:
-return set([ArcStart(self.line_for_node(node), cause=None)])
+if node_name not in self.OK_TO_DEFAULT:
+print("*** Unhandled: {0}".format(node))
+# Default for simple statements: one exit from this node.
+return set([ArcStart(self.line_for_node(node))])
+@one_of("from_start, prev_starts")
 @contract(returns='ArcStarts')
 def add_body_arcs(self, body, from_start=None, prev_starts=None):
 """Add arcs for the body of a compound statement.
 `body` is the body node.  `from_start` is a single `ArcStart` that can
 prev_starts = set([from_start])
 for body_node in body:
 lineno = self.line_for_node(body_node)
 first_line = self.multiline.get(lineno, lineno)
 if first_line not in self.statements:
-continue
+body_node = self.find_non_missing_node(body_node)
+if body_node is None:
+continue
+lineno = self.line_for_node(body_node)
 for prev_start in prev_starts:
 self.add_arc(prev_start.lineno, lineno, prev_start.cause)
 prev_starts = self.add_arcs(body_node)
 return prev_starts
+def find_non_missing_node(self, node):
+"""Search `node` looking for a child that has not been optimized away.
+This might return the node you started with, or it will work recursively
+to find a child node in self.statements.
+Returns a node, or None if none of the node remains.
+"""
+# This repeats work just done in add_body_arcs, but this duplication
+# means we can avoid a function call in the 99.9999% case of not
+# optimizing away statements.
+lineno = self.line_for_node(node)
+first_line = self.multiline.get(lineno, lineno)
+if first_line in self.statements:
+return node
+missing_fn = getattr(self, "_missing__" + node.__class__.__name__, None)
+if missing_fn:
+node = missing_fn(node)
+else:
+node = None
+return node
+def _missing__If(self, node):
+# If the if-node is missing, then one of its children might still be
+# here, but not both. So return the first of the two that isn't missing.
+# Use a NodeList to hold the clauses as a single node.
+non_missing = self.find_non_missing_node(NodeList(node.body))
+if non_missing:
+return non_missing
+if node.orelse:
+return self.find_non_missing_node(NodeList(node.orelse))
+return None
+def _missing__NodeList(self, node):
+# A NodeList might be a mixture of missing and present nodes. Find the
+# ones that are present.
+non_missing_children = []
+for child in node.body:
+child = self.find_non_missing_node(child)
+if child is not None:
+non_missing_children.append(child)
+# Return the simplest representation of the present children.
+if not non_missing_children:
+return None
+if len(non_missing_children) == 1:
+return non_missing_children[0]
+return NodeList(non_missing_children)
 def is_constant_expr(self, node):
 """Is this a compile-time constant?"""
 node_name = node.__class__.__name__
 if node_name in ["NameConstant", "Num"]:
-return True
+return "Num"
 elif node_name == "Name":
-if env.PY3 and node.id in ["True", "False", "None"]:
+if node.id in ["True", "False", "None", "__debug__"]:
-return True
+return "Name"
-return False
+return None
-# tests to write:
+# In the fullness of time, these might be good tests to write:
-# TODO: while EXPR:
+#   while EXPR:
-# TODO: while False:
+#   while False:
-# TODO: listcomps hidden deep in other expressions
+#   listcomps hidden deep in other expressions
-# TODO: listcomps hidden in lists: x = [[i for i in range(10)]]
+#   listcomps hidden in lists: x = [[i for i in range(10)]]
-# TODO: nested function definitions
+#   nested function definitions
+# Exit processing: process_*_exits
+#
+# These functions process the four kinds of jump exits: break, continue,
+# raise, and return.  To figure out where an exit goes, we have to look at
+# the block stack context.  For example, a break will jump to the nearest
+# enclosing loop block, or the nearest enclosing finally block, whichever
+# is nearer.
 @contract(exits='ArcStarts')
 def process_break_exits(self, exits):
 """Add arcs due to jumps from `exits` being breaks."""
 for block in self.nearest_blocks():
 xit.lineno, -block.start, xit.cause,
 "didn't return from function '{0}'".format(block.name),
 )
 break
-## Handlers
+# Handlers: _handle__*
+#
+# Each handler deals with a specific AST node type, dispatched from
+# add_arcs.  Each deals with a particular kind of node type, and returns
+# the set of exits from that node. These functions mirror the Python
+# semantics of each syntactic construct.  See the docstring for add_arcs to
+# understand the concept of exits from a node.
 @contract(returns='ArcStarts')
 def _handle__Break(self, node):
 here = self.line_for_node(node)
 break_start = ArcStart(here, cause="the break on line {lineno} wasn't executed")
 last = dec_start
 # The definition line may have been missed, but we should have it
 # in `self.statements`.  For some constructs, `line_for_node` is
 # not what we'd think of as the first line in the statement, so map
 # it to the first one.
-body_start = self.line_for_node(node.body[0])
+if node.body:
-body_start = self.multiline.get(body_start, body_start)
+body_start = self.line_for_node(node.body[0])
-for lineno in range(last+1, body_start):
+body_start = self.multiline.get(body_start, body_start)
-if lineno in self.statements:
+for lineno in range(last+1, body_start):
-self.add_arc(last, lineno)
+if lineno in self.statements:
-last = lineno
+self.add_arc(last, lineno)
+last = lineno
 # The body is handled in collect_arcs.
-return set([ArcStart(last, cause=None)])
+return set([ArcStart(last)])
 _handle__ClassDef = _handle_decorated
 @contract(returns='ArcStarts')
 def _handle__Continue(self, node):
 from_start = ArcStart(start, cause="the loop on line {lineno} didn't complete")
 if node.orelse:
 else_exits = self.add_body_arcs(node.orelse, from_start=from_start)
 exits |= else_exits
 else:
-# no else clause: exit from the for line.
+# No else clause: exit from the for line.
 exits.add(from_start)
 return exits
 _handle__AsyncFor = _handle__For
 from_start = ArcStart(start, cause="the condition on line {lineno} was never false")
 exits |= self.add_body_arcs(node.orelse, from_start=from_start)
 return exits
 @contract(returns='ArcStarts')
+def _handle__NodeList(self, node):
+start = self.line_for_node(node)
+exits = self.add_body_arcs(node.body, from_start=ArcStart(start))
+return exits
+@contract(returns='ArcStarts')
 def _handle__Raise(self, node):
 here = self.line_for_node(node)
 raise_start = ArcStart(here, cause="the raise on line {lineno} wasn't executed")
 self.process_raise_exits([raise_start])
 # `raise` statement jumps away, no exits from here.
 if node.finalbody:
 final_start = self.line_for_node(node.finalbody[0])
 else:
 final_start = None
-try_block = TryBlock(handler_start=handler_start, final_start=final_start)
+try_block = TryBlock(handler_start, final_start)
 self.block_stack.append(try_block)
 start = self.line_for_node(node)
-exits = self.add_body_arcs(node.body, from_start=ArcStart(start, cause=None))
+exits = self.add_body_arcs(node.body, from_start=ArcStart(start))
 # We're done with the `try` body, so this block no longer handles
 # exceptions. We keep the block so the `finally` clause can pick up
 # flows from the handlers and `else` clause.
 if node.finalbody:
 try_block.continue_from |       # or a `continue`,
 try_block.raise_from |          # or a `raise`,
 try_block.return_from           # or a `return`.
 )
-exits = self.add_body_arcs(node.finalbody, prev_starts=final_from)
+final_exits = self.add_body_arcs(node.finalbody, prev_starts=final_from)
 if try_block.break_from:
-break_exits = self._combine_finally_starts(try_block.break_from, exits)
+self.process_break_exits(
-self.process_break_exits(break_exits)
+self._combine_finally_starts(try_block.break_from, final_exits)
+)
 if try_block.continue_from:
-continue_exits = self._combine_finally_starts(try_block.continue_from, exits)
+self.process_continue_exits(
-self.process_continue_exits(continue_exits)
+self._combine_finally_starts(try_block.continue_from, final_exits)
+)
 if try_block.raise_from:
-raise_exits = self._combine_finally_starts(try_block.raise_from, exits)
+self.process_raise_exits(
-self.process_raise_exits(raise_exits)
+self._combine_finally_starts(try_block.raise_from, final_exits)
+)
 if try_block.return_from:
-return_exits = self._combine_finally_starts(try_block.return_from, exits)
+self.process_return_exits(
-self.process_return_exits(return_exits)
+self._combine_finally_starts(try_block.return_from, final_exits)
+)
+if exits:
+# The finally clause's exits are only exits for the try block
+# as a whole if the try block had some exits to begin with.
+exits = final_exits
 return exits
+@contract(starts='ArcStarts', exits='ArcStarts', returns='ArcStarts')
 def _combine_finally_starts(self, starts, exits):
-"""Helper for building the cause of `finally` branches."""
+"""Helper for building the cause of `finally` branches.
+"finally" clauses might not execute their exits, and the causes could
+be due to a failure to execute any of the exits in the try block. So
+we use the causes from `starts` as the causes for `exits`.
+"""
 causes = []
-for lineno, cause in sorted(starts):
+for start in sorted(starts):
-if cause is not None:
+if start.cause is not None:
-causes.append(cause.format(lineno=lineno))
+causes.append(start.cause.format(lineno=start.lineno))
 cause = " or ".join(causes)
-exits = set(ArcStart(ex.lineno, cause) for ex in exits)
+exits = set(ArcStart(xit.lineno, cause) for xit in exits)
 return exits
 @contract(returns='ArcStarts')
 def _handle__TryExcept(self, node):
 # Python 2.7 uses separate TryExcept and TryFinally nodes. If we get
 @contract(returns='ArcStarts')
 def _handle__While(self, node):
 constant_test = self.is_constant_expr(node.test)
 start = to_top = self.line_for_node(node.test)
-if constant_test:
+if constant_test and (env.PY3 or constant_test == "Num"):
 to_top = self.line_for_node(node.body[0])
-self.block_stack.append(LoopBlock(start=start))
+self.block_stack.append(LoopBlock(start=to_top))
 from_start = ArcStart(start, cause="the condition on line {lineno} was never true")
 exits = self.add_body_arcs(node.body, from_start=from_start)
 for xit in exits:
 self.add_arc(xit.lineno, to_top, xit.cause)
 exits = set()
 _code_object__SetComp = _make_oneline_code_method("set comprehension")
 if env.PY3:
 _code_object__ListComp = _make_oneline_code_method("list comprehension")
-SKIP_DUMP_FIELDS = ["ctx"]
+if AST_DUMP:            # pragma: debugging
+# Code only used when dumping the AST for debugging.
-def _is_simple_value(value):
-"""Is `value` simple enough to be displayed on a single line?"""
+SKIP_DUMP_FIELDS = ["ctx"]
-return (
-value in [None, [], (), {}, set()] or
+def _is_simple_value(value):
-isinstance(value, (string_class, int, float))
+"""Is `value` simple enough to be displayed on a single line?"""
-)
+return (
+value in [None, [], (), {}, set()] or
-# TODO: a test of ast_dump?
+isinstance(value, (string_class, int, float))
-def ast_dump(node, depth=0):
+)
-"""Dump the AST for `node`.
+def ast_dump(node, depth=0):
-This recursively walks the AST, printing a readable version.
+"""Dump the AST for `node`.
-"""
+This recursively walks the AST, printing a readable version.
-indent = " " * depth
-if not isinstance(node, ast.AST):
+"""
-print("{0}<{1} {2!r}>".format(indent, node.__class__.__name__, node))
+indent = " " * depth
-return
+if not isinstance(node, ast.AST):
+print("{0}<{1} {2!r}>".format(indent, node.__class__.__name__, node))
-lineno = getattr(node, "lineno", None)
+return
-if lineno is not None:
-linemark = " @ {0}".format(node.lineno)
+lineno = getattr(node, "lineno", None)
-else:
+if lineno is not None:
-linemark = ""
+linemark = " @ {0}".format(node.lineno)
-head = "{0}<{1}{2}".format(indent, node.__class__.__name__, linemark)
+else:
+linemark = ""
-named_fields = [
+head = "{0}<{1}{2}".format(indent, node.__class__.__name__, linemark)
-(name, value)
-for name, value in ast.iter_fields(node)
+named_fields = [
-if name not in SKIP_DUMP_FIELDS
+(name, value)
-]
+for name, value in ast.iter_fields(node)
-if not named_fields:
+if name not in SKIP_DUMP_FIELDS
-print("{0}>".format(head))
+]
-elif len(named_fields) == 1 and _is_simple_value(named_fields[0][1]):
+if not named_fields:
-field_name, value = named_fields[0]
+print("{0}>".format(head))
-print("{0} {1}: {2!r}>".format(head, field_name, value))
+elif len(named_fields) == 1 and _is_simple_value(named_fields[0][1]):
-else:
+field_name, value = named_fields[0]
-print(head)
+print("{0} {1}: {2!r}>".format(head, field_name, value))
-if 0:
+else:
-print("{0}# mro: {1}".format(
+print(head)
-indent, ", ".join(c.__name__ for c in node.__class__.__mro__[1:]),
+if 0:
-))
+print("{0}# mro: {1}".format(
-next_indent = indent + "    "
+indent, ", ".join(c.__name__ for c in node.__class__.__mro__[1:]),
-for field_name, value in named_fields:
+))
-prefix = "{0}{1}:".format(next_indent, field_name)
+next_indent = indent + "    "
-if _is_simple_value(value):
+for field_name, value in named_fields:
-print("{0} {1!r}".format(prefix, value))
+prefix = "{0}{1}:".format(next_indent, field_name)
-elif isinstance(value, list):
+if _is_simple_value(value):
-print("{0} [".format(prefix))
+print("{0} {1!r}".format(prefix, value))
-for n in value:
+elif isinstance(value, list):
-ast_dump(n, depth + 8)
+print("{0} [".format(prefix))
-print("{0}]".format(next_indent))
+for n in value:
-else:
+ast_dump(n, depth + 8)
-print(prefix)
+print("{0}]".format(next_indent))
-ast_dump(value, depth + 8)
+else:
+print(prefix)
-print("{0}>".format(indent))
+ast_dump(value, depth + 8)
+print("{0}>".format(indent))

Mercurial Repositories > eric / file comparison

comparison: DebugClients/Python/coverage/parser.py

DebugClients/Python/coverage/parser.py