eric7/DebugClients/Python/coverage/parser.py

branch
eric7
changeset 8775
0802ae193343
parent 8527
2bd1325d727e
child 8929
fcca2fa618bf
equal deleted inserted replaced
8774:d728227e8ebb 8775:0802ae193343
9 import re 9 import re
10 import token 10 import token
11 import tokenize 11 import tokenize
12 12
13 from coverage import env 13 from coverage import env
14 from coverage.backward import range # pylint: disable=redefined-builtin
15 from coverage.backward import bytes_to_ints, string_class
16 from coverage.bytecode import code_objects 14 from coverage.bytecode import code_objects
17 from coverage.debug import short_stack 15 from coverage.debug import short_stack
16 from coverage.exceptions import NoSource, NotPython, StopEverything
18 from coverage.misc import contract, join_regex, new_contract, nice_pair, one_of 17 from coverage.misc import contract, join_regex, new_contract, nice_pair, one_of
19 from coverage.misc import NoSource, NotPython, StopEverything
20 from coverage.phystokens import compile_unicode, generate_tokens, neuter_encoding_declaration 18 from coverage.phystokens import compile_unicode, generate_tokens, neuter_encoding_declaration
21 19
22 20
23 class PythonParser(object): 21 class PythonParser:
24 """Parse code to find executable lines, excluded lines, etc. 22 """Parse code to find executable lines, excluded lines, etc.
25 23
26 This information is all based on static analysis: no code execution is 24 This information is all based on static analysis: no code execution is
27 involved. 25 involved.
28 26
40 self.text = text 38 self.text = text
41 if not self.text: 39 if not self.text:
42 from coverage.python import get_python_source 40 from coverage.python import get_python_source
43 try: 41 try:
44 self.text = get_python_source(self.filename) 42 self.text = get_python_source(self.filename)
45 except IOError as err: 43 except OSError as err:
46 raise NoSource( 44 raise NoSource(f"No source for code: '{self.filename}': {err}") from err
47 "No source for code: '%s': %s" % (self.filename, err)
48 )
49 45
50 self.exclude = exclude 46 self.exclude = exclude
51 47
52 # The text lines of the parsed code. 48 # The text lines of the parsed code.
53 self.lines = self.text.split('\n') 49 self.lines = self.text.split('\n')
82 78
83 # A dict mapping line numbers to lexical statement starts for 79 # A dict mapping line numbers to lexical statement starts for
84 # multi-line statements. 80 # multi-line statements.
85 self._multiline = {} 81 self._multiline = {}
86 82
87 # Lazily-created ByteParser, arc data, and missing arc descriptions. 83 # Lazily-created arc data, and missing arc descriptions.
88 self._byte_parser = None
89 self._all_arcs = None 84 self._all_arcs = None
90 self._missing_arc_fragments = None 85 self._missing_arc_fragments = None
91
92 @property
93 def byte_parser(self):
94 """Create a ByteParser on demand."""
95 if not self._byte_parser:
96 self._byte_parser = ByteParser(self.text, filename=self.filename)
97 return self._byte_parser
98 86
99 def lines_matching(self, *regexes): 87 def lines_matching(self, *regexes):
100 """Find the lines matching one of a list of regexes. 88 """Find the lines matching one of a list of regexes.
101 89
102 Returns a set of line numbers, the lines that contain a match for one 90 Returns a set of line numbers, the lines that contain a match for one
103 of the regexes in `regexes`. The entire line needn't match, just a 91 of the regexes in `regexes`. The entire line needn't match, just a
104 part of it. 92 part of it.
105 93
106 """ 94 """
107 combined = join_regex(regexes) 95 combined = join_regex(regexes)
108 if env.PY2:
109 combined = combined.decode("utf8")
110 regex_c = re.compile(combined) 96 regex_c = re.compile(combined)
111 matches = set() 97 matches = set()
112 for i, ltext in enumerate(self.lines, start=1): 98 for i, ltext in enumerate(self.lines, start=1):
113 if regex_c.search(ltext): 99 if regex_c.search(ltext):
114 matches.add(i) 100 matches.add(i)
201 187
202 prev_toktype = toktype 188 prev_toktype = toktype
203 189
204 # Find the starts of the executable statements. 190 # Find the starts of the executable statements.
205 if not empty: 191 if not empty:
206 self.raw_statements.update(self.byte_parser._find_statements()) 192 byte_parser = ByteParser(self.text, filename=self.filename)
193 self.raw_statements.update(byte_parser._find_statements())
207 194
208 # The first line of modules can lie and say 1 always, even if the first 195 # The first line of modules can lie and say 1 always, even if the first
209 # line of code is later. If so, map 1 to the actual first line of the 196 # line of code is later. If so, map 1 to the actual first line of the
210 # module. 197 # module.
211 if env.PYBEHAVIOR.module_firstline_1 and self._multiline: 198 if env.PYBEHAVIOR.module_firstline_1 and self._multiline:
249 if hasattr(err, "lineno"): 236 if hasattr(err, "lineno"):
250 lineno = err.lineno # IndentationError 237 lineno = err.lineno # IndentationError
251 else: 238 else:
252 lineno = err.args[1][0] # TokenError 239 lineno = err.args[1][0] # TokenError
253 raise NotPython( 240 raise NotPython(
254 u"Couldn't parse '%s' as Python source: '%s' at line %d" % ( 241 f"Couldn't parse '{self.filename}' as Python source: " +
255 self.filename, err.args[0], lineno 242 f"{err.args[0]!r} at line {lineno}"
256 ) 243 ) from err
257 )
258 244
259 self.excluded = self.first_lines(self.raw_excluded) 245 self.excluded = self.first_lines(self.raw_excluded)
260 246
261 ignore = self.excluded | self.raw_docstrings 247 ignore = self.excluded | self.raw_docstrings
262 starts = self.raw_statements - ignore 248 starts = self.raw_statements - ignore
347 emsg = "didn't jump to the function exit" 333 emsg = "didn't jump to the function exit"
348 else: 334 else:
349 emsg = "didn't jump to line {lineno}" 335 emsg = "didn't jump to line {lineno}"
350 emsg = emsg.format(lineno=end) 336 emsg = emsg.format(lineno=end)
351 337
352 msg = "line {start} {emsg}".format(start=actual_start, emsg=emsg) 338 msg = f"line {actual_start} {emsg}"
353 if smsg is not None: 339 if smsg is not None:
354 msg += ", because {smsg}".format(smsg=smsg.format(lineno=actual_start)) 340 msg += f", because {smsg.format(lineno=actual_start)}"
355 341
356 msgs.append(msg) 342 msgs.append(msg)
357 343
358 return " or ".join(msgs) 344 return " or ".join(msgs)
359 345
360 346
361 class ByteParser(object): 347 class ByteParser:
362 """Parse bytecode to understand the structure of code.""" 348 """Parse bytecode to understand the structure of code."""
363 349
364 @contract(text='unicode') 350 @contract(text='unicode')
365 def __init__(self, text, code=None, filename=None): 351 def __init__(self, text, code=None, filename=None):
366 self.text = text 352 self.text = text
369 else: 355 else:
370 try: 356 try:
371 self.code = compile_unicode(text, filename, "exec") 357 self.code = compile_unicode(text, filename, "exec")
372 except SyntaxError as synerr: 358 except SyntaxError as synerr:
373 raise NotPython( 359 raise NotPython(
374 u"Couldn't parse '%s' as Python source: '%s' at line %d" % ( 360 "Couldn't parse '%s' as Python source: '%s' at line %d" % (
375 filename, synerr.msg, synerr.lineno 361 filename, synerr.msg, synerr.lineno
376 ) 362 )
377 ) 363 ) from synerr
378 364
379 # Alternative Python implementations don't always provide all the 365 # Alternative Python implementations don't always provide all the
380 # attributes on code objects that we need to do the analysis. 366 # attributes on code objects that we need to do the analysis.
381 for attr in ['co_lnotab', 'co_firstlineno']: 367 for attr in ['co_lnotab', 'co_firstlineno']:
382 if not hasattr(self.code, attr): 368 if not hasattr(self.code, attr):
383 raise StopEverything( # pragma: only jython 369 raise StopEverything( # pragma: only jython
384 "This implementation of Python doesn't support code analysis.\n" 370 "This implementation of Python doesn't support code analysis.\n" +
385 "Run coverage.py under another Python for this command." 371 "Run coverage.py under another Python for this command."
386 ) 372 )
387 373
388 def child_parsers(self): 374 def child_parsers(self):
389 """Iterate over all the code objects nested within this one. 375 """Iterate over all the code objects nested within this one.
403 for _, _, line in self.code.co_lines(): 389 for _, _, line in self.code.co_lines():
404 if line is not None: 390 if line is not None:
405 yield line 391 yield line
406 else: 392 else:
407 # Adapted from dis.py in the standard library. 393 # Adapted from dis.py in the standard library.
408 byte_increments = bytes_to_ints(self.code.co_lnotab[0::2]) 394 byte_increments = self.code.co_lnotab[0::2]
409 line_increments = bytes_to_ints(self.code.co_lnotab[1::2]) 395 line_increments = self.code.co_lnotab[1::2]
410 396
411 last_line_num = None 397 last_line_num = None
412 line_num = self.code.co_firstlineno 398 line_num = self.code.co_firstlineno
413 byte_num = 0 399 byte_num = 0
414 for byte_incr, line_incr in zip(byte_increments, line_increments): 400 for byte_incr, line_incr in zip(byte_increments, line_increments):
430 into all code objects reachable from `self.code`. 416 into all code objects reachable from `self.code`.
431 417
432 """ 418 """
433 for bp in self.child_parsers(): 419 for bp in self.child_parsers():
434 # Get all of the lineno information from this code. 420 # Get all of the lineno information from this code.
435 for l in bp._line_numbers(): 421 yield from bp._line_numbers()
436 yield l
437 422
438 423
439 # 424 #
440 # AST analysis 425 # AST analysis
441 # 426 #
442 427
443 class LoopBlock(object): 428 class BlockBase:
429 """
430 Blocks need to handle various exiting statements in their own ways.
431
432 All of these methods take a list of exits, and a callable `add_arc`
433 function that they can use to add arcs if needed. They return True if the
434 exits are handled, or False if the search should continue up the block
435 stack.
436 """
437 # pylint: disable=unused-argument
438 def process_break_exits(self, exits, add_arc):
439 """Process break exits."""
440 # Because break can only appear in loops, and most subclasses
441 # implement process_break_exits, this function is never reached.
442 raise AssertionError
443
444 def process_continue_exits(self, exits, add_arc):
445 """Process continue exits."""
446 # Because continue can only appear in loops, and most subclasses
447 # implement process_continue_exits, this function is never reached.
448 raise AssertionError
449
450 def process_raise_exits(self, exits, add_arc):
451 """Process raise exits."""
452 return False
453
454 def process_return_exits(self, exits, add_arc):
455 """Process return exits."""
456 return False
457
458
459 class LoopBlock(BlockBase):
444 """A block on the block stack representing a `for` or `while` loop.""" 460 """A block on the block stack representing a `for` or `while` loop."""
445 @contract(start=int) 461 @contract(start=int)
446 def __init__(self, start): 462 def __init__(self, start):
447 # The line number where the loop starts. 463 # The line number where the loop starts.
448 self.start = start 464 self.start = start
449 # A set of ArcStarts, the arcs from break statements exiting this loop. 465 # A set of ArcStarts, the arcs from break statements exiting this loop.
450 self.break_exits = set() 466 self.break_exits = set()
451 467
452 468 def process_break_exits(self, exits, add_arc):
453 class FunctionBlock(object): 469 self.break_exits.update(exits)
470 return True
471
472 def process_continue_exits(self, exits, add_arc):
473 for xit in exits:
474 add_arc(xit.lineno, self.start, xit.cause)
475 return True
476
477
478 class FunctionBlock(BlockBase):
454 """A block on the block stack representing a function definition.""" 479 """A block on the block stack representing a function definition."""
455 @contract(start=int, name=str) 480 @contract(start=int, name=str)
456 def __init__(self, start, name): 481 def __init__(self, start, name):
457 # The line number where the function starts. 482 # The line number where the function starts.
458 self.start = start 483 self.start = start
459 # The name of the function. 484 # The name of the function.
460 self.name = name 485 self.name = name
461 486
462 487 def process_raise_exits(self, exits, add_arc):
463 class TryBlock(object): 488 for xit in exits:
489 add_arc(
490 xit.lineno, -self.start, xit.cause,
491 f"didn't except from function {self.name!r}",
492 )
493 return True
494
495 def process_return_exits(self, exits, add_arc):
496 for xit in exits:
497 add_arc(
498 xit.lineno, -self.start, xit.cause,
499 f"didn't return from function {self.name!r}",
500 )
501 return True
502
503
504 class TryBlock(BlockBase):
464 """A block on the block stack representing a `try` block.""" 505 """A block on the block stack representing a `try` block."""
465 @contract(handler_start='int|None', final_start='int|None') 506 @contract(handler_start='int|None', final_start='int|None')
466 def __init__(self, handler_start, final_start): 507 def __init__(self, handler_start, final_start):
467 # The line number of the first "except" handler, if any. 508 # The line number of the first "except" handler, if any.
468 self.handler_start = handler_start 509 self.handler_start = handler_start
471 512
472 # The ArcStarts for breaks/continues/returns/raises inside the "try:" 513 # The ArcStarts for breaks/continues/returns/raises inside the "try:"
473 # that need to route through the "finally:" clause. 514 # that need to route through the "finally:" clause.
474 self.break_from = set() 515 self.break_from = set()
475 self.continue_from = set() 516 self.continue_from = set()
517 self.raise_from = set()
476 self.return_from = set() 518 self.return_from = set()
477 self.raise_from = set() 519
520 def process_break_exits(self, exits, add_arc):
521 if self.final_start is not None:
522 self.break_from.update(exits)
523 return True
524 return False
525
526 def process_continue_exits(self, exits, add_arc):
527 if self.final_start is not None:
528 self.continue_from.update(exits)
529 return True
530 return False
531
532 def process_raise_exits(self, exits, add_arc):
533 if self.handler_start is not None:
534 for xit in exits:
535 add_arc(xit.lineno, self.handler_start, xit.cause)
536 else:
537 assert self.final_start is not None
538 self.raise_from.update(exits)
539 return True
540
541 def process_return_exits(self, exits, add_arc):
542 if self.final_start is not None:
543 self.return_from.update(exits)
544 return True
545 return False
546
547
548 class WithBlock(BlockBase):
549 """A block on the block stack representing a `with` block."""
550 @contract(start=int)
551 def __init__(self, start):
552 # We only ever use this block if it is needed, so that we don't have to
553 # check this setting in all the methods.
554 assert env.PYBEHAVIOR.exit_through_with
555
556 # The line number of the with statement.
557 self.start = start
558
559 # The ArcStarts for breaks/continues/returns/raises inside the "with:"
560 # that need to go through the with-statement while exiting.
561 self.break_from = set()
562 self.continue_from = set()
563 self.return_from = set()
564
565 def _process_exits(self, exits, add_arc, from_set=None):
566 """Helper to process the four kinds of exits."""
567 for xit in exits:
568 add_arc(xit.lineno, self.start, xit.cause)
569 if from_set is not None:
570 from_set.update(exits)
571 return True
572
573 def process_break_exits(self, exits, add_arc):
574 return self._process_exits(exits, add_arc, self.break_from)
575
576 def process_continue_exits(self, exits, add_arc):
577 return self._process_exits(exits, add_arc, self.continue_from)
578
579 def process_raise_exits(self, exits, add_arc):
580 return self._process_exits(exits, add_arc)
581
582 def process_return_exits(self, exits, add_arc):
583 return self._process_exits(exits, add_arc, self.return_from)
478 584
479 585
480 class ArcStart(collections.namedtuple("Arc", "lineno, cause")): 586 class ArcStart(collections.namedtuple("Arc", "lineno, cause")):
481 """The information needed to start an arc. 587 """The information needed to start an arc.
482 588
488 "Line 17 didn't run because {cause}." The fragment can include "{lineno}" 594 "Line 17 didn't run because {cause}." The fragment can include "{lineno}"
489 to have `lineno` interpolated into it. 595 to have `lineno` interpolated into it.
490 596
491 """ 597 """
492 def __new__(cls, lineno, cause=None): 598 def __new__(cls, lineno, cause=None):
493 return super(ArcStart, cls).__new__(cls, lineno, cause) 599 return super().__new__(cls, lineno, cause)
494 600
495 601
496 # Define contract words that PyContract doesn't have. 602 # Define contract words that PyContract doesn't have.
497 # ArcStarts is for a list or set of ArcStart's. 603 # ArcStarts is for a list or set of ArcStart's.
498 new_contract('ArcStarts', lambda seq: all(isinstance(x, ArcStart) for x in seq)) 604 new_contract('ArcStarts', lambda seq: all(isinstance(x, ArcStart) for x in seq))
499 605
500 606
501 # Turn on AST dumps with an environment variable. 607 class NodeList:
502 # $set_env.py: COVERAGE_AST_DUMP - Dump the AST nodes when parsing code.
503 AST_DUMP = bool(int(os.environ.get("COVERAGE_AST_DUMP", 0)))
504
505 class NodeList(object):
506 """A synthetic fictitious node, containing a sequence of nodes. 608 """A synthetic fictitious node, containing a sequence of nodes.
507 609
508 This is used when collapsing optimized if-statements, to represent the 610 This is used when collapsing optimized if-statements, to represent the
509 unconditional execution of one of the clauses. 611 unconditional execution of one of the clauses.
510 612
511 """ 613 """
512 def __init__(self, body): 614 def __init__(self, body):
513 self.body = body 615 self.body = body
514 self.lineno = body[0].lineno 616 self.lineno = body[0].lineno
515 617
516
517 # TODO: some add_arcs methods here don't add arcs, they return them. Rename them. 618 # TODO: some add_arcs methods here don't add arcs, they return them. Rename them.
518 # TODO: the cause messages have too many commas. 619 # TODO: the cause messages have too many commas.
519 # TODO: Shouldn't the cause messages join with "and" instead of "or"? 620 # TODO: Shouldn't the cause messages join with "and" instead of "or"?
520 621
521 class AstArcAnalyzer(object): 622 def ast_parse(text):
623 """How we create an AST parse."""
624 return ast.parse(neuter_encoding_declaration(text))
625
626
627 class AstArcAnalyzer:
522 """Analyze source text with an AST to find executable code paths.""" 628 """Analyze source text with an AST to find executable code paths."""
523 629
524 @contract(text='unicode', statements=set) 630 @contract(text='unicode', statements=set)
525 def __init__(self, text, statements, multiline): 631 def __init__(self, text, statements, multiline):
526 self.root_node = ast.parse(neuter_encoding_declaration(text)) 632 self.root_node = ast_parse(text)
527 # TODO: I think this is happening in too many places. 633 # TODO: I think this is happening in too many places.
528 self.statements = {multiline.get(l, l) for l in statements} 634 self.statements = {multiline.get(l, l) for l in statements}
529 self.multiline = multiline 635 self.multiline = multiline
530 636
531 if AST_DUMP: # pragma: debugging 637 # Turn on AST dumps with an environment variable.
638 # $set_env.py: COVERAGE_AST_DUMP - Dump the AST nodes when parsing code.
639 dump_ast = bool(int(os.environ.get("COVERAGE_AST_DUMP", 0)))
640
641 if dump_ast: # pragma: debugging
532 # Dump the AST so that failing tests have helpful output. 642 # Dump the AST so that failing tests have helpful output.
533 print("Statements: {}".format(self.statements)) 643 print(f"Statements: {self.statements}")
534 print("Multiline map: {}".format(self.multiline)) 644 print(f"Multiline map: {self.multiline}")
535 ast_dump(self.root_node) 645 ast_dump(self.root_node)
536 646
537 self.arcs = set() 647 self.arcs = set()
538 648
539 # A map from arc pairs to a list of pairs of sentence fragments: 649 # A map from arc pairs to a list of pairs of sentence fragments:
562 672
563 @contract(start=int, end=int) 673 @contract(start=int, end=int)
564 def add_arc(self, start, end, smsg=None, emsg=None): 674 def add_arc(self, start, end, smsg=None, emsg=None):
565 """Add an arc, including message fragments to use if it is missing.""" 675 """Add an arc, including message fragments to use if it is missing."""
566 if self.debug: # pragma: debugging 676 if self.debug: # pragma: debugging
567 print("\nAdding arc: ({}, {}): {!r}, {!r}".format(start, end, smsg, emsg)) 677 print(f"\nAdding arc: ({start}, {end}): {smsg!r}, {emsg!r}")
568 print(short_stack(limit=6)) 678 print(short_stack(limit=6))
569 self.arcs.add((start, end)) 679 self.arcs.add((start, end))
570 680
571 if smsg is not None or emsg is not None: 681 if smsg is not None or emsg is not None:
572 self.missing_arc_fragments[(start, end)].append((smsg, emsg)) 682 self.missing_arc_fragments[(start, end)].append((smsg, emsg))
601 return self.line_for_node(node.value) 711 return self.line_for_node(node.value)
602 712
603 _line__ClassDef = _line_decorated 713 _line__ClassDef = _line_decorated
604 714
605 def _line__Dict(self, node): 715 def _line__Dict(self, node):
606 # Python 3.5 changed how dict literals are made. 716 if node.keys:
607 if env.PYVERSION >= (3, 5) and node.keys:
608 if node.keys[0] is not None: 717 if node.keys[0] is not None:
609 return node.keys[0].lineno 718 return node.keys[0].lineno
610 else: 719 else:
611 # Unpacked dict literals `{**{'a':1}}` have None as the key, 720 # Unpacked dict literals `{**{'a':1}}` have None as the key,
612 # use the value in that case. 721 # use the value in that case.
632 # Empty modules have no line number, they always start at 1. 741 # Empty modules have no line number, they always start at 1.
633 return 1 742 return 1
634 743
635 # The node types that just flow to the next node with no complications. 744 # The node types that just flow to the next node with no complications.
636 OK_TO_DEFAULT = { 745 OK_TO_DEFAULT = {
637 "Assign", "Assert", "AugAssign", "Delete", "Exec", "Expr", "Global", 746 "AnnAssign", "Assign", "Assert", "AugAssign", "Delete", "Expr", "Global",
638 "Import", "ImportFrom", "Nonlocal", "Pass", "Print", 747 "Import", "ImportFrom", "Nonlocal", "Pass",
639 } 748 }
640 749
641 @contract(returns='ArcStarts') 750 @contract(returns='ArcStarts')
642 def add_arcs(self, node): 751 def add_arcs(self, node):
643 """Add the arcs for `node`. 752 """Add the arcs for `node`.
659 handler = getattr(self, "_handle__" + node_name, None) 768 handler = getattr(self, "_handle__" + node_name, None)
660 if handler is not None: 769 if handler is not None:
661 return handler(node) 770 return handler(node)
662 else: 771 else:
663 # No handler: either it's something that's ok to default (a simple 772 # No handler: either it's something that's ok to default (a simple
664 # statement), or it's something we overlooked. Change this 0 to 1 773 # statement), or it's something we overlooked.
665 # to see if it's overlooked. 774 if env.TESTING:
666 if 0:
667 if node_name not in self.OK_TO_DEFAULT: 775 if node_name not in self.OK_TO_DEFAULT:
668 print("*** Unhandled: {}".format(node)) 776 raise Exception(f"*** Unhandled: {node}") # pragma: only failure
669 777
670 # Default for simple statements: one exit from this node. 778 # Default for simple statements: one exit from this node.
671 return {ArcStart(self.line_for_node(node))} 779 return {ArcStart(self.line_for_node(node))}
672 780
673 @one_of("from_start, prev_starts") 781 @one_of("from_start, prev_starts")
797 # is nearer. 905 # is nearer.
798 906
799 @contract(exits='ArcStarts') 907 @contract(exits='ArcStarts')
800 def process_break_exits(self, exits): 908 def process_break_exits(self, exits):
801 """Add arcs due to jumps from `exits` being breaks.""" 909 """Add arcs due to jumps from `exits` being breaks."""
802 for block in self.nearest_blocks(): 910 for block in self.nearest_blocks(): # pragma: always breaks
803 if isinstance(block, LoopBlock): 911 if block.process_break_exits(exits, self.add_arc):
804 block.break_exits.update(exits)
805 break
806 elif isinstance(block, TryBlock) and block.final_start is not None:
807 block.break_from.update(exits)
808 break 912 break
809 913
810 @contract(exits='ArcStarts') 914 @contract(exits='ArcStarts')
811 def process_continue_exits(self, exits): 915 def process_continue_exits(self, exits):
812 """Add arcs due to jumps from `exits` being continues.""" 916 """Add arcs due to jumps from `exits` being continues."""
813 for block in self.nearest_blocks(): 917 for block in self.nearest_blocks(): # pragma: always breaks
814 if isinstance(block, LoopBlock): 918 if block.process_continue_exits(exits, self.add_arc):
815 for xit in exits:
816 self.add_arc(xit.lineno, block.start, xit.cause)
817 break
818 elif isinstance(block, TryBlock) and block.final_start is not None:
819 block.continue_from.update(exits)
820 break 919 break
821 920
822 @contract(exits='ArcStarts') 921 @contract(exits='ArcStarts')
823 def process_raise_exits(self, exits): 922 def process_raise_exits(self, exits):
824 """Add arcs due to jumps from `exits` being raises.""" 923 """Add arcs due to jumps from `exits` being raises."""
825 for block in self.nearest_blocks(): 924 for block in self.nearest_blocks():
826 if isinstance(block, TryBlock): 925 if block.process_raise_exits(exits, self.add_arc):
827 if block.handler_start is not None:
828 for xit in exits:
829 self.add_arc(xit.lineno, block.handler_start, xit.cause)
830 break
831 elif block.final_start is not None:
832 block.raise_from.update(exits)
833 break
834 elif isinstance(block, FunctionBlock):
835 for xit in exits:
836 self.add_arc(
837 xit.lineno, -block.start, xit.cause,
838 "didn't except from function {!r}".format(block.name),
839 )
840 break 926 break
841 927
842 @contract(exits='ArcStarts') 928 @contract(exits='ArcStarts')
843 def process_return_exits(self, exits): 929 def process_return_exits(self, exits):
844 """Add arcs due to jumps from `exits` being returns.""" 930 """Add arcs due to jumps from `exits` being returns."""
845 for block in self.nearest_blocks(): 931 for block in self.nearest_blocks(): # pragma: always breaks
846 if isinstance(block, TryBlock) and block.final_start is not None: 932 if block.process_return_exits(exits, self.add_arc):
847 block.return_from.update(exits)
848 break 933 break
849 elif isinstance(block, FunctionBlock):
850 for xit in exits:
851 self.add_arc(
852 xit.lineno, -block.start, xit.cause,
853 "didn't return from function {!r}".format(block.name),
854 )
855 break
856
857 934
858 # Handlers: _handle__* 935 # Handlers: _handle__*
859 # 936 #
860 # Each handler deals with a specific AST node type, dispatched from 937 # Each handler deals with a specific AST node type, dispatched from
861 # add_arcs. Handlers return the set of exits from that node, and can 938 # add_arcs. Handlers return the set of exits from that node, and can
862 # also call self.add_arc to record arcs they find. These functions mirror 939 # also call self.add_arc to record arcs they find. These functions mirror
863 # the Python semantics of each syntactic construct. See the docstring 940 # the Python semantics of each syntactic construct. See the docstring
864 # for add_arcs to understand the concept of exits from a node. 941 # for add_arcs to understand the concept of exits from a node.
942 #
943 # Every node type that represents a statement should have a handler, or it
944 # should be listed in OK_TO_DEFAULT.
865 945
866 @contract(returns='ArcStarts') 946 @contract(returns='ArcStarts')
867 def _handle__Break(self, node): 947 def _handle__Break(self, node):
868 here = self.line_for_node(node) 948 here = self.line_for_node(node)
869 break_start = ArcStart(here, cause="the break on line {lineno} wasn't executed") 949 break_start = ArcStart(here, cause="the break on line {lineno} wasn't executed")
941 from_start = ArcStart(start, cause="the condition on line {lineno} was never false") 1021 from_start = ArcStart(start, cause="the condition on line {lineno} was never false")
942 exits |= self.add_body_arcs(node.orelse, from_start=from_start) 1022 exits |= self.add_body_arcs(node.orelse, from_start=from_start)
943 return exits 1023 return exits
944 1024
945 @contract(returns='ArcStarts') 1025 @contract(returns='ArcStarts')
1026 def _handle__Match(self, node):
1027 start = self.line_for_node(node)
1028 last_start = start
1029 exits = set()
1030 had_wildcard = False
1031 for case in node.cases:
1032 case_start = self.line_for_node(case.pattern)
1033 if isinstance(case.pattern, ast.MatchAs):
1034 had_wildcard = True
1035 self.add_arc(last_start, case_start, "the pattern on line {lineno} always matched")
1036 from_start = ArcStart(case_start, cause="the pattern on line {lineno} never matched")
1037 exits |= self.add_body_arcs(case.body, from_start=from_start)
1038 last_start = case_start
1039 if not had_wildcard:
1040 exits.add(from_start)
1041 return exits
1042
1043 @contract(returns='ArcStarts')
946 def _handle__NodeList(self, node): 1044 def _handle__NodeList(self, node):
947 start = self.line_for_node(node) 1045 start = self.line_for_node(node)
948 exits = self.add_body_arcs(node.body, from_start=ArcStart(start)) 1046 exits = self.add_body_arcs(node.body, from_start=ArcStart(start))
949 return exits 1047 return exits
950 1048
974 if node.finalbody: 1072 if node.finalbody:
975 final_start = self.line_for_node(node.finalbody[0]) 1073 final_start = self.line_for_node(node.finalbody[0])
976 else: 1074 else:
977 final_start = None 1075 final_start = None
978 1076
1077 # This is true by virtue of Python syntax: have to have either except
1078 # or finally, or both.
1079 assert handler_start is not None or final_start is not None
979 try_block = TryBlock(handler_start, final_start) 1080 try_block = TryBlock(handler_start, final_start)
980 self.block_stack.append(try_block) 1081 self.block_stack.append(try_block)
981 1082
982 start = self.line_for_node(node) 1083 start = self.line_for_node(node)
983 exits = self.add_body_arcs(node.body, from_start=ArcStart(start)) 1084 exits = self.add_body_arcs(node.body, from_start=ArcStart(start))
1088 cause = " or ".join(causes) 1189 cause = " or ".join(causes)
1089 exits = {ArcStart(xit.lineno, cause) for xit in exits} 1190 exits = {ArcStart(xit.lineno, cause) for xit in exits}
1090 return exits 1191 return exits
1091 1192
1092 @contract(returns='ArcStarts') 1193 @contract(returns='ArcStarts')
1093 def _handle__TryExcept(self, node):
1094 # Python 2.7 uses separate TryExcept and TryFinally nodes. If we get
1095 # TryExcept, it means there was no finally, so fake it, and treat as
1096 # a general Try node.
1097 node.finalbody = []
1098 return self._handle__Try(node)
1099
1100 @contract(returns='ArcStarts')
1101 def _handle__TryFinally(self, node):
1102 # Python 2.7 uses separate TryExcept and TryFinally nodes. If we get
1103 # TryFinally, see if there's a TryExcept nested inside. If so, merge
1104 # them. Otherwise, fake fields to complete a Try node.
1105 node.handlers = []
1106 node.orelse = []
1107
1108 first = node.body[0]
1109 if first.__class__.__name__ == "TryExcept" and node.lineno == first.lineno:
1110 assert len(node.body) == 1
1111 node.body = first.body
1112 node.handlers = first.handlers
1113 node.orelse = first.orelse
1114
1115 return self._handle__Try(node)
1116
1117 @contract(returns='ArcStarts')
1118 def _handle__While(self, node): 1194 def _handle__While(self, node):
1119 start = to_top = self.line_for_node(node.test) 1195 start = to_top = self.line_for_node(node.test)
1120 constant_test = self.is_constant_expr(node.test) 1196 constant_test = self.is_constant_expr(node.test)
1121 top_is_body0 = False 1197 top_is_body0 = False
1122 if constant_test and (env.PY3 or constant_test == "Num"): 1198 if constant_test:
1123 top_is_body0 = True 1199 top_is_body0 = True
1124 if env.PYBEHAVIOR.keep_constant_test: 1200 if env.PYBEHAVIOR.keep_constant_test:
1125 top_is_body0 = False 1201 top_is_body0 = False
1126 if top_is_body0: 1202 if top_is_body0:
1127 to_top = self.line_for_node(node.body[0]) 1203 to_top = self.line_for_node(node.body[0])
1144 return exits 1220 return exits
1145 1221
1146 @contract(returns='ArcStarts') 1222 @contract(returns='ArcStarts')
1147 def _handle__With(self, node): 1223 def _handle__With(self, node):
1148 start = self.line_for_node(node) 1224 start = self.line_for_node(node)
1225 if env.PYBEHAVIOR.exit_through_with:
1226 self.block_stack.append(WithBlock(start=start))
1149 exits = self.add_body_arcs(node.body, from_start=ArcStart(start)) 1227 exits = self.add_body_arcs(node.body, from_start=ArcStart(start))
1228 if env.PYBEHAVIOR.exit_through_with:
1229 with_block = self.block_stack.pop()
1230 with_exit = {ArcStart(start)}
1231 if exits:
1232 for xit in exits:
1233 self.add_arc(xit.lineno, start)
1234 exits = with_exit
1235 if with_block.break_from:
1236 self.process_break_exits(
1237 self._combine_finally_starts(with_block.break_from, with_exit)
1238 )
1239 if with_block.continue_from:
1240 self.process_continue_exits(
1241 self._combine_finally_starts(with_block.continue_from, with_exit)
1242 )
1243 if with_block.return_from:
1244 self.process_return_exits(
1245 self._combine_finally_starts(with_block.return_from, with_exit)
1246 )
1150 return exits 1247 return exits
1151 1248
1152 _handle__AsyncWith = _handle__With 1249 _handle__AsyncWith = _handle__With
1250
1251 # Code object dispatchers: _code_object__*
1252 #
1253 # These methods are used by analyze() as the start of the analysis.
1254 # There is one for each construct with a code object.
1153 1255
1154 def _code_object__Module(self, node): 1256 def _code_object__Module(self, node):
1155 start = self.line_for_node(node) 1257 start = self.line_for_node(node)
1156 if node.body: 1258 if node.body:
1157 exits = self.add_body_arcs(node.body, from_start=ArcStart(-start)) 1259 exits = self.add_body_arcs(node.body, from_start=ArcStart(-start))
1176 self.add_arc(-start, start) 1278 self.add_arc(-start, start)
1177 exits = self.add_body_arcs(node.body, from_start=ArcStart(start)) 1279 exits = self.add_body_arcs(node.body, from_start=ArcStart(start))
1178 for xit in exits: 1280 for xit in exits:
1179 self.add_arc( 1281 self.add_arc(
1180 xit.lineno, -start, xit.cause, 1282 xit.lineno, -start, xit.cause,
1181 "didn't exit the body of class {!r}".format(node.name), 1283 f"didn't exit the body of class {node.name!r}",
1182 ) 1284 )
1183 1285
1184 def _make_oneline_code_method(noun): # pylint: disable=no-self-argument 1286 def _make_expression_code_method(noun): # pylint: disable=no-self-argument
1185 """A function to make methods for online callable _code_object__ methods.""" 1287 """A function to make methods for expression-based callable _code_object__ methods."""
1186 def _code_object__oneline_callable(self, node): 1288 def _code_object__expression_callable(self, node):
1187 start = self.line_for_node(node) 1289 start = self.line_for_node(node)
1188 self.add_arc(-start, start, None, "didn't run the {} on line {}".format(noun, start)) 1290 self.add_arc(-start, start, None, f"didn't run the {noun} on line {start}")
1189 self.add_arc( 1291 self.add_arc(start, -start, None, f"didn't finish the {noun} on line {start}")
1190 start, -start, None, 1292 return _code_object__expression_callable
1191 "didn't finish the {} on line {}".format(noun, start), 1293
1192 ) 1294 _code_object__Lambda = _make_expression_code_method("lambda")
1193 return _code_object__oneline_callable 1295 _code_object__GeneratorExp = _make_expression_code_method("generator expression")
1194 1296 _code_object__DictComp = _make_expression_code_method("dictionary comprehension")
1195 _code_object__Lambda = _make_oneline_code_method("lambda") 1297 _code_object__SetComp = _make_expression_code_method("set comprehension")
1196 _code_object__GeneratorExp = _make_oneline_code_method("generator expression") 1298 _code_object__ListComp = _make_expression_code_method("list comprehension")
1197 _code_object__DictComp = _make_oneline_code_method("dictionary comprehension") 1299
1198 _code_object__SetComp = _make_oneline_code_method("set comprehension") 1300
1199 if env.PY3: 1301 # Code only used when dumping the AST for debugging.
1200 _code_object__ListComp = _make_oneline_code_method("list comprehension") 1302
1201 1303 SKIP_DUMP_FIELDS = ["ctx"]
1202 1304
1203 if AST_DUMP: # pragma: debugging 1305 def _is_simple_value(value):
1204 # Code only used when dumping the AST for debugging. 1306 """Is `value` simple enough to be displayed on a single line?"""
1205 1307 return (
1206 SKIP_DUMP_FIELDS = ["ctx"] 1308 value in [None, [], (), {}, set()] or
1207 1309 isinstance(value, (bytes, int, float, str))
1208 def _is_simple_value(value): 1310 )
1209 """Is `value` simple enough to be displayed on a single line?""" 1311
1210 return ( 1312 def ast_dump(node, depth=0, print=print): # pylint: disable=redefined-builtin
1211 value in [None, [], (), {}, set()] or 1313 """Dump the AST for `node`.
1212 isinstance(value, (string_class, int, float)) 1314
1213 ) 1315 This recursively walks the AST, printing a readable version.
1214 1316
1215 def ast_dump(node, depth=0): 1317 """
1216 """Dump the AST for `node`. 1318 indent = " " * depth
1217 1319 lineno = getattr(node, "lineno", None)
1218 This recursively walks the AST, printing a readable version. 1320 if lineno is not None:
1219 1321 linemark = f" @ {node.lineno},{node.col_offset}"
1220 """ 1322 if hasattr(node, "end_lineno"):
1221 indent = " " * depth 1323 linemark += ":"
1222 if not isinstance(node, ast.AST): 1324 if node.end_lineno != node.lineno:
1223 print("{}<{} {!r}>".format(indent, node.__class__.__name__, node)) 1325 linemark += f"{node.end_lineno},"
1224 return 1326 linemark += f"{node.end_col_offset}"
1225 1327 else:
1226 lineno = getattr(node, "lineno", None) 1328 linemark = ""
1227 if lineno is not None: 1329 head = f"{indent}<{node.__class__.__name__}{linemark}"
1228 linemark = " @ {}".format(node.lineno) 1330
1229 else: 1331 named_fields = [
1230 linemark = "" 1332 (name, value)
1231 head = "{}<{}{}".format(indent, node.__class__.__name__, linemark) 1333 for name, value in ast.iter_fields(node)
1232 1334 if name not in SKIP_DUMP_FIELDS
1233 named_fields = [ 1335 ]
1234 (name, value) 1336 if not named_fields:
1235 for name, value in ast.iter_fields(node) 1337 print(f"{head}>")
1236 if name not in SKIP_DUMP_FIELDS 1338 elif len(named_fields) == 1 and _is_simple_value(named_fields[0][1]):
1237 ] 1339 field_name, value = named_fields[0]
1238 if not named_fields: 1340 print(f"{head} {field_name}: {value!r}>")
1239 print("{}>".format(head)) 1341 else:
1240 elif len(named_fields) == 1 and _is_simple_value(named_fields[0][1]): 1342 print(head)
1241 field_name, value = named_fields[0] 1343 if 0:
1242 print("{} {}: {!r}>".format(head, field_name, value)) 1344 print("{}# mro: {}".format(
1243 else: 1345 indent, ", ".join(c.__name__ for c in node.__class__.__mro__[1:]),
1244 print(head) 1346 ))
1245 if 0: 1347 next_indent = indent + " "
1246 print("{}# mro: {}".format( 1348 for field_name, value in named_fields:
1247 indent, ", ".join(c.__name__ for c in node.__class__.__mro__[1:]), 1349 prefix = f"{next_indent}{field_name}:"
1248 )) 1350 if _is_simple_value(value):
1249 next_indent = indent + " " 1351 print(f"{prefix} {value!r}")
1250 for field_name, value in named_fields: 1352 elif isinstance(value, list):
1251 prefix = "{}{}:".format(next_indent, field_name) 1353 print(f"{prefix} [")
1252 if _is_simple_value(value): 1354 for n in value:
1253 print("{} {!r}".format(prefix, value)) 1355 if _is_simple_value(n):
1254 elif isinstance(value, list): 1356 print(f"{next_indent} {n!r}")
1255 print("{} [".format(prefix)) 1357 else:
1256 for n in value: 1358 ast_dump(n, depth + 8, print=print)
1257 ast_dump(n, depth + 8) 1359 print(f"{next_indent}]")
1258 print("{}]".format(next_indent)) 1360 else:
1259 else: 1361 print(prefix)
1260 print(prefix) 1362 ast_dump(value, depth + 8, print=print)
1261 ast_dump(value, depth + 8) 1363
1262 1364 print(f"{indent}>")
1263 print("{}>".format(indent))

eric ide

mercurial