--- a/Plugins/CheckerPlugins/SyntaxChecker/pyflakes/checker.py Sun Apr 27 19:18:34 2014 +0200
+++ b/Plugins/CheckerPlugins/SyntaxChecker/pyflakes/checker.py Sun Apr 27 19:40:48 2014 +0200
@@ -14,16 +14,14 @@
import doctest
import os
import sys
-try:
- builtin_vars = dir(__import__('builtins'))
- PY2 = False
-except ImportError:
- builtin_vars = dir(__import__('__builtin__'))
- PY2 = True
+
+PY2 = sys.version_info < (3, 0)
+PY32 = sys.version_info < (3, 3) # Python 2.5 to 3.2
+PY33 = sys.version_info < (3, 4) # Python 2.5 to 3.3
+builtin_vars = dir(__import__('__builtin__' if PY2 else 'builtins'))
try:
import ast
- iter_child_nodes = ast.iter_child_nodes
except ImportError: # Python 2.5
import _ast as ast
@@ -32,26 +30,6 @@
ast.ClassDef.decorator_list = ()
ast.FunctionDef.decorator_list = property(lambda s: s.decorators)
- def iter_child_nodes(node):
- """
- Yield all direct child nodes of *node*, that is, all fields that
- are nodes and all items of fields that are lists of nodes.
- """
- for name in node._fields:
- field = getattr(node, name, None)
- if isinstance(field, ast.AST):
- yield field
- elif isinstance(field, list):
- for item in field:
- yield item
-# Python >= 3.3 uses ast.Try instead of (ast.TryExcept + ast.TryFinally)
-if hasattr(ast, 'Try'):
- ast_TryExcept = ast.Try
- ast_TryFinally = ()
-else:
- ast_TryExcept = ast.TryExcept
- ast_TryFinally = ast.TryFinally
-
from . import messages
@@ -63,6 +41,55 @@
def getNodeType(node_class):
return node_class.__name__.upper()
+# Python >= 3.3 uses ast.Try instead of (ast.TryExcept + ast.TryFinally)
+if PY32:
+ def getAlternatives(n):
+ if isinstance(n, (ast.If, ast.TryFinally)):
+ return [n.body]
+ if isinstance(n, ast.TryExcept):
+ return [n.body + n.orelse] + [[hdl] for hdl in n.handlers]
+else:
+ def getAlternatives(n):
+ if isinstance(n, ast.If):
+ return [n.body]
+ if isinstance(n, ast.Try):
+ return [n.body + n.orelse] + [[hdl] for hdl in n.handlers]
+
+
+class _FieldsOrder(dict):
+ """Fix order of AST node fields."""
+
+ def _get_fields(self, node_class):
+ # handle iter before target, and generators before element
+ fields = node_class._fields
+ if 'iter' in fields:
+ key_first = 'iter'.find
+ elif 'generators' in fields:
+ key_first = 'generators'.find
+ else:
+ key_first = 'value'.find
+ return tuple(sorted(fields, key=key_first, reverse=True))
+
+ def __missing__(self, node_class):
+ self[node_class] = fields = self._get_fields(node_class)
+ return fields
+
+
+def iter_child_nodes(node, omit=None, _fields_order=_FieldsOrder()):
+ """
+ Yield all direct child nodes of *node*, that is, all fields that
+ are nodes and all items of fields that are lists of nodes.
+ """
+ for name in _fields_order[node.__class__]:
+ if name == omit:
+ continue
+ field = getattr(node, name, None)
+ if isinstance(field, ast.AST):
+ yield field
+ elif isinstance(field, list):
+ for item in field:
+ yield item
+
class Binding(object):
"""
@@ -90,8 +117,17 @@
self.source.lineno,
id(self))
+ def redefines(self, other):
+ return isinstance(other, Definition) and self.name == other.name
-class Importation(Binding):
+
+class Definition(Binding):
+ """
+ A binding that defines a function or a class.
+ """
+
+
+class Importation(Definition):
"""
A binding created by an import statement.
@@ -99,11 +135,18 @@
possibly including multiple dotted components.
@type fullName: C{str}
"""
+
def __init__(self, name, source):
self.fullName = name
+ self.redefined = []
name = name.split('.')[0]
super(Importation, self).__init__(name, source)
+ def redefines(self, other):
+ if isinstance(other, Importation):
+ return self.fullName == other.fullName
+ return isinstance(other, Definition) and self.name == other.name
+
class Argument(Binding):
"""
@@ -111,12 +154,6 @@
"""
-class Definition(Binding):
- """
- A binding that defines a function or a class.
- """
-
-
class Assignment(Binding):
"""
Represents binding a name with an explicit assignment.
@@ -149,16 +186,17 @@
Names which are imported and not otherwise used but appear in the value of
C{__all__} will not have an unused import warning reported for them.
"""
- def names(self):
- """
- Return a list of the names referenced by this binding.
- """
- names = []
- if isinstance(self.source, ast.List):
- for node in self.source.elts:
+
+ def __init__(self, name, source, scope):
+ if '__all__' in scope and isinstance(source, ast.AugAssign):
+ self.names = list(scope['__all__'].names)
+ else:
+ self.names = []
+ if isinstance(source.value, (ast.List, ast.Tuple)):
+ for node in source.value.elts:
if isinstance(node, ast.Str):
- names.append(node.s)
- return names
+ self.names.append(node.s)
+ super(ExportBinding, self).__init__(name, source)
class Scope(dict):
@@ -187,6 +225,8 @@
super(FunctionScope, self).__init__()
# Simplify: manage the special locals as globals
self.globals = self.alwaysUsed.copy()
+ self.returnValue = None # First non-empty return
+ self.isGenerator = False # Detect a generator
def unusedAssignments(self):
"""
@@ -236,7 +276,6 @@
nodeDepth = 0
offset = None
traceTree = False
- withDoctest = ('PYFLAKES_NODOCTEST' not in os.environ)
builtIns = set(builtin_vars).union(_MAGIC_GLOBALS)
_customBuiltIns = os.environ.get('PYFLAKES_BUILTINS')
@@ -244,7 +283,8 @@
builtIns.update(_customBuiltIns.split(','))
del _customBuiltIns
- def __init__(self, tree, filename='(none)', builtins=None):
+ def __init__(self, tree, filename='(none)', builtins=None,
+ withDoctest='PYFLAKES_DOCTEST' in os.environ):
self._nodeHandlers = {}
self._deferredFunctions = []
self._deferredAssignments = []
@@ -253,6 +293,7 @@
self.filename = filename
if builtins:
self.builtIns = self.builtIns.union(builtins)
+ self.withDoctest = withDoctest
self.scopeStack = [ModuleScope()]
self.exceptHandlers = [()]
self.futuresAllowed = True
@@ -310,126 +351,111 @@
which were imported but unused.
"""
for scope in self.deadScopes:
- export = isinstance(scope.get('__all__'), ExportBinding)
- if export:
- all = scope['__all__'].names()
+ if isinstance(scope.get('__all__'), ExportBinding):
+ all_names = set(scope['__all__'].names)
if not scope.importStarred and \
os.path.basename(self.filename) != '__init__.py':
# Look for possible mistakes in the export list
- undefined = set(all) - set(scope)
+ undefined = all_names.difference(scope)
for name in undefined:
self.report(messages.UndefinedExport,
scope['__all__'].source, name)
else:
- all = []
+ all_names = []
# Look for imported names that aren't used.
- for importation in scope.values():
- if isinstance(importation, Importation):
- if not importation.used and importation.name not in all:
- self.report(messages.UnusedImport,
- importation.source, importation.name)
+ for value in scope.values():
+ if isinstance(value, Importation):
+ used = value.used or value.name in all_names
+ if not used:
+ messg = messages.UnusedImport
+ self.report(messg, value.source, value.name)
+ for node in value.redefined:
+ if isinstance(self.getParent(node), ast.For):
+ messg = messages.ImportShadowedByLoopVar
+ elif used:
+ continue
+ else:
+ messg = messages.RedefinedWhileUnused
+ self.report(messg, node, value.name, value.source)
def pushScope(self, scopeClass=FunctionScope):
self.scopeStack.append(scopeClass())
- def pushFunctionScope(self): # XXX Deprecated
- self.pushScope(FunctionScope)
-
- def pushClassScope(self): # XXX Deprecated
- self.pushScope(ClassScope)
-
def report(self, messageClass, *args, **kwargs):
self.messages.append(messageClass(self.filename, *args, **kwargs))
- def hasParent(self, node, kind):
- while hasattr(node, 'parent'):
+ def getParent(self, node):
+ # Lookup the first parent which is not Tuple, List or Starred
+ while True:
node = node.parent
- if isinstance(node, kind):
- return True
+ if not hasattr(node, 'elts') and not hasattr(node, 'ctx'):
+ return node
- def getCommonAncestor(self, lnode, rnode, stop=None):
- if not stop:
- stop = self.root
+ def getCommonAncestor(self, lnode, rnode, stop):
+ if stop in (lnode, rnode) or not (hasattr(lnode, 'parent') and
+ hasattr(rnode, 'parent')):
+ return None
if lnode is rnode:
return lnode
- if stop in (lnode, rnode):
- return stop
- if not hasattr(lnode, 'parent') or not hasattr(rnode, 'parent'):
- return
- if (lnode.level > rnode.level):
+ if (lnode.depth > rnode.depth):
return self.getCommonAncestor(lnode.parent, rnode, stop)
- if (rnode.level > lnode.level):
+ if (lnode.depth < rnode.depth):
return self.getCommonAncestor(lnode, rnode.parent, stop)
return self.getCommonAncestor(lnode.parent, rnode.parent, stop)
- def descendantOf(self, node, ancestors, stop=None):
+ def descendantOf(self, node, ancestors, stop):
for a in ancestors:
- if self.getCommonAncestor(node, a, stop) not in (stop, None):
+ if self.getCommonAncestor(node, a, stop):
return True
return False
- def onFork(self, parent, lnode, rnode, items):
- return (self.descendantOf(lnode, items, parent) ^
- self.descendantOf(rnode, items, parent))
-
def differentForks(self, lnode, rnode):
"""True, if lnode and rnode are located on different forks of IF/TRY"""
- ancestor = self.getCommonAncestor(lnode, rnode)
- if isinstance(ancestor, ast.If):
- for fork in (ancestor.body, ancestor.orelse):
- if self.onFork(ancestor, lnode, rnode, fork):
+ ancestor = self.getCommonAncestor(lnode, rnode, self.root)
+ parts = getAlternatives(ancestor)
+ if parts:
+ for items in parts:
+ if self.descendantOf(lnode, items, ancestor) ^ \
+ self.descendantOf(rnode, items, ancestor):
return True
- elif isinstance(ancestor, ast_TryExcept):
- body = ancestor.body + ancestor.orelse
- for fork in [body] + [[hdl] for hdl in ancestor.handlers]:
- if self.onFork(ancestor, lnode, rnode, fork):
- return True
- elif isinstance(ancestor, ast_TryFinally):
- if self.onFork(ancestor, lnode, rnode, ancestor.body):
- return True
return False
- def addBinding(self, node, value, reportRedef=True):
+ def addBinding(self, node, value):
"""
Called when a binding is altered.
- `node` is the statement responsible for the change
- - `value` is the optional new value, a Binding instance, associated
- with the binding; if None, the binding is deleted if it exists.
- - if `reportRedef` is True (default), rebinding while unused will be
- reported.
+ - `value` is the new value, a Binding instance
"""
- redefinedWhileUnused = False
- if not isinstance(self.scope, ClassScope):
- for scope in self.scopeStack[::-1]:
- existing = scope.get(value.name)
- if (isinstance(existing, Importation)
- and not existing.used
- and (not isinstance(value, Importation) or
- value.fullName == existing.fullName)
- and reportRedef
- and not self.differentForks(node, existing.source)):
- redefinedWhileUnused = True
+ # assert value.source in (node, node.parent):
+ for scope in self.scopeStack[::-1]:
+ if value.name in scope:
+ break
+ existing = scope.get(value.name)
+
+ if existing and not self.differentForks(node, existing.source):
+
+ parent_stmt = self.getParent(value.source)
+ if isinstance(existing, Importation) and isinstance(parent_stmt, ast.For):
+ self.report(messages.ImportShadowedByLoopVar,
+ node, value.name, existing.source)
+
+ elif scope is self.scope:
+ if (isinstance(parent_stmt, ast.comprehension) and
+ not isinstance(self.getParent(existing.source),
+ (ast.For, ast.comprehension))):
+ self.report(messages.RedefinedInListComp,
+ node, value.name, existing.source)
+ elif not existing.used and value.redefines(existing):
self.report(messages.RedefinedWhileUnused,
node, value.name, existing.source)
- existing = self.scope.get(value.name)
- if not redefinedWhileUnused and self.hasParent(value.source, ast.ListComp):
- if (existing and reportRedef
- and not self.hasParent(existing.source, (ast.For, ast.ListComp))
- and not self.differentForks(node, existing.source)):
- self.report(messages.RedefinedInListComp,
- node, value.name, existing.source)
+ elif isinstance(existing, Importation) and value.redefines(existing):
+ existing.redefined.append(node)
- if (isinstance(existing, Definition)
- and not existing.used
- and not self.differentForks(node, existing.source)):
- self.report(messages.RedefinedWhileUnused,
- node, value.name, existing.source)
- else:
- self.scope[value.name] = value
+ self.scope[value.name] = value
def getNodeHandler(self, node_class):
try:
@@ -498,12 +524,13 @@
scope[name].used[1], name, scope[name].source)
break
- parent = getattr(node, 'parent', None)
- if isinstance(parent, (ast.For, ast.comprehension, ast.Tuple, ast.List)):
+ parent_stmt = self.getParent(node)
+ if isinstance(parent_stmt, (ast.For, ast.comprehension)) or (
+ parent_stmt != node.parent and
+ not self.isLiteralTupleUnpacking(parent_stmt)):
binding = Binding(name, node)
- elif (parent is not None and name == '__all__' and
- isinstance(self.scope, ModuleScope)):
- binding = ExportBinding(name, parent.value)
+ elif name == '__all__' and isinstance(self.scope, ModuleScope):
+ binding = ExportBinding(name, node.parent, self.scope)
else:
binding = Assignment(name, node)
if name in self.scope:
@@ -522,10 +549,17 @@
except KeyError:
self.report(messages.UndefinedName, node, name)
- def handleChildren(self, tree):
- for node in iter_child_nodes(tree):
+ def handleChildren(self, tree, omit=None):
+ for node in iter_child_nodes(tree, omit=omit):
self.handleNode(node, tree)
+ def isLiteralTupleUnpacking(self, node):
+ if isinstance(node, ast.Assign):
+ for child in node.targets + [node.value]:
+ if not hasattr(child, 'elts'):
+ return False
+ return True
+
def isDocstring(self, node):
"""
Determine if the given node is a docstring, as long as it is at the
@@ -555,7 +589,7 @@
self.isDocstring(node)):
self.futuresAllowed = False
self.nodeDepth += 1
- node.level = self.nodeDepth
+ node.depth = self.nodeDepth
node.parent = parent
try:
handler = self.getNodeHandler(node.__class__)
@@ -569,44 +603,49 @@
def handleDoctests(self, node):
try:
- docstring, node_lineno = self.getDocstring(node.body[0])
- if not docstring:
- return
- examples = self._getDoctestExamples(docstring)
+ (docstring, node_lineno) = self.getDocstring(node.body[0])
+ examples = docstring and self._getDoctestExamples(docstring)
except (ValueError, IndexError):
# e.g. line 6 of the docstring for <string> has inconsistent
# leading whitespace: ...
return
+ if not examples:
+ return
node_offset = self.offset or (0, 0)
self.pushScope()
+ underscore_in_builtins = '_' in self.builtIns
+ if not underscore_in_builtins:
+ self.builtIns.add('_')
for example in examples:
try:
tree = compile(example.source, "<doctest>", "exec", ast.PyCF_ONLY_AST)
except SyntaxError:
e = sys.exc_info()[1]
position = (node_lineno + example.lineno + e.lineno,
- example.indent + 4 + e.offset)
+ example.indent + 4 + (e.offset or 0))
self.report(messages.DoctestSyntaxError, node, position)
else:
self.offset = (node_offset[0] + node_lineno + example.lineno,
node_offset[1] + example.indent + 4)
self.handleChildren(tree)
self.offset = node_offset
+ if not underscore_in_builtins:
+ self.builtIns.remove('_')
self.popScope()
def ignore(self, node):
pass
# "stmt" type nodes
- RETURN = DELETE = PRINT = WHILE = IF = WITH = WITHITEM = RAISE = \
- TRYFINALLY = ASSERT = EXEC = EXPR = handleChildren
+ DELETE = PRINT = FOR = WHILE = IF = WITH = WITHITEM = RAISE = \
+ TRYFINALLY = ASSERT = EXEC = EXPR = ASSIGN = handleChildren
CONTINUE = BREAK = PASS = ignore
# "expr" type nodes
- BOOLOP = BINOP = UNARYOP = IFEXP = DICT = SET = YIELD = YIELDFROM = \
+ BOOLOP = BINOP = UNARYOP = IFEXP = DICT = SET = \
COMPARE = CALL = REPR = ATTRIBUTE = SUBSCRIPT = LIST = TUPLE = \
- STARRED = handleChildren
+ STARRED = NAMECONSTANT = handleChildren
NUM = STR = BYTES = ELLIPSIS = ignore
@@ -622,7 +661,7 @@
EQ = NOTEQ = LT = LTE = GT = GTE = IS = ISNOT = IN = NOTIN = ignore
# additional node types
- COMPREHENSION = KEYWORD = handleChildren
+ LISTCOMP = COMPREHENSION = KEYWORD = handleChildren
def GLOBAL(self, node):
"""
@@ -633,54 +672,12 @@
NONLOCAL = GLOBAL
- def LISTCOMP(self, node):
- # handle generators before element
- for gen in node.generators:
- self.handleNode(gen, node)
- self.handleNode(node.elt, node)
-
def GENERATOREXP(self, node):
self.pushScope(GeneratorScope)
- # handle generators before element
- for gen in node.generators:
- self.handleNode(gen, node)
- self.handleNode(node.elt, node)
- self.popScope()
-
- SETCOMP = GENERATOREXP
-
- def DICTCOMP(self, node):
- self.pushScope(GeneratorScope)
- for gen in node.generators:
- self.handleNode(gen, node)
- self.handleNode(node.key, node)
- self.handleNode(node.value, node)
+ self.handleChildren(node)
self.popScope()
- def FOR(self, node):
- """
- Process bindings for loop variables.
- """
- vars = []
-
- def collectLoopVars(n):
- if isinstance(n, ast.Name):
- vars.append(n.id)
- elif isinstance(n, ast.expr_context):
- return
- else:
- for c in iter_child_nodes(n):
- collectLoopVars(c)
-
- collectLoopVars(node.target)
- for varn in vars:
- if (isinstance(self.scope.get(varn), Importation)
- # unused ones will get an unused import warning
- and self.scope[varn].used):
- self.report(messages.ImportShadowedByLoopVar,
- node, varn, self.scope[varn].source)
-
- self.handleChildren(node)
+ DICTCOMP = SETCOMP = GENERATOREXP
def NAME(self, node):
"""
@@ -702,16 +699,28 @@
# arguments, but these aren't dispatched through here
raise RuntimeError("Got impossible expression context: %r" % (node.ctx,))
+ def RETURN(self, node):
+ if node.value and not self.scope.returnValue:
+ self.scope.returnValue = node.value
+ self.handleNode(node.value, node)
+
+ def YIELD(self, node):
+ self.scope.isGenerator = True
+ self.handleNode(node.value, node)
+
+ YIELDFROM = YIELD
+
def FUNCTIONDEF(self, node):
for deco in node.decorator_list:
self.handleNode(deco, node)
+ self.LAMBDA(node)
self.addBinding(node, FunctionDefinition(node.name, node))
- self.LAMBDA(node)
if self.withDoctest:
self.deferFunction(lambda: self.handleDoctests(node))
def LAMBDA(self, node):
args = []
+ annotations = []
if PY2:
def addArgs(arglist):
@@ -719,40 +728,47 @@
if isinstance(arg, ast.Tuple):
addArgs(arg.elts)
else:
- if arg.id in args:
- self.report(messages.DuplicateArgument,
- node, arg.id)
args.append(arg.id)
addArgs(node.args.args)
defaults = node.args.defaults
else:
for arg in node.args.args + node.args.kwonlyargs:
- if arg.arg in args:
- self.report(messages.DuplicateArgument,
- node, arg.arg)
args.append(arg.arg)
- self.handleNode(arg.annotation, node)
- if hasattr(node, 'returns'): # Only for FunctionDefs
- for annotation in (node.args.varargannotation,
- node.args.kwargannotation, node.returns):
- self.handleNode(annotation, node)
+ annotations.append(arg.annotation)
defaults = node.args.defaults + node.args.kw_defaults
- # vararg/kwarg identifiers are not Name nodes
- for wildcard in (node.args.vararg, node.args.kwarg):
+ # Only for Python3 FunctionDefs
+ is_py3_func = hasattr(node, 'returns')
+
+ for arg_name in ('vararg', 'kwarg'):
+ wildcard = getattr(node.args, arg_name)
if not wildcard:
continue
- if wildcard in args:
- self.report(messages.DuplicateArgument, node, wildcard)
- args.append(wildcard)
- for default in defaults:
- self.handleNode(default, node)
+ args.append(wildcard if PY33 else wildcard.arg)
+ if is_py3_func:
+ if PY33: # Python 2.5 to 3.3
+ argannotation = arg_name + 'annotation'
+ annotations.append(getattr(node.args, argannotation))
+ else: # Python >= 3.4
+ annotations.append(wildcard.annotation)
+
+ if is_py3_func:
+ annotations.append(node.returns)
+
+ if len(set(args)) < len(args):
+ for (idx, arg) in enumerate(args):
+ if arg in args[:idx]:
+ self.report(messages.DuplicateArgument, node, arg)
+
+ for child in annotations + defaults:
+ if child:
+ self.handleNode(child, node)
def runFunction():
self.pushScope()
for name in args:
- self.addBinding(node, Argument(name, node), reportRedef=False)
+ self.addBinding(node, Argument(name, node))
if isinstance(node.body, list):
# case for FunctionDefs
for stmt in node.body:
@@ -768,6 +784,17 @@
for name, binding in self.scope.unusedAssignments():
self.report(messages.UnusedVariable, binding.source, name)
self.deferAssignment(checkUnusedAssignments)
+
+ if PY32:
+ def checkReturnWithArgumentInsideGenerator():
+ """
+ Check to see if there is any return statement with
+ arguments but the function is a generator.
+ """
+ if self.scope.isGenerator and self.scope.returnValue:
+ self.report(messages.ReturnWithArgsInsideGenerator,
+ self.scope.returnValue)
+ self.deferAssignment(checkReturnWithArgumentInsideGenerator)
self.popScope()
self.deferFunction(runFunction)
@@ -793,11 +820,6 @@
self.popScope()
self.addBinding(node, ClassDefinition(node.name, node))
- def ASSIGN(self, node):
- self.handleNode(node.value, node)
- for target in node.targets:
- self.handleNode(target, node)
-
def AUGASSIGN(self, node):
self.handleNodeLoad(node.target)
self.handleNode(node.value, node)
@@ -843,9 +865,7 @@
self.handleNode(child, node)
self.exceptHandlers.pop()
# Process the other nodes: "except:", "else:", "finally:"
- for child in iter_child_nodes(node):
- if child not in node.body:
- self.handleNode(child, node)
+ self.handleChildren(node, omit='body')
TRYEXCEPT = TRY
