Plugins/CheckerPlugins/Pep8/Pep8Fixer.py

# -*- coding: utf-8 -*-

# Copyright (c) 2011 - 2013 Detlev Offenbach <detlev@die-offenbachs.de>
#

"""
Module implementing a class to fix certain PEP 8 issues.
"""

import os
import re
import tokenize
import io

from PyQt4.QtCore import QObject

from E5Gui import E5MessageBox

import Utilities

Pep8FixableIssues = ["E101", "E111", "W191", "E201", "E202", "E203",
                     "E211", "E221", "E222", "E223", "E224", "E225",
                     "E226", "E227", "E228", "E231", "E241", "E242",
                     "E251", "E261", "E262", "E271", "E272", "E273",
                     "E274", "W291", "W292", "W293", "E301", "E302",
                     "E303", "E304", "W391", "E401", "E502", "W603",
                     "E701", "E702", "E703", "E711", "E712"
                    ]


class Pep8Fixer(QObject):
    """
    Class implementing a fixer for certain PEP 8 issues.
    """
    def __init__(self, project, filename, sourceLines, fixCodes, inPlace):
        """
        Constructor
        
        @param project  reference to the project object (Project)
        @param filename name of the file to be fixed (string)
        @param sourceLines list of source lines including eol marker
            (list of string)
        @param fixCodes list of codes to be fixed as a comma separated
            string (string)
        @param inPlace flag indicating to modify the file in place (boolean)
        """
        super().__init__()
        
        self.__project = project
        self.__filename = filename
        self.__origName = ""
        self.__source = sourceLines[:]  # save a copy
        self.__fixCodes = [c.strip() for c in fixCodes.split(",") if c.strip()]
        self.fixed = 0
        
        self.__reindenter = None
        self.__eol = ""
        self.__indentWord = self.__getIndentWord()
        
        if not inPlace:
            self.__origName = self.__filename
            self.__filename = os.path.join(os.path.dirname(self.__filename),
                "fixed_" + os.path.basename(self.__filename))
        
        self.__fixes = {
            "E101": self.__fixE101,
            "E111": self.__fixE101,
            "W191": self.__fixE101,
            "E201": self.__fixE201,
            "E202": self.__fixE201,
            "E203": self.__fixE201,
            "E211": self.__fixE201,
            "E221": self.__fixE221,
            "E222": self.__fixE221,
            "E223": self.__fixE221,
            "E224": self.__fixE221,
            "E225": self.__fixE221,
            "E226": self.__fixE221,
            "E227": self.__fixE221,
            "E228": self.__fixE221,
            "E231": self.__fixE231,
            "E241": self.__fixE221,
            "E242": self.__fixE221,
            "E251": self.__fixE251,
            "E261": self.__fixE261,
            "E262": self.__fixE261,
            "E271": self.__fixE221,
            "E272": self.__fixE221,
            "E273": self.__fixE221,
            "E274": self.__fixE221,
            "W291": self.__fixW291,
            "W292": self.__fixW292,
            "W293": self.__fixW291,
            "E301": self.__fixE301,
            "E302": self.__fixE302,
            "E303": self.__fixE303,
            "E304": self.__fixE304,
            "W391": self.__fixW391,
            "E401": self.__fixE401,
            "E502": self.__fixE502,
            "W603": self.__fixW603,
            "E701": self.__fixE701,
            "E702": self.__fixE702,
            "E703": self.__fixE702,
            "E711": self.__fixE711,
            "E712": self.__fixE711,
        }
        self.__modified = False
        self.__stack = []   # these need to be fixed before the file is saved
                            # but after all inline fixes
    
    def saveFile(self, encoding):
        """
        Public method to save the modified file.
        
        @param encoding encoding of the source file (string)
        @return flag indicating success (boolean)
        """
        if not self.__modified:
            # no need to write
            return True
        
        # apply deferred fixes
        self.__finalize()
        
        txt = "".join(self.__source)
        try:
            Utilities.writeEncodedFile(self.__filename, txt, encoding)
        except (IOError, Utilities.CodingError, UnicodeError) as err:
            E5MessageBox.critical(self,
                self.trUtf8("Fix PEP 8 issues"),
                self.trUtf8(
                    """<p>Could not save the file <b>{0}</b>."""
                    """ Skipping it.</p><p>Reason: {1}</p>""")\
                    .format(self.__filename, str(err))
            )
            return False
        
        return True
    
    def fixIssue(self, line, pos, message):
        """
        Public method to fix the fixable issues.
        
        @param line line number of issue (integer)
        @param pos character position of issue (integer)
        @param message message text (string)
        @return flag indicating an applied fix (boolean) and a message for
            the fix (string)
        """
        code = message.split(None, 1)[0].strip()
        
        if line <= len(self.__source) and \
           (code in self.__fixCodes or len(self.__fixCodes) == 0) and \
           code in self.__fixes:
            res = self.__fixes[code](code, line, pos)
            if res[0]:
                self.__modified = True
                self.fixed += 1
        else:
            res = (False, "")
        
        return res
    
    def __finalize(self):
        """
        Private method to apply all deferred fixes.
        """
        for code, line, pos in reversed(self.__stack):
            self.__fixes[code](code, line, pos, apply=True)
    
    def __getEol(self):
        """
        Private method to get the applicable eol string.
        
        @return eol string (string)
        """
        if not self.__eol:
            if self.__origName:
                fn = self.__origName
            else:
                fn = self.__filename
            
            if self.__project.isOpen() and self.__project.isProjectFile(fn):
                self.__eol = self.__project.getEolString()
            else:
                self.__eol = Utilities.linesep()
        return self.__eol
    
    def __getIndentWord(self):
        """
        Private method to determine the indentation type.
        
        @return string to be used for an indentation (string)
        """
        sio = io.StringIO("".join(self.__source))
        indentWord = "    "     # default in case of failure
        try:
            for token in tokenize.generate_tokens(sio.readline):
                if token[0] == tokenize.INDENT:
                    indentWord = token[1]
                    break
        except (SyntaxError, tokenize.TokenError):
            pass
        return indentWord
    
    def __getIndent(self, line):
        """
        Private method to get the indentation string.
        
        @param line line to determine the indentation string from (string)
        @return indentation string (string)
        """
        return line.replace(line.lstrip(), "")
    
    def __fixWhitespace(self, line, offset, replacement):
        """
        Private method to correct whitespace at the given offset.
        
        @param line line to be corrected (string)
        @param offset offset within line (integer)
        @param replacement replacement string (string)
        @return corrected line
        """
        left = line[:offset].rstrip(" \t")
        right = line[offset:].lstrip(" \t")
        if right.startswith("#"):
            return line
        else:
            return left + replacement + right
    
    def __fixE101(self, code, line, pos):
        """
        Private method to fix obsolete tab usage and indentation errors
        (E101, E111, W191).
        
        @param code code of the issue (string)
        @param line line number of the issue (integer)
        @param pos position inside line (integer)
        @return flag indicating an applied fix (boolean) and a message for
            the fix (string)
        """
        if self.__reindenter is None:
            self.__reindenter = Pep8Reindenter(self.__source)
            self.__reindenter.run()
        fixedLine = self.__reindenter.fixedLine(line - 1)
        if fixedLine is not None:
            self.__source[line - 1] = fixedLine
            if code in ["E101", "W191"]:
                msg = self.trUtf8("Tab converted to 4 spaces.")
            else:
                msg = self.trUtf8("Indentation adjusted to be a multiple of four.")
            return (True, msg)
        else:
            return (False, self.trUtf8("Fix for {0} failed.").format(code))
    
    def __fixE201(self, code, line, pos):
        """
        Private method to fix extraneous whitespace (E201, E202,
        E203, E211).
        
        @param code code of the issue (string)
        @param line line number of the issue (integer)
        @param pos position inside line (integer)
        @return flag indicating an applied fix (boolean) and a message for
            the fix (string)
        """
        line = line - 1
        text = self.__source[line]
        
        if '"""' in text or "'''" in text or text.rstrip().endswith('\\'):
            return (False, self.trUtf8("Extraneous whitespace cannot be removed."))
        
        newText = self.__fixWhitespace(text, pos, '')
        if newText == text:
            return (False, "")
        
        self.__source[line] = newText
        return (True, self.trUtf8("Extraneous whitespace removed."))
    
    def __fixE221(self, code, line, pos):
        """
        Private method to fix extraneous whitespace around operator or
        keyword (E221, E222, E223, E224, E225, E226, E227, E228, E241,
        E242, E271, E272, E273, E274).
        
        @param code code of the issue (string)
        @param line line number of the issue (integer)
        @param pos position inside line (integer)
        @return flag indicating an applied fix (boolean) and a message for
            the fix (string)
        """
        line = line - 1
        text = self.__source[line]
        
        if '"""' in text or "'''" in text or text.rstrip().endswith('\\'):
            return (False, self.trUtf8("Extraneous whitespace cannot be removed."))
        
        newText = self.__fixWhitespace(text, pos, ' ')
        if newText == text:
            return (False, "")
        
        self.__source[line] = newText
        if code in ["E225", "E226", "E227", "E228"]:
            return (True, self.trUtf8("Missing whitespace added."))
        else:
            return (True, self.trUtf8("Extraneous whitespace removed."))
    
    def __fixE231(self, code, line, pos):
        """
        Private method to fix missing whitespace after ',;:'.
        
        @param code code of the issue (string)
        @param line line number of the issue (integer)
        @param pos position inside line (integer)
        @return flag indicating an applied fix (boolean) and a message for
            the fix (string)
        """
        line = line - 1
        pos = pos + 1
        self.__source[line] = self.__source[line][:pos] + \
                               " " + \
                               self.__source[line][pos:]
        return (True, self.trUtf8("Missing whitespace added."))
    
    def __fixE251(self, code, line, pos):
        """
        Private method to fix extraneous whitespace around keyword and
        default parameter equals (E251).
        
        @param code code of the issue (string)
        @param line line number of the issue (integer)
        @param pos position inside line (integer)
        @return flag indicating an applied fix (boolean) and a message for
            the fix (string)
        """
        line = line - 1
        text = self.__source[line]
        
        # This is necessary since pep8 sometimes reports columns that goes
        # past the end of the physical line. This happens in cases like,
        # foo(bar\n=None)
        col = min(pos, len(text) - 1)
        if text[col].strip():
            newText = text
        else:
            newText = text[:col].rstrip() + text[col:].lstrip()
        
        # There could be an escaped newline
        #
        #     def foo(a=\
        #             1)
        if newText.endswith(('=\\\n', '=\\\r\n', '=\\\r')):
            self.__source[line] = newText.rstrip("\n\r \t\\")
            self.__source[line + 1] = self.__source[line + 1].lstrip()
        else:
            self.__source[line] = newText
        return (True, self.trUtf8("Extraneous whitespace removed."))
    
    def __fixE261(self, code, line, pos):
        """
        Private method to fix whitespace before or after inline comment
        (E261, E262).
        
        @param code code of the issue (string)
        @param line line number of the issue (integer)
        @param pos position inside line (integer)
        @return flag indicating an applied fix (boolean) and a message for
            the fix (string)
        """
        line = line - 1
        text = self.__source[line]
        left = text[:pos].rstrip(' \t#')
        right = text[pos:].lstrip(' \t#')
        newText = left + ("  # " + right if right.strip() else right)
        self.__source[line] = newText
        return (True, self.trUtf8("Whitespace around comment sign corrected."))
    
    def __fixE301(self, code, line, pos, apply=False):
        """
        Private method to fix the need for one blank line (E301).
        
        @param code code of the issue (string)
        @param line line number of the issue (integer)
        @param pos position inside line (integer)
        @keyparam apply flag indicating, that the fix should be applied
            (boolean)
        @return flag indicating an applied fix (boolean) and a message for
            the fix (string)
        """
        if apply:
            self.__source.insert(line - 1, self.__getEol())
        else:
            self.__stack.append((code, line, pos))
        return (True, self.trUtf8("One blank line inserted."))
    
    def __fixE302(self, code, line, pos, apply=False):
        """
        Private method to fix the need for two blank lines (E302).
        """
        # count blank lines
        index = line - 1
        blanks = 0
        while index:
            if self.__source[index - 1].strip() == "":
                blanks += 1
                index -= 1
            else:
                break
        delta = blanks - 2
        
        if apply:
            line -= 1
            if delta < 0:
                # insert blank lines (one or two)
                while delta < 0:
                    self.__source.insert(line, self.__getEol())
                    delta += 1
            elif delta > 0:
                # delete superfluous blank lines
                while delta > 0:
                    del self.__source[line - 1]
                    line -= 1
                    delta -= 1
        else:
            self.__stack.append((code, line, pos))
        
        if delta < 0:
            msg = self.trUtf8("%n blank line(s) inserted.", "", -delta)
        elif delta > 0:
            msg = self.trUtf8("%n superfluous lines removed", "", delta)
        else:
            msg = ""
        return (True, msg)
    
    def __fixE303(self, code, line, pos, apply=False):
        """
        Private method to fix superfluous blank lines (E303).
        
        @param code code of the issue (string)
        @param line line number of the issue (integer)
        @param pos position inside line (integer)
        @keyparam apply flag indicating, that the fix should be applied
            (boolean)
        @return flag indicating an applied fix (boolean) and a message for
            the fix (string)
        """
        if apply:
            index = line - 3
            while index:
                if self.__source[index].strip() == "":
                    del self.__source[index]
                    index -= 1
                else:
                    break
        else:
            self.__stack.append((code, line, pos))
        return (True, self.trUtf8("Superfluous blank lines removed."))
    
    def __fixE304(self, code, line, pos, apply=False):
        """
        Private method to fix superfluous blank lines after a function
        decorator (E304).
        
        @param code code of the issue (string)
        @param line line number of the issue (integer)
        @param pos position inside line (integer)
        @keyparam apply flag indicating, that the fix should be applied
            (boolean)
        @return flag indicating an applied fix (boolean) and a message for
            the fix (string)
        """
        if apply:
            index = line - 2
            while index:
                if self.__source[index].strip() == "":
                    del self.__source[index]
                    index -= 1
                else:
                    break
        else:
            self.__stack.append((code, line, pos))
        return (True, self.trUtf8(
            "Superfluous blank lines after function decorator removed."))
    
    def __fixE401(self, code, line, pos, apply=False):
        """
        Private method to fix multiple imports on one line (E401).
        
        @param code code of the issue (string)
        @param line line number of the issue (integer)
        @param pos position inside line (integer)
        @keyparam apply flag indicating, that the fix should be applied
            (boolean)
        @return flag indicating an applied fix (boolean) and a message for
            the fix (string)
        """
        if apply:
            line = line - 1
            text = self.__source[line]
            if not text.lstrip().startswith("import"):
                return (False, "")
            
            # pep8 (1.3.1) reports false positive if there is an import
            # statement followed by a semicolon and some unrelated
            # statement with commas in it.
            if ';' in text:
                return (False, "")
            
            newText = text[:pos].rstrip("\t ,") + self.__getEol() + \
                self.__getIndent(text) + "import " + text[pos:].lstrip("\t ,")
            self.__source[line] = newText
        else:
            self.__stack.append((code, line, pos))
        return (True, self.trUtf8("Imports were put on separate lines."))
    
    def __fixE502(self, code, line, pos):
        """
        Private method to fix redundant backslash within brackets (E502).
        
        @param code code of the issue (string)
        @param line line number of the issue (integer)
        @param pos position inside line (integer)
        @return flag indicating an applied fix (boolean) and a message for
            the fix (string)
        """
        self.__source[line - 1] = self.__source[line - 1].rstrip("\n\r \t\\") + \
            self.__getEol()
        return (True, self.trUtf8("Redundant backslash in brackets removed."))
    
    def __fixE701(self, code, line, pos, apply=False):
        """
        Private method to fix colon-separated compund statements (E701).
        
        @param code code of the issue (string)
        @param line line number of the issue (integer)
        @param pos position inside line (integer)
        @keyparam apply flag indicating, that the fix should be applied
            (boolean)
        @return flag indicating an applied fix (boolean) and a message for
            the fix (string)
        """
        if apply:
            line = line - 1
            text = self.__source[line]
            pos = pos + 1
            
            newText = text[:pos] + self.__getEol() + self.__getIndent(text) + \
                self.__indentWord + text[pos:].lstrip("\n\r \t\\") + \
                self.__getEol()
            self.__source[line] = newText
        else:
            self.__stack.append((code, line, pos))
        return (True, self.trUtf8("Compound statement corrected."))
    
    def __fixE702(self, code, line, pos, apply=False):
        """
        Private method to fix semicolon-separated compound statements
        (E702, E703).
        
        @param code code of the issue (string)
        @param line line number of the issue (integer)
        @param pos position inside line (integer)
        @keyparam apply flag indicating, that the fix should be applied
            (boolean)
        @return flag indicating an applied fix (boolean) and a message for
            the fix (string)
        """
        if apply:
            line = line - 1
            text = self.__source[line]
            
            if text.rstrip().endswith("\\"):
                # normalize '1; \\\n2' into '1; 2'
                self.__source[line] = text.rstrip("\n\r \t\\")
                self.__source[line + 1] = self.__source[line + 1].lstrip()
            elif text.rstrip().endswith(";"):
                self.__source[line] = text.rstrip("\n\r \t;") + self.__getEol()
            else:
                first = text[:pos].rstrip("\n\r \t;") + self.__getEol()
                second = text[pos:].lstrip("\n\r \t;")
                self.__source[line] = first + self.__getIndent(text) + second
        else:
            self.__stack.append((code, line, pos))
        return (True, self.trUtf8("Compound statement corrected."))
    
    def __fixE711(self, code, line, pos):
        """
        Private method to fix comparison with None (E711, E712).
        
        @param code code of the issue (string)
        @param line line number of the issue (integer)
        @param pos position inside line (integer)
        @return flag indicating an applied fix (boolean) and a message for
            the fix (string)
        """
        line = line - 1
        text = self.__source[line]
        
        rightPos = pos + 2
        if rightPos >= len(text):
            return (False, "")
        
        left = text[:pos].rstrip()
        center = text[pos:rightPos]
        right = text[rightPos:].lstrip()
        
        if not right.startswith(("None", "True", "False")):
            return (False, "")
        
        if center.strip() == "==":
            center = "is"
        elif center.strip() == "!=":
            center = "is not"
        else:
            return (False, "")
        
        self.__source[line] = " ".join([left, center, right])
        return (True, self.trUtf8("Comparison to None/True/False corrected."))
    
    def __fixW291(self, code, line, pos):
        """
        Private method to fix trailing whitespace (W291, W293).
        
        @param code code of the issue (string)
        @param line line number of the issue (integer)
        @param pos position inside line (integer)
        @return flag indicating an applied fix (boolean) and a message for
            the fix (string)
        """
        self.__source[line - 1] = re.sub(r'[\t ]+(\r?)$', r"\1",
                                         self.__source[line - 1])
        return (True, self.trUtf8("Whitespace stripped from end of line."))
    
    def __fixW292(self, code, line, pos):
        """
        Private method to fix a missing newline at the end of file (W292).
        
        @param code code of the issue (string)
        @param line line number of the issue (integer)
        @param pos position inside line (integer)
        @return flag indicating an applied fix (boolean) and a message for
            the fix (string)
        """
        self.__source[line - 1] += self.__getEol()
        return (True, self.trUtf8("newline added to end of file."))
    
    def __fixW391(self, code, line, pos):
        """
        Private method to fix trailing blank lines (W391).
        
        @param code code of the issue (string)
        @param line line number of the issue (integer)
        @param pos position inside line (integer)
        @return flag indicating an applied fix (boolean) and a message for
            the fix (string)
        """
        index = line - 1
        while index:
            if self.__source[index].strip() == "":
                del self.__source[index]
                index -= 1
            else:
                break
        return (True, self.trUtf8(
            "Superfluous trailing blank lines removed from end of file."))
    
    def __fixW603(self, code, line, pos):
        """
        Private method to fix the not equal notation (W603).
        
        @param code code of the issue (string)
        @param line line number of the issue (integer)
        @param pos position inside line (integer)
        @return flag indicating an applied fix (boolean) and a message for
            the fix (string)
        """
        self.__source[line - 1] = self.__source[line - 1].replace("<>", "!=")
        return (True, self.trUtf8("'<>' replaced by '!='."))


class Pep8Reindenter(object):
    """
    Class to reindent badly-indented code to uniformly use four-space indentation.

    Released to the public domain, by Tim Peters, 03 October 2000.
    """
    def __init__(self, sourceLines):
        """
        Constructor
        
        @param sourceLines list of source lines including eol marker
            (list of string)
        """
        # Raw file lines.
        self.raw = sourceLines
        self.after = []

        # File lines, rstripped & tab-expanded.  Dummy at start is so
        # that we can use tokenize's 1-based line numbering easily.
        # Note that a line is all-blank iff it's "\n".
        self.lines = [line.rstrip().expandtabs() + "\n"
                      for line in self.raw]
        self.lines.insert(0, None)
        self.index = 1  # index into self.lines of next line

        # List of (lineno, indentlevel) pairs, one for each stmt and
        # comment line.  indentlevel is -1 for comment lines, as a
        # signal that tokenize doesn't know what to do about them;
        # indeed, they're our headache!
        self.stats = []
    
    def run(self):
        """
        Public method to run the re-indenter.
        """
        try:
            stats = self.__genStats(tokenize.generate_tokens(self.getline))
        except (SyntaxError, tokenize.TokenError):
            return False
        
        # Remove trailing empty lines.
        lines = self.lines
        while lines and lines[-1] == "\n":
            lines.pop()
        # Sentinel.
        stats.append((len(lines), 0))
        # Map count of leading spaces to # we want.
        have2want = {}
        # Program after transformation.
        after = self.after = []
        # Copy over initial empty lines -- there's nothing to do until
        # we see a line with *something* on it.
        i = stats[0][0]
        after.extend(lines[1:i])
        for i in range(len(stats)-1):
            thisstmt, thislevel = stats[i]
            nextstmt = stats[i+1][0]
            have = self.__getlspace(lines[thisstmt])
            want = thislevel * 4
            if want < 0:
                # A comment line.
                if have:
                    # An indented comment line.  If we saw the same
                    # indentation before, reuse what it most recently
                    # mapped to.
                    want = have2want.get(have, -1)
                    if want < 0:
                        # Then it probably belongs to the next real stmt.
                        for j in range(i + 1, len(stats) - 1):
                            jline, jlevel = stats[j]
                            if jlevel >= 0:
                                if have == self.__getlspace(lines[jline]):
                                    want = jlevel * 4
                                break
                    if want < 0:           # Maybe it's a hanging
                                           # comment like this one,
                        # in which case we should shift it like its base
                        # line got shifted.
                        for j in range(i - 1, -1, -1):
                            jline, jlevel = stats[j]
                            if jlevel >= 0:
                                want = have + self.__getlspace(after[jline-1]) - \
                                       self.__getlspace(lines[jline])
                                break
                    if want < 0:
                        # Still no luck -- leave it alone.
                        want = have
                else:
                    want = 0
            assert want >= 0
            have2want[have] = want
            diff = want - have
            if diff == 0 or have == 0:
                after.extend(lines[thisstmt:nextstmt])
            else:
                for line in lines[thisstmt:nextstmt]:
                    if diff > 0:
                        if line == "\n":
                            after.append(line)
                        else:
                            after.append(" " * diff + line)
                    else:
                        remove = min(self.__getlspace(line), -diff)
                        after.append(line[remove:])
        return self.raw != self.after
    
    def fixedLine(self, line):
        """
        Public method to get a fixed line.
        
        @param line number of the line to retrieve (integer)
        @return fixed line (string)
        """
        if line < len(self.after):
            return self.after[line]
    
    def getline(self):
        """
        Public method to get a line of text for tokenize.
        
        @return line of text (string)
        """
        if self.index >= len(self.lines):
            line = ""
        else:
            line = self.lines[self.index]
            self.index += 1
        return line

    def __genStats(self, tokens):
        """
        Private method to generate the re-indent statistics.
        
        @param tokens tokens generator (tokenize._tokenize)
        """
        find_stmt = True  # next token begins a fresh stmt?
        level = 0  # current indent level
        stats = []

        for t in tokens:
            token_type = t[0]
            sline = t[2][0]
            line = t[4]

            if token_type == tokenize.NEWLINE:
                # A program statement, or ENDMARKER, will eventually follow,
                # after some (possibly empty) run of tokens of the form
                #     (NL | COMMENT)* (INDENT | DEDENT+)?
                self.find_stmt = True

            elif token_type == tokenize.INDENT:
                find_stmt = True
                level += 1

            elif token_type == tokenize.DEDENT:
                find_stmt = True
                level -= 1

            elif token_type == tokenize.COMMENT:
                if find_stmt:
                    stats.append((sline, -1))
                    # but we're still looking for a new stmt, so leave
                    # find_stmt alone

            elif token_type == tokenize.NL:
                pass

            elif find_stmt:
                # This is the first "real token" following a NEWLINE, so it
                # must be the first token of the next program statement, or an
                # ENDMARKER.
                find_stmt = False
                if line:   # not endmarker
                    stats.append((sline, level))
        
        return stats
    
    def __getlspace(self, line):
        """
        Private method to count number of leading blanks.
        
        @param line line to check (string)
        @return number of leading blanks (integer)
        """
        i = 0
        n = len(line)
        while i < n and line[i] == " ":
            i += 1
        return i