71 self.lines = text.count(os.linesep) |
71 self.lines = text.count(os.linesep) |
72 |
72 |
73 source = io.BytesIO(text.encode("utf-8")) |
73 source = io.BytesIO(text.encode("utf-8")) |
74 try: |
74 try: |
75 gen = tokenize.tokenize(source.readline) |
75 gen = tokenize.tokenize(source.readline) |
76 for toktype, toktext, start, end, line in gen: |
76 for toktype, toktext, start, _end, line in gen: |
77 (srow, scol) = start |
77 (srow, scol) = start |
78 (erow, ecol) = end |
|
79 if toktype in [token.NEWLINE, tokenize.NL]: |
78 if toktype in [token.NEWLINE, tokenize.NL]: |
80 self.__addToken(toktype, os.linesep, srow, scol, line) |
79 self.__addToken(toktype, os.linesep, srow, scol, line) |
81 elif toktype in [token.INDENT, token.DEDENT]: |
80 elif toktype in [token.INDENT, token.DEDENT]: |
82 self.__addToken(toktype, "", srow, scol, line) |
81 self.__addToken(toktype, "", srow, scol, line) |
83 elif toktype == token.NAME and keyword.iskeyword(toktext): |
82 elif toktype == token.NAME and keyword.iskeyword(toktext): |