66 |
65 |
67 self.lines = text.count(os.linesep) |
66 self.lines = text.count(os.linesep) |
68 |
67 |
69 source = io.BytesIO(text.encode("utf-8")) |
68 source = io.BytesIO(text.encode("utf-8")) |
70 try: |
69 try: |
71 if sys.version_info[0] == 2: |
70 gen = tokenize.tokenize(source.readline) |
72 gen = tokenize.generate_tokens(source.readline) |
|
73 else: |
|
74 gen = tokenize.tokenize(source.readline) |
|
75 for toktype, toktext, start, end, line in gen: |
71 for toktype, toktext, start, end, line in gen: |
76 (srow, scol) = start |
72 (srow, scol) = start |
77 (erow, ecol) = end |
73 (erow, ecol) = end |
78 if toktype in [token.NEWLINE, tokenize.NL]: |
74 if toktype in [token.NEWLINE, tokenize.NL]: |
79 self.__addToken(toktype, os.linesep, srow, scol, line) |
75 self.__addToken(toktype, os.linesep, srow, scol, line) |