Mercurial Repositories > eric / file comparison
comparison: DebugClients/Python/coverage/phystokens.py
DebugClients/Python/coverage/phystokens.py
- changeset 6219
- d6c795b5ce33
- parent 5178
- 878ce843ca9f
equal
deleted
inserted
replaced
| 6218:bedab77d0fa3 | 6219:d6c795b5ce33 |
|---|---|
| 9 import sys | 9 import sys |
| 10 import token | 10 import token |
| 11 import tokenize | 11 import tokenize |
| 12 | 12 |
| 13 from coverage import env | 13 from coverage import env |
| 14 from coverage.backward import iternext | 14 from coverage.backward import iternext, unicode_class |
| 15 from coverage.misc import contract | 15 from coverage.misc import contract |
| 16 | 16 |
| 17 | 17 |
| 18 def phys_tokens(toks): | 18 def phys_tokens(toks): |
| 19 """Return all physical tokens, even line continuations. | 19 """Return all physical tokens, even line continuations. |
| 279 | 279 |
| 280 This function neuters the coding declaration, and compiles it. | 280 This function neuters the coding declaration, and compiles it. |
| 281 | 281 |
| 282 """ | 282 """ |
| 283 source = neuter_encoding_declaration(source) | 283 source = neuter_encoding_declaration(source) |
| 284 if env.PY2 and isinstance(filename, unicode): | 284 if env.PY2 and isinstance(filename, unicode_class): |
| 285 filename = filename.encode(sys.getfilesystemencoding(), "replace") | 285 filename = filename.encode(sys.getfilesystemencoding(), "replace") |
| 286 code = compile(source, filename, mode) | 286 code = compile(source, filename, mode) |
| 287 return code | 287 return code |
| 288 | 288 |
| 289 | 289 |
| 290 @contract(source='unicode', returns='unicode') | 290 @contract(source='unicode', returns='unicode') |
| 291 def neuter_encoding_declaration(source): | 291 def neuter_encoding_declaration(source): |
| 292 """Return `source`, with any encoding declaration neutered.""" | 292 """Return `source`, with any encoding declaration neutered.""" |
| 293 source = COOKIE_RE.sub("# (deleted declaration)", source, count=2) | 293 if COOKIE_RE.search(source): |
| 294 source_lines = source.splitlines(True) | |
| 295 for lineno in range(min(2, len(source_lines))): | |
| 296 source_lines[lineno] = COOKIE_RE.sub("# (deleted declaration)", source_lines[lineno]) | |
| 297 source = "".join(source_lines) | |
| 294 return source | 298 return source |
