Utilities/__init__.py

branch
Py2 comp.
changeset 2571
e6bb19eb87ea
parent 2525
8b507a9a2d40
child 2572
dc6d76ab5d10
equal deleted inserted replaced
2552:1e893ea4e366 2571:e6bb19eb87ea
7 Package implementing various functions/classes needed everywhere within eric5. 7 Package implementing various functions/classes needed everywhere within eric5.
8 """ 8 """
9 9
10 from __future__ import unicode_literals # __IGNORE_WARNING__ 10 from __future__ import unicode_literals # __IGNORE_WARNING__
11 try: 11 try:
12 str = unicode 12 str = unicode # __IGNORE_WARNING__
13 except (NameError): 13 except (NameError):
14 pass 14 pass
15 15
16 import os 16 import os
17 import sys 17 import sys
18 import codecs
18 import re 19 import re
19 import fnmatch 20 import fnmatch
20 import glob 21 import glob
21 import getpass 22 import getpass
22 23
55 from E5Gui.E5Application import e5App 56 from E5Gui.E5Application import e5App
56 57
57 from UI.Info import Program, Version 58 from UI.Info import Program, Version
58 59
59 import Preferences 60 import Preferences
61 from .SyntaxCheck import *
60 62
61 from eric5config import getConfig 63 from eric5config import getConfig
62 64
63 configDir = None 65 configDir = None
64 66
65 codingBytes_regexps = [
66 (2, re.compile(br'''coding[:=]\s*([-\w_.]+)''')),
67 (1, re.compile(br'''<\?xml.*\bencoding\s*=\s*['"]([-\w_.]+)['"]\?>''')),
68 ]
69 coding_regexps = [ 67 coding_regexps = [
70 (2, re.compile(r'''coding[:=]\s*([-\w_.]+)''')), 68 (2, re.compile(r'''coding[:=]\s*([-\w_.]+)''')),
71 (1, re.compile(r'''<\?xml.*\bencoding\s*=\s*['"]([-\w_.]+)['"]\?>''')), 69 (1, re.compile(r'''<\?xml.*\bencoding\s*=\s*['"]([-\w_.]+)['"]\?>''')),
72 ] 70 ]
73 71
115 @return string representing the error message 113 @return string representing the error message
116 """ 114 """
117 return str(self.errorMessage) 115 return str(self.errorMessage)
118 116
119 117
120 def get_codingBytes(text):
121 """
122 Function to get the coding of a bytes text.
123
124 @param text bytes text to inspect (bytes)
125 @return coding string
126 """
127 lines = text.splitlines()
128 for coding in codingBytes_regexps:
129 coding_re = coding[1]
130 head = lines[:coding[0]]
131 for l in head:
132 m = coding_re.search(l)
133 if m:
134 return str(m.group(1), "ascii").lower()
135 return None
136
137
138 def get_coding(text): 118 def get_coding(text):
139 """ 119 """
140 Function to get the coding of a text. 120 Function to get the coding of a text.
141 121
142 @param text text to inspect (string) 122 @param text text to inspect (string)
151 if m: 131 if m:
152 return m.group(1).lower() 132 return m.group(1).lower()
153 return None 133 return None
154 134
155 135
156 def readEncodedFile(filename):
157 """
158 Function to read a file and decode it's contents into proper text.
159
160 @param filename name of the file to read (string)
161 @return tuple of decoded text and encoding (string, string)
162 """
163 f = open(filename, "rb")
164 text = f.read()
165 f.close()
166 return decode(text)
167
168
169 def readEncodedFileWithHash(filename): 136 def readEncodedFileWithHash(filename):
170 """ 137 """
171 Function to read a file, calculate a hash value and decode it's contents 138 Function to read a file, calculate a hash value and decode it's contents
172 into proper text. 139 into proper text.
173 140
178 text = f.read() 145 text = f.read()
179 f.close() 146 f.close()
180 hash = str(QCryptographicHash.hash(QByteArray(text), QCryptographicHash.Md5).toHex(), 147 hash = str(QCryptographicHash.hash(QByteArray(text), QCryptographicHash.Md5).toHex(),
181 encoding="ASCII") 148 encoding="ASCII")
182 return decode(text) + (hash, ) 149 return decode(text) + (hash, )
183
184
185 def decode(text):
186 """
187 Function to decode some byte text into a string.
188
189 @param text byte text to decode (bytes)
190 @return tuple of decoded text and encoding (string, string)
191 """
192 try:
193 if text.startswith(BOM_UTF8):
194 # UTF-8 with BOM
195 return str(text[len(BOM_UTF8):], 'utf-8'), 'utf-8-bom'
196 elif text.startswith(BOM_UTF16):
197 # UTF-16 with BOM
198 return str(text[len(BOM_UTF16):], 'utf-16'), 'utf-16'
199 elif text.startswith(BOM_UTF32):
200 # UTF-32 with BOM
201 return str(text[len(BOM_UTF32):], 'utf-32'), 'utf-32'
202 coding = get_codingBytes(text)
203 if coding:
204 return str(text, coding), coding
205 except (UnicodeError, LookupError):
206 pass
207
208 # Assume UTF-8
209 try:
210 return str(text, 'utf-8'), 'utf-8-guessed'
211 except (UnicodeError, LookupError):
212 pass
213
214 guess = None
215 if Preferences.getEditor("AdvancedEncodingDetection"):
216 # Try the universal character encoding detector
217 try:
218 import ThirdParty.CharDet.chardet
219 guess = ThirdParty.CharDet.chardet.detect(text)
220 if guess and guess['confidence'] > 0.95 and guess['encoding'] is not None:
221 codec = guess['encoding'].lower()
222 return str(text, codec), '{0}-guessed'.format(codec)
223 except (UnicodeError, LookupError):
224 pass
225 except ImportError:
226 pass
227
228 # Try default encoding
229 try:
230 codec = Preferences.getEditor("DefaultEncoding")
231 return str(text, codec), '{0}-default'.format(codec)
232 except (UnicodeError, LookupError):
233 pass
234
235 if Preferences.getEditor("AdvancedEncodingDetection"):
236 # Use the guessed one even if confifence level is low
237 if guess and guess['encoding'] is not None:
238 try:
239 codec = guess['encoding'].lower()
240 return str(text, codec), '{0}-guessed'.format(codec)
241 except (UnicodeError, LookupError):
242 pass
243
244 # Assume UTF-8 loosing information
245 return str(text, "utf-8", "ignore"), 'utf-8-ignore'
246 150
247 151
248 def writeEncodedFile(filename, text, orig_coding): 152 def writeEncodedFile(filename, text, orig_coding):
249 """ 153 """
250 Function to write a file with properly encoded text. 154 Function to write a file with properly encoded text.
389 @return string read from the stream (string) 293 @return string read from the stream (string)
390 """ 294 """
391 data = stream.readString() 295 data = stream.readString()
392 if data is None: 296 if data is None:
393 data = b"" 297 data = b""
394 return data.decode() 298 return data.decode('utf-8')
395 299
396 300
397 _escape = re.compile("[&<>\"'\u0080-\uffff]") 301 _escape = re.compile("[&<>\"'\u0080-\uffff]")
398 302
399 _escape_map = { 303 _escape_map = {
594 return {} 498 return {}
595 499
596 return extractFlags(source) 500 return extractFlags(source)
597 501
598 502
599 def extractLineFlags(line, startComment="#", endComment=""):
600 """
601 Function to extract flags starting and ending with '__' from a line comment.
602
603 @param line line to extract flags from (string)
604 @keyparam startComment string identifying the start of the comment (string)
605 @keyparam endComment string identifying the end of a comment (string)
606 @return list containing the extracted flags (list of strings)
607 """
608 flags = []
609
610 pos = line.rfind(startComment)
611 if pos >= 0:
612 comment = line[pos + len(startComment):].strip()
613 if endComment:
614 comment = comment.replace("endComment", "")
615 flags = [f.strip() for f in comment.split()
616 if (f.startswith("__") and f.endswith("__"))]
617 return flags
618
619
620 def toNativeSeparators(path): 503 def toNativeSeparators(path):
621 """ 504 """
622 Function returning a path, that is using native separator characters. 505 Function returning a path, that is using native separator characters.
623 506
624 @param path path to be converted (string) 507 @param path path to be converted (string)
799 682
800 if normcaseabspath(os.path.realpath(f1)) == normcaseabspath(os.path.realpath(f2)): 683 if normcaseabspath(os.path.realpath(f1)) == normcaseabspath(os.path.realpath(f2)):
801 return True 684 return True
802 685
803 return False 686 return False
804 687
688
689 def samefilepath(f1, f2):
690 """
691 Function to compare two paths. Strips the filename.
692
693 @param f1 first filepath for the compare (string)
694 @param f2 second filepath for the compare (string)
695 @return flag indicating whether the two paths represent the
696 same path on disk.
697 """
698 if f1 is None or f2 is None:
699 return False
700
701 if (normcaseabspath(os.path.dirname(os.path.realpath(f1))) ==
702 normcaseabspath(os.path.dirname(os.path.realpath(f2)))):
703 return True
704
705 return False
706
805 try: 707 try:
806 EXTSEP = os.extsep 708 EXTSEP = os.extsep
807 except AttributeError: 709 except AttributeError:
808 EXTSEP = "." 710 EXTSEP = "."
809 711
1210 @return An integer representing major and minor version number (integer) 1112 @return An integer representing major and minor version number (integer)
1211 """ 1113 """
1212 return sys.hexversion >> 16 1114 return sys.hexversion >> 16
1213 1115
1214 1116
1215 def compile(file, codestring=""): 1117 def compile(file, codestring="", isPy2=False):
1216 """ 1118 """
1217 Function to compile one Python source file to Python bytecode. 1119 Function to compile one Python source file to Python bytecode.
1218 1120
1219 @param file source filename (string) 1121 @param file source filename (string)
1220 @param codestring string containing the code to compile (string) 1122 @param codestring string containing the code to compile (string)
1123 @param isPy2 shows which interperter to use (boolean)
1221 @return A tuple indicating status (True = an error was found), the 1124 @return A tuple indicating status (True = an error was found), the
1222 file name, the line number, the index number, the code string 1125 file name, the line number, the index number, the code string
1223 and the error message (boolean, string, string, string, string, 1126 and the error message (boolean, string, string, string, string,
1224 string). The values are only valid, if the status is True. 1127 string). The values are only valid, if the status is True.
1225 """ 1128 """
1226 import builtins 1129 from PyQt4.QtCore import QCoreApplication
1227 if not codestring: 1130
1228 try: 1131 interpreter_name = 'Python' if isPy2 else 'Python3'
1229 codestring = readEncodedFile(file)[0] 1132 interpreter = Preferences.getDebugger(interpreter_name+"Interpreter")
1230 except (UnicodeDecodeError, IOError): 1133 checkFlakes = Preferences.getFlakes("IncludeInSyntaxCheck")
1231 return (False, None, None, None, None, None) 1134 ignoreStarImportWarnings = Preferences.getFlakes("IgnoreStarImportWarnings")
1232 1135 if samefilepath(interpreter, sys.executable):
1233 codestring = codestring.replace("\r\n", "\n") 1136 ret = compile_and_check(file, codestring, checkFlakes, ignoreStarImportWarnings)
1234 codestring = codestring.replace("\r", "\n") 1137 else:
1235 1138 #TODO: create temporary file if only a codestring is given
1236 if codestring and codestring[-1] != '\n': 1139 ret = compile_extern(file, isPy2, checkFlakes, ignoreStarImportWarnings)
1237 codestring = codestring + '\n' 1140
1238 1141 # Translate messages
1239 try: 1142 for warning in ret[6]:
1240 if file.endswith('.ptl'): 1143 msg_args = warning.pop()
1241 try: 1144 translated = QCoreApplication.translate('py3Flakes', warning[-1]).format(*msg_args)
1242 import quixote.ptl_compile 1145 # Avoid leading "u" at Python2 unicode strings
1243 except ImportError: 1146 if translated.startswith("u'"):
1244 return (False, None, None, None, None, None) 1147 translated = translated[1:]
1245 template = quixote.ptl_compile.Template(codestring, file) 1148 warning[3] = translated.replace(" u'", " '")
1246 template.compile() 1149
1247 else: 1150 return ret
1248 builtins.compile(codestring, file, 'exec') 1151
1249 except SyntaxError as detail: 1152
1250 import traceback 1153 def compile_extern(file, isPy2, checkFlakes=True, ignoreStarImportWarnings=False):
1251 import re 1154 """
1252 index = "0" 1155 Function to compile one Python source file to Python bytecode.
1253 code = ""
1254 error = ""
1255 lines = traceback.format_exception_only(SyntaxError, detail)
1256 match = re.match('\s*File "(.+)", line (\d+)',
1257 lines[0].replace('<string>', '{0}'.format(file)))
1258 if match is not None:
1259 fn, line = match.group(1, 2)
1260 if lines[1].startswith('SyntaxError:'):
1261 error = re.match('SyntaxError: (.+)', lines[1]).group(1)
1262 else:
1263 code = re.match('(.+)', lines[1]).group(1)
1264 for seLine in lines[2:]:
1265 if seLine.startswith('SyntaxError:'):
1266 error = re.match('SyntaxError: (.+)', seLine).group(1)
1267 elif seLine.rstrip().endswith('^'):
1268 index = len(seLine.rstrip()) - 4
1269 else:
1270 fn = detail.filename
1271 line = detail.lineno and detail.lineno or 1
1272 error = detail.msg
1273 return (True, fn, line, index, code, error)
1274 except ValueError as detail:
1275 index = "0"
1276 code = ""
1277 try:
1278 fn = detail.filename
1279 line = detail.lineno
1280 error = detail.msg
1281 except AttributeError:
1282 fn = file
1283 line = "1"
1284 error = str(detail)
1285 return (True, fn, line, index, code, error)
1286 except Exception as detail:
1287 try:
1288 fn = detail.filename
1289 line = detail.lineno
1290 index = "0"
1291 code = ""
1292 error = detail.msg
1293 return (True, fn, line, index, code, error)
1294 except: # this catchall is intentional
1295 pass
1296
1297 return (False, None, None, None, None, None)
1298
1299
1300 def py2compile(file, checkFlakes=False):
1301 """
1302 Function to compile one Python 2 source file to Python 2 bytecode.
1303 1156
1304 @param file source filename (string) 1157 @param file source filename (string)
1305 @keyparam checkFlakes flag indicating to do a pyflakes check (boolean) 1158 @keyparam checkFlakes flag indicating to do a pyflakes check (boolean)
1306 @return A tuple indicating status (True = an error was found), the 1159 @return A tuple indicating status (True = an error was found), the
1307 file name, the line number, the index number, the code string, 1160 file name, the line number, the index number, the code string,
1309 file name, line number and message (boolean, string, string, string, 1162 file name, line number and message (boolean, string, string, string,
1310 string, string, list of (string, string, string)). The syntax error 1163 string, string, list of (string, string, string)). The syntax error
1311 values are only valid, if the status is True. The pyflakes list will 1164 values are only valid, if the status is True. The pyflakes list will
1312 be empty, if a syntax error was detected by the syntax checker. 1165 be empty, if a syntax error was detected by the syntax checker.
1313 """ 1166 """
1314 interpreter = Preferences.getDebugger("PythonInterpreter") 1167 interpreter_name = 'Python' if isPy2 else 'Python3'
1168 interpreter = Preferences.getDebugger(interpreter_name+"Interpreter")
1315 if interpreter == "" or not isinpath(interpreter): 1169 if interpreter == "" or not isinpath(interpreter):
1316 return (False, "", "", "", "", "", [( 1170 return (False, "", -1, -1, "", "", [(
1317 file, "1", 1171 "", "", 1, QCoreApplication.translate("Utilities",
1318 QCoreApplication.translate("Utilities", 1172 "{0} interpreter not configured.").format(interpreter_name)
1319 "Python2 interpreter not configured.")
1320 )]) 1173 )])
1321
1322 syntaxChecker = os.path.join(getConfig('ericDir'), 1174 syntaxChecker = os.path.join(getConfig('ericDir'),
1323 "UtilitiesPython2", "Py2SyntaxChecker.py") 1175 "Utilities", "SyntaxCheck.py")
1324 args = [syntaxChecker] 1176 args = [syntaxChecker]
1325 if checkFlakes: 1177 if checkFlakes:
1326 if Preferences.getFlakes("IgnoreStarImportWarnings"): 1178 if ignoreStarImportWarnings:
1327 args.append("-fi") 1179 args.append("-fi")
1328 else: 1180 else:
1329 args.append("-fs") 1181 args.append("-fs")
1330 args.append(file) 1182 args.append(file)
1331 proc = QProcess() 1183 proc = QProcess()
1332 proc.setProcessChannelMode(QProcess.MergedChannels) 1184 proc.setProcessChannelMode(QProcess.MergedChannels)
1333 proc.start(interpreter, args) 1185 proc.start(interpreter, args)
1334 finished = proc.waitForFinished(30000) 1186 finished = proc.waitForFinished(30000)
1335 if finished: 1187 if finished:
1336 output = \ 1188 output = codecs.decode(proc.readAllStandardOutput(),
1337 str(proc.readAllStandardOutput(), 1189 sys.getfilesystemencoding(), 'strict').splitlines()
1338 Preferences.getSystem("IOEncoding"),
1339 'replace').splitlines()
1340 1190
1341 if output: 1191 if output:
1342 syntaxerror = output[0] == "ERROR" 1192 syntaxerror = output[0] == "ERROR"
1343 if syntaxerror: 1193 if syntaxerror:
1344 fn = output[1] 1194 fn = output[1]
1345 line = output[2] 1195 line = int(output[2])
1346 index = output[3] 1196 index = int(output[3])
1347 code = output[4] 1197 code = output[4]
1348 error = output[5] 1198 error = output[5]
1349 return (True, fn, line, index, code, error, []) 1199 return (True, fn, line, index, code, error, [])
1350 else: 1200 else:
1351 index = 6 1201 index = 6
1352 warnings = [] 1202 warnings = []
1353 while len(output) - index > 3: 1203 while len(output) - index > 3:
1354 if output[index] == "FLAKES_ERROR": 1204 if output[index] == "FLAKES_ERROR":
1355 return (True, output[index + 1], output[index + 2], "", 1205 return (True, output[index + 1], int(output[index + 2]), -1,
1356 output[index + 3], []) 1206 '', output[index + 3], [])
1357 else: 1207 else:
1358 warnings.append((output[index + 1], output[index + 2], 1208 msg_args = output[index + 4].split('#')
1359 output[index + 3])) 1209 warnings.append([output[index], output[index + 1],
1360 index += 4 1210 int(output[index + 2]), output[index + 3], msg_args])
1211 index += 5
1361 1212
1362 return (False, None, None, None, None, None, warnings) 1213 return (False, None, None, None, None, None, warnings)
1363 else: 1214 else:
1364 return (False, "", "", "", "", "", []) 1215 return (False, "", -1, -1, "", "", [])
1365 1216
1366 return (True, file, "1", "0", "", 1217 return (True, file, 1, 0, "", QCoreApplication.translate("Utilities",
1367 QCoreApplication.translate("Utilities", 1218 "{0} interpreter did not finish within 30s.").format(
1368 "Python2 interpreter did not finish within 30s."), 1219 interpreter_name), [])
1369 [])
1370 1220
1371 1221
1372 ################################################################################ 1222 ################################################################################
1373 # functions for environment handling 1223 # functions for environment handling
1374 ################################################################################ 1224 ################################################################################

eric ide

mercurial