eric7/Utilities/ClassBrowsers/pyclbr.py

branch
eric7
changeset 8312
800c432b34c8
parent 8217
385f60c94548
child 8495
e8278859d9fd
equal deleted inserted replaced
8311:4e8b98454baa 8312:800c432b34c8
1 # -*- coding: utf-8 -*-
2
3 # Copyright (c) 2005 - 2021 Detlev Offenbach <detlev@die-offenbachs.de>
4 #
5
6 """
7 Parse a Python file and retrieve classes, functions/methods and attributes.
8
9 Parse enough of a Python file to recognize class and method definitions and
10 to find out the superclasses of a class as well as its attributes.
11 """
12
13 import sys
14 import re
15 from functools import reduce
16
17 import Utilities
18 import Utilities.ClassBrowsers as ClassBrowsers
19 from . import ClbrBaseClasses
20
21 TABWIDTH = 4
22
23 SUPPORTED_TYPES = [ClassBrowsers.PY_SOURCE, ClassBrowsers.PTL_SOURCE]
24
25 _getnext = re.compile(
26 r"""
27 (?P<CodingLine>
28 ^ \# \s* [*_-]* \s* coding[:=] \s* (?P<Coding> [-\w_.]+ ) \s* [*_-]* $
29 )
30
31 | (?P<String>
32 \# .*? $ # ignore everything in comments
33 |
34 \""" [^"\\]* (?:
35 (?: \\. | "(?!"") )
36 [^"\\]*
37 )*
38 \"""
39
40 | ''' [^'\\]* (?:
41 (?: \\. | '(?!'') )
42 [^'\\]*
43 )*
44 '''
45
46 | " [^"\\\n]* (?: \\. [^"\\\n]*)* "
47
48 | ' [^'\\\n]* (?: \\. [^'\\\n]*)* '
49 )
50
51 | (?P<Publics>
52 ^
53 [ \t]* __all__ [ \t]* = [ \t]* \[
54 (?P<Identifiers> [^\]]*? )
55 \]
56 )
57
58 | (?P<MethodModifier>
59 ^
60 (?P<MethodModifierIndent> [ \t]* )
61 (?P<MethodModifierType> @classmethod | @staticmethod )
62 )
63
64 | (?P<Method>
65 ^
66 (?P<MethodIndent> [ \t]* )
67 (?: async [ \t]+ )? (?: cdef | cpdef | def) [ \t]+
68 (?P<MethodName> \w+ )
69 (?: [ \t]* \[ (?: plain | html ) \] )?
70 [ \t]* \(
71 (?P<MethodSignature> (?: [^)] | \)[ \t]*,? )*? )
72 \) [ \t]*
73 (?P<MethodReturnAnnotation> (?: -> [ \t]* [^:]+ )? )
74 [ \t]* :
75 )
76
77 | (?P<Class>
78 ^
79 (?P<ClassIndent> [ \t]* )
80 (?: cdef [ \t]+ )?
81 class [ \t]+
82 (?P<ClassName> \w+ )
83 [ \t]*
84 (?P<ClassSupers> \( [^)]* \) )?
85 [ \t]* :
86 )
87
88 | (?P<Attribute>
89 ^
90 (?P<AttributeIndent> [ \t]* )
91 self [ \t]* \. [ \t]*
92 (?P<AttributeName> \w+ )
93 [ \t]* =
94 )
95
96 | (?P<Variable>
97 ^
98 (?P<VariableIndent> [ \t]* )
99 (?P<VariableName> \w+ )
100 [ \t]* =
101 )
102
103 | (?P<Main>
104 ^
105 if \s+ __name__ \s* == \s* [^:]+ : $
106 )
107
108 | (?P<ConditionalDefine>
109 ^
110 (?P<ConditionalDefineIndent> [ \t]* )
111 (?: (?: if | elif ) [ \t]+ [^:]* | else [ \t]* ) :
112 (?= \s* (?: async [ \t]+ )? def)
113 )
114
115 | (?P<Import>
116 ^ [ \t]* (?: c? import | from [ \t]+ \. [ \t]+ c? import ) [ \t]+
117 (?P<ImportList> (?: [^#;\\\n]* (?: \\\n )* )* )
118 )
119
120 | (?P<ImportFrom>
121 ^ [ \t]* from [ \t]+
122 (?P<ImportFromPath>
123 \.* \w+
124 (?:
125 [ \t]* \. [ \t]* \w+
126 )*
127 )
128 [ \t]+
129 c? import [ \t]+
130 (?P<ImportFromList>
131 (?: \( \s* .*? \s* \) )
132 |
133 (?: [^#;\\\n]* (?: \\\n )* )* )
134 )""",
135 re.VERBOSE | re.DOTALL | re.MULTILINE).search
136
137 _commentsub = re.compile(r"""#[^\n]*\n|#[^\n]*$""").sub
138
139 _modules = {} # cache of modules we've seen
140
141
142 class VisibilityMixin(ClbrBaseClasses.ClbrVisibilityMixinBase):
143 """
144 Mixin class implementing the notion of visibility.
145 """
146 def __init__(self):
147 """
148 Constructor
149 """
150 if self.name.startswith('__'):
151 self.setPrivate()
152 elif self.name.startswith('_'):
153 self.setProtected()
154 else:
155 self.setPublic()
156
157
158 class Class(ClbrBaseClasses.Class, VisibilityMixin):
159 """
160 Class to represent a Python class.
161 """
162 def __init__(self, module, name, superClasses, file, lineno):
163 """
164 Constructor
165
166 @param module name of the module containing this class
167 @param name name of this class
168 @param superClasses list of class names this class is inherited from
169 @param file filename containing this class
170 @param lineno linenumber of the class definition
171 """
172 ClbrBaseClasses.Class.__init__(self, module, name, superClasses, file,
173 lineno)
174 VisibilityMixin.__init__(self)
175
176
177 class Function(ClbrBaseClasses.Function, VisibilityMixin):
178 """
179 Class to represent a Python function.
180 """
181 def __init__(self, module, name, file, lineno, signature='', separator=',',
182 modifierType=ClbrBaseClasses.Function.General, annotation=""):
183 """
184 Constructor
185
186 @param module name of the module containing this function
187 @param name name of this function
188 @param file filename containing this class
189 @param lineno linenumber of the class definition
190 @param signature parameterlist of the method
191 @param separator string separating the parameters
192 @param modifierType type of the function
193 @param annotation return annotation
194 """
195 ClbrBaseClasses.Function.__init__(self, module, name, file, lineno,
196 signature, separator, modifierType,
197 annotation)
198 VisibilityMixin.__init__(self)
199
200
201 class Attribute(ClbrBaseClasses.Attribute, VisibilityMixin):
202 """
203 Class to represent a class attribute.
204 """
205 def __init__(self, module, name, file, lineno):
206 """
207 Constructor
208
209 @param module name of the module containing this class
210 @param name name of this class
211 @param file filename containing this attribute
212 @param lineno linenumber of the class definition
213 """
214 ClbrBaseClasses.Attribute.__init__(self, module, name, file, lineno)
215 VisibilityMixin.__init__(self)
216
217
218 class Publics:
219 """
220 Class to represent the list of public identifiers.
221 """
222 def __init__(self, module, file, lineno, idents):
223 """
224 Constructor
225
226 @param module name of the module containing this function
227 @param file filename containing this class
228 @param lineno linenumber of the class definition
229 @param idents list of public identifiers
230 """
231 self.module = module
232 self.name = '__all__'
233 self.file = file
234 self.lineno = lineno
235 self.identifiers = [e.replace('"', '').replace("'", "").strip()
236 for e in idents.split(',')]
237
238
239 class Imports:
240 """
241 Class to represent the list of imported modules.
242 """
243 def __init__(self, module, file):
244 """
245 Constructor
246
247 @param module name of the module containing the import (string)
248 @param file file name containing the import (string)
249 """
250 self.module = module
251 self.name = 'import'
252 self.file = file
253 self.imports = {}
254
255 def addImport(self, moduleName, names, lineno):
256 """
257 Public method to add a list of imported names.
258
259 @param moduleName name of the imported module (string)
260 @param names list of names (list of strings)
261 @param lineno line number of the import
262 """
263 if moduleName not in self.imports:
264 module = ImportedModule(self.module, self.file, moduleName)
265 self.imports[moduleName] = module
266 else:
267 module = self.imports[moduleName]
268 module.addImport(lineno, names)
269
270 def getImport(self, moduleName):
271 """
272 Public method to get an imported module item.
273
274 @param moduleName name of the imported module (string)
275 @return imported module item (ImportedModule) or None
276 """
277 if moduleName in self.imports:
278 return self.imports[moduleName]
279 else:
280 return None
281
282 def getImports(self):
283 """
284 Public method to get all imported module names.
285
286 @return dictionary of imported module names with name as key and list
287 of line numbers of imports as value
288 """
289 return self.imports
290
291
292 class ImportedModule:
293 """
294 Class to represent an imported module.
295 """
296 def __init__(self, module, file, importedModule):
297 """
298 Constructor
299
300 @param module name of the module containing the import (string)
301 @param file file name containing the import (string)
302 @param importedModule name of the imported module (string)
303 """
304 self.module = module
305 self.name = 'import'
306 self.file = file
307 self.importedModuleName = importedModule
308 self.linenos = []
309 self.importedNames = {}
310 # dictionary of imported names with name as key and list of line
311 # numbers as value
312
313 def addImport(self, lineno, importedNames):
314 """
315 Public method to add a list of imported names.
316
317 @param lineno line number of the import
318 @param importedNames list of imported names (list of strings)
319 """
320 if lineno not in self.linenos:
321 self.linenos.append(lineno)
322
323 for name in importedNames:
324 if name not in self.importedNames:
325 self.importedNames[name] = [lineno]
326 else:
327 self.importedNames[name].append(lineno)
328
329
330 def readmodule_ex(module, path=None, inpackage=False, isPyFile=False):
331 """
332 Read a module file and return a dictionary of classes.
333
334 Search for MODULE in PATH and sys.path, read and parse the
335 module and return a dictionary with one entry for each class
336 found in the module.
337
338 @param module name of the module file
339 @type str
340 @param path path the module should be searched in
341 @type list of str
342 @param inpackage flag indicating a module inside a package is scanned
343 @type bool
344 @param isPyFile flag indicating a Python file
345 @type bool
346 @return the resulting dictionary
347 @rtype dict
348 """
349 global _modules
350
351 if module in _modules:
352 # we've seen this module before...
353 return _modules[module]
354 if module in sys.builtin_module_names:
355 # this is a built-in module
356 _modules[module] = {}
357 return {}
358
359 # search the path for the module
360 path = [] if path is None else path[:]
361 f = None
362 if inpackage:
363 try:
364 f, file, (suff, mode, type) = ClassBrowsers.find_module(
365 module, path)
366 except ImportError:
367 f = None
368 if f is None:
369 fullpath = path[:] + sys.path[:]
370 f, file, (suff, mode, type) = ClassBrowsers.find_module(
371 module, fullpath, isPyFile)
372 if f:
373 f.close()
374 if type not in SUPPORTED_TYPES:
375 # not Python source, can't do anything with this module
376 _modules[module] = {}
377 return {}
378
379 try:
380 src = Utilities.readEncodedFile(file)[0]
381 except (UnicodeError, OSError):
382 # can't do anything with this module
383 _modules[module] = {}
384 return {}
385
386 _modules[module] = scan(src, file, module)
387 return _modules[module]
388
389
390 def scan(src, file, module):
391 """
392 Public method to scan the given source text.
393
394 @param src source text to be scanned
395 @type str
396 @param file file name associated with the source text
397 @type str
398 @param module module name associated with the source text
399 @type str
400 @return dictionary containing the extracted data
401 @rtype dict
402 """
403 def calculateEndline(lineno, lines, indent):
404 """
405 Function to calculate the end line of a class or method/function.
406
407 @param lineno line number to start at (one based)
408 @type int
409 @param lines list of source lines
410 @type list of str
411 @param indent indent length the class/method/function definition
412 @type int
413 @return end line of the class/method/function (one based)
414 @rtype int
415 """
416 # start with zero based line after start line
417 while lineno < len(lines):
418 line = lines[lineno]
419 if line.strip():
420 # line contains some text
421 lineIndent = _indent(line.replace(line.lstrip(), ""))
422 if lineIndent <= indent:
423 return lineno
424 lineno += 1
425
426 # nothing found
427 return -1
428
429 # convert eol markers the Python style
430 src = src.replace("\r\n", "\n").replace("\r", "\n")
431 srcLines = src.splitlines()
432
433 dictionary = {}
434 dict_counts = {}
435
436 classstack = [] # stack of (class, indent) pairs
437 conditionalsstack = [] # stack of indents of conditional defines
438 deltastack = []
439 deltaindent = 0
440 deltaindentcalculated = False
441
442 lineno, last_lineno_pos = 1, 0
443 i = 0
444 modifierType = ClbrBaseClasses.Function.General
445 modifierIndent = -1
446 while True:
447 m = _getnext(src, i)
448 if not m:
449 break
450 start, i = m.span()
451
452 if m.start("MethodModifier") >= 0:
453 modifierIndent = _indent(m.group("MethodModifierIndent"))
454 modifierType = m.group("MethodModifierType")
455
456 elif m.start("Method") >= 0:
457 # found a method definition or function
458 thisindent = _indent(m.group("MethodIndent"))
459 meth_name = m.group("MethodName")
460 meth_sig = m.group("MethodSignature")
461 meth_sig = meth_sig.replace('\\\n', '')
462 meth_sig = _commentsub('', meth_sig)
463 meth_ret = m.group("MethodReturnAnnotation")
464 meth_ret = meth_ret.replace('\\\n', '')
465 meth_ret = _commentsub('', meth_ret)
466 lineno += src.count('\n', last_lineno_pos, start)
467 last_lineno_pos = start
468 if modifierType and modifierIndent == thisindent:
469 if modifierType == "@staticmethod":
470 modifier = ClbrBaseClasses.Function.Static
471 elif modifierType == "@classmethod":
472 modifier = ClbrBaseClasses.Function.Class
473 else:
474 modifier = ClbrBaseClasses.Function.General
475 else:
476 modifier = ClbrBaseClasses.Function.General
477 # modify indentation level for conditional defines
478 if conditionalsstack:
479 if thisindent > conditionalsstack[-1]:
480 if not deltaindentcalculated:
481 deltastack.append(thisindent - conditionalsstack[-1])
482 deltaindent = reduce(lambda x, y: x + y, deltastack)
483 deltaindentcalculated = True
484 thisindent -= deltaindent
485 else:
486 while (
487 conditionalsstack and
488 conditionalsstack[-1] >= thisindent
489 ):
490 del conditionalsstack[-1]
491 if deltastack:
492 del deltastack[-1]
493 deltaindentcalculated = False
494 # close all classes indented at least as much
495 while classstack and classstack[-1][1] >= thisindent:
496 del classstack[-1]
497 if classstack:
498 # it's a class method
499 cur_class = classstack[-1][0]
500 if cur_class:
501 # it's a method/nested def
502 f = Function(None, meth_name,
503 file, lineno, meth_sig, annotation=meth_ret,
504 modifierType=modifier)
505 cur_class._addmethod(meth_name, f)
506 else:
507 f = None
508 else:
509 # it's a function
510 f = Function(module, meth_name,
511 file, lineno, meth_sig, annotation=meth_ret,
512 modifierType=modifier)
513 if meth_name in dict_counts:
514 dict_counts[meth_name] += 1
515 meth_name = "{0}_{1:d}".format(
516 meth_name, dict_counts[meth_name])
517 else:
518 dict_counts[meth_name] = 0
519 dictionary[meth_name] = f
520 if f:
521 endlineno = calculateEndline(lineno, srcLines, thisindent)
522 f.setEndLine(endlineno)
523 classstack.append((f, thisindent)) # Marker for nested fns
524
525 # reset the modifier settings
526 modifierType = ClbrBaseClasses.Function.General
527 modifierIndent = -1
528
529 elif m.start("String") >= 0:
530 pass
531
532 elif m.start("Class") >= 0:
533 # we found a class definition
534 thisindent = _indent(m.group("ClassIndent"))
535 # close all classes indented at least as much
536 while classstack and classstack[-1][1] >= thisindent:
537 del classstack[-1]
538 lineno += src.count('\n', last_lineno_pos, start)
539 last_lineno_pos = start
540 class_name = m.group("ClassName")
541 inherit = m.group("ClassSupers")
542 if inherit:
543 # the class inherits from other classes
544 inherit = inherit[1:-1].strip()
545 inherit = _commentsub('', inherit)
546 names = []
547 for n in inherit.split(','):
548 n = n.strip()
549 if n in dictionary:
550 # we know this super class
551 n = dictionary[n]
552 else:
553 c = n.split('.')
554 if len(c) > 1:
555 # super class
556 # is of the
557 # form module.class:
558 # look in
559 # module for class
560 m = c[-2]
561 c = c[-1]
562 if m in _modules:
563 d = _modules[m]
564 n = d.get(c, n)
565 names.append(n)
566 inherit = names
567 # modify indentation level for conditional defines
568 if conditionalsstack:
569 if thisindent > conditionalsstack[-1]:
570 if not deltaindentcalculated:
571 deltastack.append(thisindent - conditionalsstack[-1])
572 deltaindent = reduce(lambda x, y: x + y, deltastack)
573 deltaindentcalculated = True
574 thisindent -= deltaindent
575 else:
576 while (
577 conditionalsstack and
578 conditionalsstack[-1] >= thisindent
579 ):
580 del conditionalsstack[-1]
581 if deltastack:
582 del deltastack[-1]
583 deltaindentcalculated = False
584 # remember this class
585 cur_class = Class(module, class_name, inherit,
586 file, lineno)
587 endlineno = calculateEndline(lineno, srcLines, thisindent)
588 cur_class.setEndLine(endlineno)
589 if not classstack:
590 if class_name in dict_counts:
591 dict_counts[class_name] += 1
592 class_name = "{0}_{1:d}".format(
593 class_name, dict_counts[class_name])
594 else:
595 dict_counts[class_name] = 0
596 dictionary[class_name] = cur_class
597 else:
598 classstack[-1][0]._addclass(class_name, cur_class)
599 classstack.append((cur_class, thisindent))
600
601 elif m.start("Attribute") >= 0:
602 lineno += src.count('\n', last_lineno_pos, start)
603 last_lineno_pos = start
604 index = -1
605 while index >= -len(classstack):
606 if (
607 classstack[index][0] is not None and
608 not isinstance(classstack[index][0], Function)
609 ):
610 attr = Attribute(
611 module, m.group("AttributeName"), file, lineno)
612 classstack[index][0]._addattribute(attr)
613 break
614 else:
615 index -= 1
616
617 elif m.start("Main") >= 0:
618 # 'main' part of the script, reset class stack
619 lineno += src.count('\n', last_lineno_pos, start)
620 last_lineno_pos = start
621 classstack = []
622
623 elif m.start("Variable") >= 0:
624 thisindent = _indent(m.group("VariableIndent"))
625 variable_name = m.group("VariableName")
626 lineno += src.count('\n', last_lineno_pos, start)
627 last_lineno_pos = start
628 if thisindent == 0 or not classstack:
629 # global variable, reset class stack first
630 classstack = []
631
632 if "@@Globals@@" not in dictionary:
633 dictionary["@@Globals@@"] = ClbrBaseClasses.ClbrBase(
634 module, "Globals", file, lineno)
635 dictionary["@@Globals@@"]._addglobal(
636 Attribute(module, variable_name, file, lineno))
637 else:
638 index = -1
639 while index >= -len(classstack):
640 if classstack[index][1] >= thisindent:
641 index -= 1
642 else:
643 if isinstance(classstack[index][0], Class):
644 classstack[index][0]._addglobal(
645 Attribute(module, variable_name, file, lineno))
646 break
647
648 elif m.start("Publics") >= 0:
649 idents = m.group("Identifiers")
650 lineno += src.count('\n', last_lineno_pos, start)
651 last_lineno_pos = start
652 pubs = Publics(module, file, lineno, idents)
653 dictionary['__all__'] = pubs
654
655 elif m.start("Import") >= 0:
656 #- import module
657 names = [n.strip() for n in
658 "".join(m.group("ImportList").splitlines())
659 .replace("\\", "").split(',')]
660 lineno += src.count('\n', last_lineno_pos, start)
661 last_lineno_pos = start
662 if "@@Import@@" not in dictionary:
663 dictionary["@@Import@@"] = Imports(module, file)
664 for name in names:
665 dictionary["@@Import@@"].addImport(name, [], lineno)
666
667 elif m.start("ImportFrom") >= 0:
668 #- from module import stuff
669 mod = m.group("ImportFromPath")
670 namesLines = (m.group("ImportFromList")
671 .replace("(", "").replace(")", "")
672 .replace("\\", "")
673 .strip().splitlines())
674 namesLines = [line.split("#")[0].strip()
675 for line in namesLines]
676 names = [n.strip() for n in
677 "".join(namesLines)
678 .split(',')]
679 lineno += src.count('\n', last_lineno_pos, start)
680 last_lineno_pos = start
681 if "@@Import@@" not in dictionary:
682 dictionary["@@Import@@"] = Imports(module, file)
683 dictionary["@@Import@@"].addImport(mod, names, lineno)
684
685 elif m.start("ConditionalDefine") >= 0:
686 # a conditional function/method definition
687 thisindent = _indent(m.group("ConditionalDefineIndent"))
688 while conditionalsstack and conditionalsstack[-1] >= thisindent:
689 del conditionalsstack[-1]
690 if deltastack:
691 del deltastack[-1]
692 conditionalsstack.append(thisindent)
693 deltaindentcalculated = False
694
695 elif m.start("CodingLine") >= 0:
696 # a coding statement
697 coding = m.group("Coding")
698 lineno += src.count('\n', last_lineno_pos, start)
699 last_lineno_pos = start
700 if "@@Coding@@" not in dictionary:
701 dictionary["@@Coding@@"] = ClbrBaseClasses.Coding(
702 module, file, lineno, coding)
703
704 if '__all__' in dictionary:
705 # set visibility of all top level elements
706 pubs = dictionary['__all__']
707 for key in dictionary:
708 if key == '__all__' or key.startswith("@@"):
709 continue
710 if key in pubs.identifiers:
711 dictionary[key].setPublic()
712 else:
713 dictionary[key].setPrivate()
714 del dictionary['__all__']
715
716 return dictionary
717
718
719 def _indent(ws):
720 """
721 Module function to return the indentation depth.
722
723 @param ws the whitespace to be checked (string)
724 @return length of the whitespace string (integer)
725 """
726 return len(ws.expandtabs(TABWIDTH))

eric ide

mercurial