Sun, 16 Oct 2016 12:18:42 +0200
Merged with debugger stuff from Tobias.
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/DebugClients/Python/AsyncFile.py Sun Oct 16 12:18:42 2016 +0200 @@ -0,0 +1,344 @@ +# -*- coding: utf-8 -*- + +# Copyright (c) 2002 - 2016 Detlev Offenbach <detlev@die-offenbachs.de> +# + +""" +Module implementing an asynchronous file like socket interface for the +debugger. +""" + +import socket + +from DebugUtilities import prepareJsonCommand + +try: + unicode +except NameError: + unicode = str + raw_input = input + + +def AsyncPendingWrite(file): + """ + Module function to check for data to be written. + + @param file The file object to be checked (file) + @return Flag indicating if there is data wating (int) + """ + try: + pending = file.pendingWrite() + except Exception: + pending = 0 + + return pending + + +class AsyncFile(object): + """ + Class wrapping a socket object with a file interface. + """ + maxtries = 10 + maxbuffersize = 1024 * 1024 * 4 + + def __init__(self, sock, mode, name): + """ + Constructor + + @param sock the socket object being wrapped + @param mode mode of this file (string) + @param name name of this file (string) + """ + # Initialise the attributes. + self.closed = False + self.sock = sock + self.mode = mode + self.name = name + self.nWriteErrors = 0 + self.encoding = "utf-8" + + self.wpending = unicode('') + + def __checkMode(self, mode): + """ + Private method to check the mode. + + This method checks, if an operation is permitted according to + the mode of the file. If it is not, an IOError is raised. + + @param mode the mode to be checked (string) + @exception IOError raised to indicate a bad file descriptor + """ + if mode != self.mode: + raise IOError((9, '[Errno 9] Bad file descriptor')) + + def __nWrite(self, n): + """ + Private method to write a specific number of pending bytes. + + @param n the number of bytes to be written (int) + """ + if n: + try: + buf = self.wpending[:n] + try: + buf = buf.encode('utf-8', 'backslashreplace') + except (UnicodeEncodeError, UnicodeDecodeError): + pass + self.sock.sendall(buf) + self.wpending = self.wpending[n:] + self.nWriteErrors = 0 + except socket.error: + self.nWriteErrors += 1 + if self.nWriteErrors > self.maxtries: + self.wpending = unicode('') # delete all output + + def pendingWrite(self): + """ + Public method that returns the number of bytes waiting to be written. + + @return the number of bytes to be written (int) + """ + return self.wpending.rfind('\n') + 1 + + def close(self, closeit=False): + """ + Public method to close the file. + + @param closeit flag to indicate a close ordered by the debugger code + (boolean) + """ + if closeit and not self.closed: + self.flush() + self.sock.close() + self.closed = True + + def flush(self): + """ + Public method to write all pending bytes. + """ + self.__nWrite(len(self.wpending)) + + def isatty(self): + """ + Public method to indicate whether a tty interface is supported. + + @return always false + """ + return False + + def fileno(self): + """ + Public method returning the file number. + + @return file number (int) + """ + try: + return self.sock.fileno() + except socket.error: + return -1 + + def readable(self): + """ + Public method to check, if the stream is readable. + + @return flag indicating a readable stream (boolean) + """ + return self.mode == "r" + + def read_p(self, size=-1): + """ + Public method to read bytes from this file. + + @param size maximum number of bytes to be read (int) + @return the bytes read (any) + """ + self.__checkMode('r') + + if size < 0: + size = 20000 + + return self.sock.recv(size).decode('utf8', 'backslashreplace') + + def read(self, size=-1): + """ + Public method to read bytes from this file. + + @param size maximum number of bytes to be read (int) + @return the bytes read (any) + """ + self.__checkMode('r') + + buf = raw_input() + if size >= 0: + buf = buf[:size] + return buf + + def readline_p(self, size=-1): + """ + Public method to read a line from this file. + + <b>Note</b>: This method will not block and may return + only a part of a line if that is all that is available. + + @param size maximum number of bytes to be read (int) + @return one line of text up to size bytes (string) + """ + self.__checkMode('r') + + if size < 0: + size = 20000 + + # The integration of the debugger client event loop and the connection + # to the debugger relies on the two lines of the debugger command being + # delivered as two separate events. Therefore we make sure we only + # read a line at a time. + line = self.sock.recv(size, socket.MSG_PEEK) + + eol = line.find(b'\n') + + if eol >= 0: + size = eol + 1 + else: + size = len(line) + + # Now we know how big the line is, read it for real. + return self.sock.recv(size).decode('utf8', 'backslashreplace') + + def readlines(self, sizehint=-1): + """ + Public method to read all lines from this file. + + @param sizehint hint of the numbers of bytes to be read (int) + @return list of lines read (list of strings) + """ + self.__checkMode('r') + + lines = [] + room = sizehint + + line = self.readline_p(room) + linelen = len(line) + + while linelen > 0: + lines.append(line) + + if sizehint >= 0: + room = room - linelen + + if room <= 0: + break + + line = self.readline_p(room) + linelen = len(line) + + return lines + + def readline(self, sizehint=-1): + """ + Public method to read one line from this file. + + @param sizehint hint of the numbers of bytes to be read (int) + @return one line read (string) + """ + self.__checkMode('r') + + line = raw_input() + '\n' + if sizehint >= 0: + line = line[:sizehint] + return line + + def seekable(self): + """ + Public method to check, if the stream is seekable. + + @return flag indicating a seekable stream (boolean) + """ + return False + + def seek(self, offset, whence=0): + """ + Public method to move the filepointer. + + @param offset offset to move the filepointer to (integer) + @param whence position the offset relates to + @exception IOError This method is not supported and always raises an + IOError. + """ + raise IOError((29, '[Errno 29] Illegal seek')) + + def tell(self): + """ + Public method to get the filepointer position. + + @exception IOError This method is not supported and always raises an + IOError. + """ + raise IOError((29, '[Errno 29] Illegal seek')) + + def truncate(self, size=-1): + """ + Public method to truncate the file. + + @param size size to truncate to (integer) + @exception IOError This method is not supported and always raises an + IOError. + """ + raise IOError((29, '[Errno 29] Illegal seek')) + + def writable(self): + """ + Public method to check, if a stream is writable. + + @return flag indicating a writable stream (boolean) + """ + return self.mode == "w" + + def write(self, s): + """ + Public method to write a string to the file. + + @param s text to be written (string) + """ + self.__checkMode('w') + + cmd = prepareJsonCommand("ClientOutput", { + "text": s, + }) + self.write_p(cmd) + + def write_p(self, s): + """ + Public method to write a string to the file. + + @param s text to be written (string) + @exception socket.error raised to indicate too many send attempts + """ + self.__checkMode('w') + tries = 0 + if not self.wpending: + self.wpending = s + elif len(self.wpending) + len(s) > self.maxbuffersize: + # flush wpending if it is too big + while self.wpending: + # if we have a persistent error in sending the data, an + # exception will be raised in __nWrite + self.flush() + tries += 1 + if tries > self.maxtries: + raise socket.error("Too many attempts to send data") + self.wpending = s + else: + self.wpending += s + self.__nWrite(self.pendingWrite()) + + def writelines(self, lines): + """ + Public method to write a list of strings to the file. + + @param lines list of texts to be written (list of string) + """ + self.write("".join(lines)) + +# +# eflag: noqa = M702
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/DebugClients/Python/BreakpointWatch.py Sun Oct 16 12:18:42 2016 +0200 @@ -0,0 +1,332 @@ +# -*- coding: utf-8 -*- + +# Copyright (c) 2016 Detlev Offenbach <detlev@die-offenbachs.de> +# + +""" +Module implementing the breakpoint and watch class. +""" + +import os + + +class Breakpoint: + """ + Breakpoint class. + + Implements temporary breakpoints, ignore counts, disabling and + (re)-enabling, and conditionals. + + Breakpoints are indexed by the file,line tuple using breaks. It + points to a single Breakpoint instance. This is rather different to + the original bdb, since there may be more than one breakpoint per line. + + To test for a specific line in a file there is another dict breakInFile, + which is indexed only by filename and holds all line numbers where + breakpoints are. + """ + breaks = {} # indexed by (filename, lineno) tuple: Breakpoint + breakInFile = {} # indexed by filename: [lineno] + breakInFrameCache = {} + + def __init__(self, filename, lineno, temporary=False, cond=None): + """ + Constructor + + @param filename file name where a breakpoint is set + @type str + @param lineno line number of the breakpoint + @type int + @keyparam temporary flag to indicate a temporary breakpoint + @type bool + @keyparam cond Python expression which dynamically enables this bp + @type str + """ + filename = os.path.abspath(filename) + self.file = filename + self.line = lineno + self.temporary = temporary + self.cond = cond + self.enabled = True + self.ignore = 0 + self.hits = 0 + self.breaks[filename, lineno] = self + lines = self.breakInFile.setdefault(filename, []) + if lineno not in lines: + lines.append(lineno) + self.breakInFrameCache.clear() + + def deleteMe(self): + """ + Public method to clear this breakpoint. + """ + try: + del self.breaks[(self.file, self.line)] + self.breakInFile[self.file].remove(self.line) + if not self.breakInFile[self.file]: + del self.breakInFile[self.file] + except KeyError: + pass + + def enable(self): + """ + Public method to enable this breakpoint. + """ + self.enabled = True + + def disable(self): + """ + Public method to disable this breakpoint. + """ + self.enabled = False + + @staticmethod + def clear_break(filename, lineno): + """ + Public method reimplemented from bdb.py to clear a breakpoint. + + @param filename file name of the bp to retrieve + @type str + @param lineno line number of the bp to retrieve + @type int + """ + bp = Breakpoint.breaks.get((filename, lineno)) + if bp: + bp.deleteMe() + Breakpoint.breakInFrameCache.clear() + + @staticmethod + def clear_all_breaks(): + """ + Public method to clear all breakpoints. + """ + for bp in Breakpoint.breaks.copy(): + bp.deleteMe() + Breakpoint.breakInFrameCache.clear() + + @staticmethod + def get_break(filename, lineno): + """ + Public method to get the breakpoint of a particular line. + + Because eric6 supports only one breakpoint per line, this + method will return only one breakpoint. + + @param filename file name of the bp to retrieve + @type str + @param lineno line number of the bp to retrieve + @type int + @return Breakpoint or None, if there is no bp + @rtype Breakpoint object or None + """ + return Breakpoint.breaks.get((filename, lineno)) + + @staticmethod + def effectiveBreak(filename, lineno, frame): + """ + Public method to determine which breakpoint for this filename:lineno + is to be acted upon. + + Called only if we know there is a bpt at this + location. Returns breakpoint that was triggered and a flag + that indicates if it is ok to delete a temporary bp. + + @param filename file name of the bp to retrieve + @type str + @param lineno line number of the bp to retrieve + @type int + @param frame the current execution frame + @type frame object + @return tuple of Breakpoint and a flag to indicate, that a + temporary breakpoint may be deleted + @rtype tuple of Breakpoint, bool + """ + b = Breakpoint.breaks[filename, lineno] + if not b.enabled: + return (None, False) + + # Count every hit when bp is enabled + b.hits += 1 + if not b.cond: + # If unconditional, and ignoring, + # go on to next, else break + if b.ignore > 0: + b.ignore -= 1 + return (None, False) + else: + # breakpoint and marker that's ok + # to delete if temporary + return (b, True) + else: + # Conditional bp. + # Ignore count applies only to those bpt hits where the + # condition evaluates to true. + try: + val = eval(b.cond, frame.f_globals, frame.f_locals) + if val: + if b.ignore > 0: + b.ignore -= 1 + # continue + else: + return (b, True) + # else: + # continue + except Exception: + # if eval fails, most conservative + # thing is to stop on breakpoint + # regardless of ignore count. + # Don't delete temporary, + # as another hint to user. + return (b, False) + return (None, False) + + +class Watch: + """ + Watch class. + + Implements temporary watches, ignore counts, disabling and + (re)-enabling, and conditionals. + """ + watches = [] + + def __init__(self, cond, compiledCond, flag, temporary=False): + """ + Constructor + + @param cond condition as string with flag + @type str + @param compiledCond precompiled condition + @type code object + @param flag indicates type of watch (created or changed) + @type str + @keyparam temporary flag for temporary watches + @type bool + """ + # Should not occur + if not cond: + return + + self.cond = cond + self.compiledCond = compiledCond + self.temporary = temporary + + self.enabled = True + self.ignore = 0 + + self.created = False + self.changed = False + if flag == '??created??': + self.created = True + elif flag == '??changed??': + self.changed = True + + self.values = {} + self.watches.append(self) + + def deleteMe(self): + """ + Public method to clear this watch expression. + """ + try: + del self.watches[self] + except ValueError: + pass + + def enable(self): + """ + Public method to enable this watch. + """ + self.enabled = True + + def disable(self): + """ + Public method to disable this watch. + """ + self.enabled = False + + @staticmethod + def clear_watch(cond): + """ + Public method to clear a watch expression. + + @param cond expression of the watch expression to be cleared + @type str + """ + try: + Watch.watches.remove(Watch.get_watch(cond)) + except ValueError: + pass + + @staticmethod + def clear_all_watches(): + """ + Public method to clear all watch expressions. + """ + del Watch.watches[:] + + @staticmethod + def get_watch(cond): + """ + Public method to get a watch expression. + + @param cond expression of the watch expression to be cleared + @type str + @return reference to the watch point + @rtype Watch or None + """ + for b in Watch.watches: + if b.cond == cond: + return b + + @staticmethod + def effectiveWatch(frame): + """ + Public method to determine, if a watch expression is effective. + + @param frame the current execution frame + @type frame object + @return tuple of watch expression and a flag to indicate, that a + temporary watch expression may be deleted + @rtype tuple of Watch, int + """ + for b in Watch.watches: + if not b.enabled: + continue + try: + val = eval(b.compiledCond, frame.f_globals, frame.f_locals) + if b.created: + if frame in b.values: + continue + else: + b.values[frame] = [1, val, b.ignore] + return (b, True) + + elif b.changed: + try: + if b.values[frame][1] != val: + b.values[frame][1] = val + else: + continue + except KeyError: + b.values[frame] = [1, val, b.ignore] + + if b.values[frame][2] > 0: + b.values[frame][2] -= 1 + continue + else: + return (b, True) + + elif val: + if b.ignore > 0: + b.ignore -= 1 + continue + else: + return (b, True) + except Exception: + continue + return (None, False) + + +# +# eflag: noqa = M702
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/DebugClients/Python/DCTestResult.py Sun Oct 16 12:18:42 2016 +0200 @@ -0,0 +1,130 @@ +# -*- coding: utf-8 -*- + +# Copyright (c) 2003 - 2016 Detlev Offenbach <detlev@die-offenbachs.de> +# + +""" +Module implementing a TestResult derivative for the eric6 debugger. +""" + +import select +from unittest import TestResult + + +class DCTestResult(TestResult): + """ + A TestResult derivative to work with eric6's debug client. + + For more details see unittest.py of the standard python distribution. + """ + def __init__(self, dbgClient): + """ + Constructor + + @param dbgClient reference to the debug client + @type DebugClientBase + """ + TestResult.__init__(self) + self.__dbgClient = dbgClient + + def addFailure(self, test, err): + """ + Public method called if a test failed. + + @param test Reference to the test object + @param err The error traceback + """ + TestResult.addFailure(self, test, err) + tracebackLines = self._exc_info_to_string(err, test) + self.__dbgClient.sendJsonCommand("ResponseUTTestFailed", { + "testname": str(test), + "traceback": tracebackLines, + "id": test.id(), + }) + + def addError(self, test, err): + """ + Public method called if a test errored. + + @param test Reference to the test object + @param err The error traceback + """ + TestResult.addError(self, test, err) + tracebackLines = self._exc_info_to_string(err, test) + self.__dbgClient.sendJsonCommand("ResponseUTTestErrored", { + "testname": str(test), + "traceback": tracebackLines, + "id": test.id(), + }) + + def addSkip(self, test, reason): + """ + Public method called if a test was skipped. + + @param test reference to the test object + @param reason reason for skipping the test (string) + """ + TestResult.addSkip(self, test, reason) + self.__dbgClient.sendJsonCommand("ResponseUTTestSkipped", { + "testname": str(test), + "reason": reason, + "id": test.id(), + }) + + def addExpectedFailure(self, test, err): + """ + Public method called if a test failed expected. + + @param test reference to the test object + @param err error traceback + """ + TestResult.addExpectedFailure(self, test, err) + tracebackLines = self._exc_info_to_string(err, test) + self.__dbgClient.sendJsonCommand("ResponseUTTestFailedExpected", { + "testname": str(test), + "traceback": tracebackLines, + "id": test.id(), + }) + + def addUnexpectedSuccess(self, test): + """ + Public method called if a test succeeded expectedly. + + @param test reference to the test object + """ + TestResult.addUnexpectedSuccess(self, test) + self.__dbgClient.sendJsonCommand("ResponseUTTestSucceededUnexpected", { + "testname": str(test), + "id": test.id(), + }) + + def startTest(self, test): + """ + Public method called at the start of a test. + + @param test Reference to the test object + """ + TestResult.startTest(self, test) + self.__dbgClient.sendJsonCommand("ResponseUTStartTest", { + "testname": str(test), + "description": test.shortDescription(), + }) + + def stopTest(self, test): + """ + Public method called at the end of a test. + + @param test Reference to the test object + """ + TestResult.stopTest(self, test) + self.__dbgClient.sendJsonCommand("ResponseUTStopTest", {}) + + # ensure that pending input is processed + rrdy, wrdy, xrdy = select.select( + [self.__dbgClient.readstream], [], [], 0.01) + + if self.__dbgClient.readstream in rrdy: + self.__dbgClient.readReady(self.__dbgClient.readstream) + +# +# eflag: noqa = M702
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/DebugClients/Python/DebugBase.py Sun Oct 16 12:18:42 2016 +0200 @@ -0,0 +1,997 @@ +# -*- coding: utf-8 -*- + +# Copyright (c) 2002 - 2016 Detlev Offenbach <detlev@die-offenbachs.de> +# + +""" +Module implementing the debug base class which based originally on bdb. +""" + +import sys +import os +import types +import atexit +import inspect +import ctypes +import time +from inspect import CO_GENERATOR + +from BreakpointWatch import Breakpoint, Watch + +if sys.version_info[0] == 2: + import thread as _thread + from inspect import getargvalues, formatargvalues +else: + import _thread + from DebugUtilities import getargvalues, formatargvalues + unicode = str + basestring = str + +gRecursionLimit = 64 + + +def printerr(s): + """ + Module function used for debugging the debug client. + + @param s data to be printed + """ + sys.__stderr__.write('{0!s}\n'.format(s)) + sys.__stderr__.flush() + + +def setRecursionLimit(limit): + """ + Module function to set the recursion limit. + + @param limit recursion limit (integer) + """ + global gRecursionLimit + gRecursionLimit = limit + + +class DebugBase(object): + """ + Class implementing base class of the debugger. + + Provides methods for the 'owning' client to call to step etc. + """ + # Don't thrust distutils.sysconfig.get_python_lib: possible case mismatch + # on Windows + lib = os.path.dirname(inspect.__file__) + # tuple required because it's accessed a lot of times by startswith method + pathsToSkip = ('<', os.path.dirname(__file__), inspect.__file__[:-1]) + filesToSkip = {} + + # cache for fixed file names + _fnCache = {} + + def __init__(self, dbgClient): + """ + Constructor + + @param dbgClient the owning client + """ + self._dbgClient = dbgClient + + # Some informations about the thread + self.isMainThread = False + self.quitting = False + self.id = -1 + self.name = '' + + self.tracePythonLibs(0) + + # Special handling of a recursion error + self.skipFrames = 0 + + self.isBroken = False + self.cFrame = None + + # current frame we are at + self.currentFrame = None + + # frames, where we want to stop or release debugger + self.botframe = None + self.stopframe = None + self.returnframe = None + self.stop_everywhere = False + + self.__recursionDepth = -1 + self.setRecursionDepth(inspect.currentframe()) + + # background task to periodicaly check for client interactions + self.eventPollFlag = False + self.timer = _thread.start_new_thread(self.__eventPollTimer, ()) + + def __eventPollTimer(self): + """ + Private method to set a flag every 0.5 s to check for new messages. + """ + while True: + time.sleep(0.5) + self.eventPollFlag = True + + def getCurrentFrame(self): + """ + Public method to return the current frame. + + @return the current frame + @rtype frame object + """ + # Don't show any local frames after the program was stopped + if self.quitting: + return None + + return self.currentFrame + + def getFrameLocals(self, frmnr=0): + """ + Public method to return the locals dictionary of the current frame + or a frame below. + + @keyparam frmnr distance of frame to get locals dictionary of. 0 is + the current frame (int) + @return locals dictionary of the frame + """ + f = self.currentFrame + while f is not None and frmnr > 0: + f = f.f_back + frmnr -= 1 + return f.f_locals + + def storeFrameLocals(self, frmnr=0): + """ + Public method to store the locals into the frame, so an access to + frame.f_locals returns the last data. + + @keyparam frmnr distance of frame to store locals dictionary to. 0 is + the current frame (int) + """ + cf = self.currentFrame + while cf is not None and frmnr > 0: + cf = cf.f_back + frmnr -= 1 + + try: + if "__pypy__" in sys.builtin_module_names: + import __pypy__ + __pypy__.locals_to_fast(cf) + return + except Exception: + pass + + ctypes.pythonapi.PyFrame_LocalsToFast( + ctypes.py_object(cf), + ctypes.c_int(0)) + + def step(self, traceMode): + """ + Public method to perform a step operation in this thread. + + @param traceMode If it is True, then the step is a step into, + otherwise it is a step over. + """ + if traceMode: + self.set_step() + else: + self.set_next(self.currentFrame) + + def stepOut(self): + """ + Public method to perform a step out of the current call. + """ + self.set_return(self.currentFrame) + + def go(self, special): + """ + Public method to resume the thread. + + It resumes the thread stopping only at breakpoints or exceptions. + + @param special flag indicating a special continue operation + """ + self.set_continue(special) + + def setRecursionDepth(self, frame): + """ + Public method to determine the current recursion depth. + + @param frame The current stack frame. + """ + self.__recursionDepth = 0 + while frame is not None: + self.__recursionDepth += 1 + frame = frame.f_back + + def profileWithRecursion(self, frame, event, arg): + """ + Public method used to trace some stuff independent of the debugger + trace function. + + @param frame current stack frame + @type frame object + @param event trace event + @type str + @param arg arguments + @type depends on the previous event parameter + @exception RuntimeError raised to indicate too many recursions + """ + if event == 'return': + self.cFrame = frame.f_back + self.__recursionDepth -= 1 + if self._dbgClient.callTraceEnabled: + self.__sendCallTrace(event, frame, self.cFrame) + elif event == 'call': + if self._dbgClient.callTraceEnabled: + self.__sendCallTrace(event, self.cFrame, frame) + self.cFrame = frame + self.__recursionDepth += 1 + if self.__recursionDepth > gRecursionLimit: + raise RuntimeError( + 'maximum recursion depth exceeded\n' + '(offending frame is two down the stack)') + + def profile(self, frame, event, arg): + """ + Public method used to trace some stuff independent of the debugger + trace function. + + @param frame current stack frame + @type frame object + @param event trace event + @type str + @param arg arguments + @type depends on the previous event parameter + """ + if event == 'return': + self.__sendCallTrace(event, frame, frame.f_back) + elif event == 'call': + self.__sendCallTrace(event, frame.f_back, frame) + + def __sendCallTrace(self, event, fromFrame, toFrame): + """ + Private method to send a call/return trace. + + @param event trace event + @type str + @param fromFrame originating frame + @type frame object + @param toFrame destination frame + @type frame object + """ + if not self.__skipFrame(fromFrame) and not self.__skipFrame(toFrame): + fromInfo = { + "filename": self._dbgClient.absPath( + self.fix_frame_filename(fromFrame)), + "linenumber": fromFrame.f_lineno, + "codename": fromFrame.f_code.co_name, + } + toInfo = { + "filename": self._dbgClient.absPath( + self.fix_frame_filename(toFrame)), + "linenumber": toFrame.f_lineno, + "codename": toFrame.f_code.co_name, + } + self._dbgClient.sendCallTrace(event, fromInfo, toInfo) + + def trace_dispatch(self, frame, event, arg): + """ + Public method reimplemented from bdb.py to do some special things. + + This specialty is to check the connection to the debug server + for new events (i.e. new breakpoints) while we are going through + the code. + + @param frame The current stack frame + @type frame object + @param event The trace event + @type str + @param arg The arguments + @type depends on the previous event parameter + @return local trace function + @rtype trace function or None + @exception SystemExit + """ + # give the client a chance to push through new break points. + if self.eventPollFlag: + self._dbgClient.eventPoll() + self.eventPollFlag = False + + if self.quitting: + raise SystemExit + + if event == 'line': + if self.stop_here(frame) or self.break_here(frame): + self.user_line(frame) + return + + if event == 'call': + if self.botframe is None and frame.f_lineno > 1: + self.botframe = frame + frame.f_trace = self.trace_dispatch + + self.user_line(frame) + return self.trace_dispatch + + if not (self.stop_here(frame) or + self.__checkBreakInFrame(frame) or + Watch.watches != []): + # No need to trace this function + return + return self.trace_dispatch + + if event == 'return': + if self.stop_here(frame) or frame == self.botframe: + # Ignore return events in generator except when stepping. + if self.stopframe and frame.f_code.co_flags & CO_GENERATOR: + return + # The program has finished if we have just left the first frame + if frame == self.botframe: + if self.isMainThread: + atexit._run_exitfuncs() + self._dbgClient.progTerminated(arg) + else: + self._dbgClient.threadTerminated(self.id) + + if self.quitting and not self._dbgClient.passive: + raise SystemExit + return + + if event == 'exception': + if not self.__skipFrame(frame): + # When stepping with next/until/return in a generator frame, + # skip the internal StopIteration exception (with no traceback) + # triggered by a subiterator run with the 'yield from' + # statement. + if not (frame.f_code.co_flags & CO_GENERATOR and + arg[0] is StopIteration and arg[2] is None): + self.user_exception(frame, arg) + # Stop at the StopIteration or GeneratorExit exception when the + # user has set stopframe in a generator by issuing a return + # command, or a next/until command at the last statement in the + # generator before the exception. + elif (self.stopframe and frame is not self.stopframe and + self.stopframe.f_code.co_flags & CO_GENERATOR and + arg[0] in (StopIteration, GeneratorExit)): + self.user_exception(frame, arg) + return + + if event == 'c_call': + return + if event == 'c_exception': + return + if event == 'c_return': + return + + print('DebugBase.trace_dispatch:' # __IGNORE_WARNING__ + ' unknown debugging event: ', + repr(event)) + return self.trace_dispatch + + def set_trace(self, frame=None): + """ + Public method to start debugging from 'frame'. + + If frame is not specified, debugging starts from caller's frame. + Because of jump optimizations it's not possible to use sys.breakpoint() + as last instruction in a function or method. + + @keyparam frame frame to start debugging from + @type frame object + """ + if frame is None: + frame = sys._getframe().f_back # Skip set_trace method + + if sys.version_info[0] == 2: + stopOnHandleLine = self._dbgClient.handleLine.func_code + bootstrap = '__bootstrap' + else: + stopOnHandleLine = self._dbgClient.handleLine.__code__ + bootstrap = 'bootstrap' + + frame.f_trace = self.trace_dispatch + while frame is not None: + # stop at erics debugger frame or the threading bootstrap + if (frame.f_back.f_code == stopOnHandleLine or + frame.f_back.f_code.co_name == bootstrap): + frame.f_trace = self.trace_dispatch + self.botframe = frame + break + + frame = frame.f_back + + self.stop_everywhere = True + sys.settrace(self.trace_dispatch) + sys.setprofile(self._dbgClient.callTraceEnabled) + + def bootstrap(self, target, args, kwargs): + """ + Public method to bootstrap a thread. + + It wraps the call to the user function to enable tracing + before hand. + + @param target function which is called in the new created thread + @type function pointer + @param args arguments to pass to target + @type tuple + @param kwargs keyword arguments to pass to target + @type dict + """ + try: + frame = sys._getframe() + self.botframe = frame + # First time the dispach function is called, a "base debug" + # function has to be returned, which is called at every user code + # function call. Because of setting botframe manually, the + # specific branch returning the trace_dispatch itself is skipped. + # Because the next step is always in threading.py and we assume + # that there is no breakpoint in it, it's save to return + # trace_dispatch unconditionally. + sys.settrace(lambda frame, event, arg: self.trace_dispatch) + frame.f_trace = self.trace_dispatch + + target(*args, **kwargs) + except SystemExit: + pass + finally: + sys.setprofile(None) + + def run(self, cmd, globals=None, locals=None): + """ + Public method to start a given command under debugger control. + + @param cmd command / code to execute under debugger control + @type str or CodeType + @keyparam globals dictionary of global variables for cmd + @type dict + @keyparam locals dictionary of local variables for cmd + @type dict + """ + if globals is None: + import __main__ + globals = __main__.__dict__ + + if locals is None: + locals = globals + + sys.settrace(self.trace_dispatch) + if not isinstance(cmd, types.CodeType): + cmd = compile(cmd, "<string>", "exec") + + try: + exec(cmd, globals, locals) + except SystemExit: + pass + finally: + self.quitting = True + sys.settrace(None) + + def _set_stopinfo(self, stopframe, returnframe): + """ + Protected method to update the frame pointers. + + @param stopframe the frame object where to stop + @type frame object + @param returnframe the frame object where to stop on a function return + @type frame object + """ + self.stopframe = stopframe + self.returnframe = returnframe + self.stop_everywhere = False + + def set_continue(self, special): + """ + Public method to stop only on next breakpoint. + + @param special flag indicating a special continue operation + @type bool + """ + # Here we only set a new stop frame if it is a normal continue. + if not special: + self._set_stopinfo(None, None) + + # Disable tracing if not started in debug mode + if not self._dbgClient.debugging: + sys.settrace(None) + sys.setprofile(None) + + def set_step(self): + """ + Public method to stop after one line of code. + """ + self._set_stopinfo(None, None) + self.stop_everywhere = True + + def set_next(self, frame): + """ + Public method to stop on the next line in or below the given frame. + + @param frame the frame object + @type frame object + """ + self._set_stopinfo(frame, frame.f_back) + frame.f_back.f_trace = self.trace_dispatch + frame.f_trace = self.trace_dispatch + + def set_return(self, frame): + """ + Public method to stop when returning from the given frame. + + @param frame the frame object + @type frame object + """ + self._set_stopinfo(None, frame.f_back) + + def set_quit(self): + """ + Public method to quit. + + Disables the trace functions and resets all frame pointer. + """ + sys.setprofile(None) + sys.settrace(None) + self.stopframe = None + self.returnframe = None + self.quitting = True + + def fix_frame_filename(self, frame): + """ + Public method used to fixup the filename for a given frame. + + The logic employed here is that if a module was loaded + from a .pyc file, then the correct .py to operate with + should be in the same path as the .pyc. The reason this + logic is needed is that when a .pyc file is generated, the + filename embedded and thus what is readable in the code object + of the frame object is the fully qualified filepath when the + pyc is generated. If files are moved from machine to machine + this can break debugging as the .pyc will refer to the .py + on the original machine. Another case might be sharing + code over a network... This logic deals with that. + + @param frame the frame object + @type frame object + @return fixed up file name + @rtype str + """ + # get module name from __file__ + fn = frame.f_globals.get('__file__') + try: + return self._fnCache[fn] + except KeyError: + if fn and fn != frame.f_code.co_filename: + absFilename = os.path.abspath(fn) + if absFilename.endswith(('.pyc', '.pyo')): + fixedName = absFilename[:-1] + if not os.path.exists(fixedName): + fixedName = absFilename + else: + fixedName = absFilename + else: + fixedName = frame.f_code.co_filename + # update cache + self._fnCache[fn] = fixedName + return fixedName + + def __checkBreakInFrame(self, frame): + """ + Private method to check if the function / method has a line number + which is a breakpoint. + + @param frame the frame object + @type frame object + @return Flag indicating a function / method with breakpoint + @rtype bool + """ + try: + return Breakpoint.breakInFrameCache[ + frame.f_globals.get('__file__'), + frame.f_code.co_firstlineno] + except KeyError: + filename = self.fix_frame_filename(frame) + if filename not in Breakpoint.breakInFile: + Breakpoint.breakInFrameCache[ + frame.f_globals.get('__file__'), + frame.f_code.co_firstlineno] = False + return False + lineNo = frame.f_code.co_firstlineno + lineNumbers = [lineNo] + + if sys.version_info[0] == 2: + co_lnotab = map(ord, frame.f_code.co_lnotab[1::2]) + else: + co_lnotab = frame.f_code.co_lnotab[1::2] + + # No need to handle special case if a lot of lines between + # (e.g. closure), because the additional lines won't cause a bp + for co_lno in co_lnotab: + lineNo += co_lno + lineNumbers.append(lineNo) + + for bp in Breakpoint.breakInFile[filename]: + if bp in lineNumbers: + Breakpoint.breakInFrameCache[ + frame.f_globals.get('__file__'), + frame.f_code.co_firstlineno] = True + return True + Breakpoint.breakInFrameCache[ + frame.f_globals.get('__file__'), + frame.f_code.co_firstlineno] = False + return False + + def break_here(self, frame): + """ + Public method reimplemented from bdb.py to fix the filename from the + frame. + + See fix_frame_filename for more info. + + @param frame the frame object + @type frame object + @return flag indicating the break status + @rtype bool + """ + filename = self.fix_frame_filename(frame) + if (filename, frame.f_lineno) in Breakpoint.breaks: + bp, flag = Breakpoint.effectiveBreak( + filename, frame.f_lineno, frame) + if bp: + # flag says ok to delete temp. bp + if flag and bp.temporary: + self.__do_clearBreak(filename, frame.f_lineno) + return True + + if Watch.watches != []: + bp, flag = Watch.effectiveWatch(frame) + if bp: + # flag says ok to delete temp. watch + if flag and bp.temporary: + self.__do_clearWatch(bp.cond) + return True + + return False + + def __do_clearBreak(self, filename, lineno): + """ + Private method called to clear a temporary breakpoint. + + @param filename name of the file the bp belongs to + @type str + @param lineno linenumber of the bp + @type int + """ + Breakpoint.clear_break(filename, lineno) + self._dbgClient.sendClearTemporaryBreakpoint(filename, lineno) + + def __do_clearWatch(self, cond): + """ + Private method called to clear a temporary watch expression. + + @param cond expression of the watch expression to be cleared + @type str + """ + Watch.clear_watch(cond) + self._dbgClient.sendClearTemporaryWatch(cond) + + def getStack(self, frame=None, applyTrace=False): + """ + Public method to get the stack. + + @keyparam frame frame object to inspect + @type frame object or list + @keyparam applyTrace flag to assign trace function to fr.f_trace + @type bool + @return list of lists with file name (string), line number (integer) + and function name (string) + """ + if frame is None: + fr = self.getCurrentFrame() + elif type(frame) == list: + fr = frame.pop(0) + else: + fr = frame + + stack = [] + while fr is not None: + if applyTrace: + # Reset the trace function so we can be sure + # to trace all functions up the stack... This gets around + # problems where an exception/breakpoint has occurred + # but we had disabled tracing along the way via a None + # return from dispatch_call + fr.f_trace = self.trace_dispatch + + fname = self._dbgClient.absPath(self.fix_frame_filename(fr)) + # Always show at least one stack frame, even if it's from eric. + if stack and os.path.basename(fname).startswith( + ("DebugBase.py", "DebugClientBase.py", + "ThreadExtension.py", "threading.py")): + break + + fline = fr.f_lineno + ffunc = fr.f_code.co_name + + if ffunc == '?': + ffunc = '' + + if ffunc and not ffunc.startswith("<"): + argInfo = getargvalues(fr) + try: + fargs = formatargvalues( + argInfo.args, argInfo.varargs, + argInfo.keywords, argInfo.locals) + except Exception: + fargs = "" + else: + fargs = "" + + stack.append([fname, fline, ffunc, fargs]) + + # is it a stack frame or exception list? + if type(frame) == list: + if frame != []: + fr = frame.pop(0) + else: + fr = None + else: + fr = fr.f_back + + return stack + + def user_line(self, frame): + """ + Public method reimplemented to handle the program about to execute a + particular line. + + @param frame the frame object + """ + # We never stop on line 0. + if frame.f_lineno == 0: + return + + self.currentFrame = frame + stack = self.getStack(frame, applyTrace=True) + + self.isBroken = True + self._dbgClient.lockClient() + self._dbgClient.currentThread = self + self._dbgClient.currentThreadExec = self + + self._dbgClient.sendResponseLine(stack) + self._dbgClient.eventLoop() + + self.isBroken = False + self._dbgClient.unlockClient() + + def user_exception(self, frame, excinfo, unhandled=False): + """ + Public method reimplemented to report an exception to the debug server. + + @param frame the frame object + @type frame object + @param excinfo details about the exception + @type tuple(Exception, excval object, traceback frame object) + @keyparam unhandled flag indicating an uncaught exception + @type bool + """ + exctype, excval, exctb = excinfo + + if exctype in [GeneratorExit, StopIteration]: + # ignore these + return + + if exctype == SystemExit: + atexit._run_exitfuncs() + if excval is None: + exitcode = 0 + message = "" + elif isinstance(excval, basestring): + exitcode = 1 + message = excval + elif isinstance(excval, bytes): + exitcode = 1 + message = excval.decode() + elif isinstance(excval, int): + exitcode = excval + message = "" + elif isinstance(excval, SystemExit): + code = excval.code + if isinstance(code, basestring): + exitcode = 1 + message = code + elif isinstance(code, bytes): + exitcode = 1 + message = code.decode() + elif isinstance(code, int): + exitcode = code + message = "" + else: + exitcode = 1 + message = str(code) + else: + exitcode = 1 + message = str(excval) + self._dbgClient.progTerminated(exitcode, message) + return + + if exctype in [SyntaxError, IndentationError]: + try: + # tuple could only occure on Python 2, but not always! + if type(excval) == tuple: + message, details = excval + filename, lineno, charno, text = details + else: + message = excval.msg + filename = excval.filename + lineno = excval.lineno + charno = excval.offset + + if charno is None: + charno = 0 + + filename = os.path.abspath(filename) + realSyntaxError = os.path.exists(filename) + except (AttributeError, ValueError): + message = "" + filename = "" + lineno = 0 + charno = 0 + realSyntaxError = True + + if realSyntaxError: + self._dbgClient.sendSyntaxError( + message, filename, lineno, charno) + self._dbgClient.eventLoop() + return + + self.skipFrames = 0 + if (exctype == RuntimeError and + str(excval).startswith('maximum recursion depth exceeded') or + sys.version_info >= (3, 5) and + exctype == RecursionError): # __IGNORE_WARNING__ + excval = 'maximum recursion depth exceeded' + depth = 0 + tb = exctb + while tb: + tb = tb.tb_next + + if (tb and tb.tb_frame.f_code.co_name == 'trace_dispatch' and + __file__.startswith(tb.tb_frame.f_code.co_filename)): + depth = 1 + self.skipFrames += depth + + # always 1 if running without debugger + self.skipFrames = max(1, self.skipFrames) + + exctype = self.__extractExceptionName(exctype) + + if excval is None: + excval = '' + + if unhandled: + exctypetxt = "unhandled {0!s}".format(str(exctype)) + else: + exctypetxt = str(exctype) + + if sys.version_info[0] == 2: + try: + excvaltxt = unicode(excval).encode(self._dbgClient.getCoding()) + except UnicodeError: + excvaltxt = str(excval) + else: + excvaltxt = str(excval) + + stack = [] + if exctb: + frlist = self.__extract_stack(exctb) + frlist.reverse() + + self.currentFrame = frlist[0] + + stack = self.getStack(frlist[self.skipFrames:]) + + self._dbgClient.sendException(exctypetxt, excvaltxt, stack) + + if exctb is None: + return + + self._dbgClient.eventLoop() + self.skipFrames = 0 + + def __extractExceptionName(self, exctype): + """ + Private method to extract the exception name given the exception + type object. + + @param exctype type of the exception + @return exception name (string) + """ + if sys.version_info[0] == 2: + if type(exctype) in [types.ClassType, # Python up to 2.4 + types.TypeType]: # Python 2.5+ + return exctype.__name__ + else: + return exctype + else: + return str(exctype).replace("<class '", "").replace("'>", "") + + def __extract_stack(self, exctb): + """ + Private member to return a list of stack frames. + + @param exctb exception traceback + @return list of stack frames + """ + tb = exctb + stack = [] + while tb is not None: + stack.append(tb.tb_frame) + tb = tb.tb_next + tb = None + return stack + + def stop_here(self, frame): + """ + Public method reimplemented to filter out debugger files. + + Tracing is turned off for files that are part of the + debugger that are called from the application being debugged. + + @param frame the frame object + @type frame object + @return flag indicating whether the debugger should stop here + @rtype bool + """ + if self.__skipFrame(frame): + return False + + return (self.stop_everywhere or + frame is self.stopframe or + frame is self.returnframe) + + def tracePythonLibs(self, enable): + """ + Public method to update the settings to trace into Python libraries. + + @param enable flag to debug into Python libraries + @type bool + """ + pathsToSkip = list(self.pathsToSkip) + # don't trace into Python library? + if enable: + pathsToSkip = [x for x in pathsToSkip if not x.endswith( + ("site-packages", "dist-packages", self.lib))] + else: + pathsToSkip.append(self.lib) + localLib = [x for x in sys.path if x.endswith(("site-packages", + "dist-packages")) and not x.startswith(self.lib)] + pathsToSkip.extend(localLib) + + self.pathsToSkip = tuple(pathsToSkip) + + def __skipFrame(self, frame): + """ + Private method to filter out debugger files. + + Tracing is turned off for files that are part of the + debugger that are called from the application being debugged. + + @param frame the frame object + @type frame object + @return flag indicating whether the debugger should skip this frame + @rtype bool + """ + try: + return self.filesToSkip[frame.f_code.co_filename] + except KeyError: + ret = frame.f_code.co_filename.startswith(self.pathsToSkip) + self.filesToSkip[frame.f_code.co_filename] = ret + return ret + except AttributeError: + # if frame is None + return True + +# +# eflag: noqa = M702
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/DebugClients/Python/DebugClient.py Sun Oct 16 12:18:42 2016 +0200 @@ -0,0 +1,41 @@ +# -*- coding: utf-8 -*- + +# Copyright (c) 2003 - 2016 Detlev Offenbach <detlev@die-offenbachs.de> +# + +""" +Module implementing the standard debug client. +""" + +from DebugBase import DebugBase +from DebugClientBase import DebugClientBase +from ThreadExtension import ThreadExtension + + +class DebugClient(DebugClientBase, DebugBase, ThreadExtension): + """ + Class implementing the client side of the debugger. + + This variant of the debugger implements the standard debugger client + by subclassing all relevant base classes. + """ + def __init__(self): + """ + Constructor + """ + DebugClientBase.__init__(self) + + DebugBase.__init__(self, self) + + ThreadExtension.__init__(self) + + self.variant = 'Standard' + +# We are normally called by the debugger to execute directly. + +if __name__ == '__main__': + debugClient = DebugClient() + debugClient.main() + +# +# eflag: noqa = M702
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/DebugClients/Python/DebugClientBase.py Sun Oct 16 12:18:42 2016 +0200 @@ -0,0 +1,2106 @@ +# -*- coding: utf-8 -*- + +# Copyright (c) 2002 - 2016 Detlev Offenbach <detlev@die-offenbachs.de> +# + +""" +Module implementing a debug client base class. +""" + +import sys +import socket +import select +import codeop +import codecs +import traceback +import os +import json +import imp +import re +import atexit +import signal + + +import DebugClientCapabilities +import DebugVariables +from DebugBase import setRecursionLimit, printerr # __IGNORE_WARNING__ +from AsyncFile import AsyncFile, AsyncPendingWrite +from DebugConfig import ConfigVarTypeStrings +from FlexCompleter import Completer +from DebugUtilities import prepareJsonCommand +from BreakpointWatch import Breakpoint, Watch + +if sys.version_info[0] == 2: + from inspect import getargvalues, formatargvalues +else: + unichr = chr + from DebugUtilities import getargvalues, formatargvalues + +DebugClientInstance = None + +############################################################################### + + +def DebugClientRawInput(prompt="", echo=True): + """ + Replacement for the standard raw_input builtin. + + This function works with the split debugger. + + @param prompt prompt to be shown. (string) + @param echo flag indicating echoing of the input (boolean) + @return result of the raw_input() call + """ + if DebugClientInstance is None or not DebugClientInstance.redirect: + return DebugClientOrigRawInput(prompt) + + return DebugClientInstance.raw_input(prompt, echo) + + +def DebugClientInput(prompt="", echo=True): + """ + Replacement for the standard input builtin. + + This function works with the split debugger. + + @param prompt prompt to be shown (string) + @param echo flag indicating to echo the output (boolean) + @return result of the input() call + """ + if DebugClientInstance is None or not DebugClientInstance.redirect: + return DebugClientOrigInput(prompt) + + return DebugClientInstance.input(prompt, echo) + +# Use our own input() and on Python 2 raw_input(). +if sys.version_info[0] == 2: + try: + DebugClientOrigRawInput = __builtins__.__dict__['raw_input'] + __builtins__.__dict__['raw_input'] = DebugClientRawInput + except (AttributeError, KeyError): + import __main__ + DebugClientOrigRawInput = __main__.__builtins__.__dict__['raw_input'] + __main__.__builtins__.__dict__['raw_input'] = DebugClientRawInput + + try: + DebugClientOrigInput = __builtins__.__dict__['input'] + __builtins__.__dict__['input'] = DebugClientInput + except (AttributeError, KeyError): + import __main__ + DebugClientOrigInput = __main__.__builtins__.__dict__['input'] + __main__.__builtins__.__dict__['input'] = DebugClientInput +else: + try: + DebugClientOrigInput = __builtins__.__dict__['input'] + __builtins__.__dict__['input'] = DebugClientRawInput + except (AttributeError, KeyError): + import __main__ + DebugClientOrigInput = __main__.__builtins__.__dict__['input'] + __main__.__builtins__.__dict__['input'] = DebugClientRawInput + +############################################################################### + + +def DebugClientFork(): + """ + Replacement for the standard os.fork(). + + @return result of the fork() call + """ + if DebugClientInstance is None: + return DebugClientOrigFork() + + return DebugClientInstance.fork() + +# use our own fork(). +if 'fork' in dir(os): + DebugClientOrigFork = os.fork + os.fork = DebugClientFork + +############################################################################### + + +def DebugClientClose(fd): + """ + Replacement for the standard os.close(fd). + + @param fd open file descriptor to be closed (integer) + """ + if DebugClientInstance is None: + DebugClientOrigClose(fd) + + DebugClientInstance.close(fd) + +# use our own close(). +if 'close' in dir(os): + DebugClientOrigClose = os.close + os.close = DebugClientClose + +############################################################################### + + +def DebugClientSetRecursionLimit(limit): + """ + Replacement for the standard sys.setrecursionlimit(limit). + + @param limit recursion limit (integer) + """ + rl = max(limit, 64) + setRecursionLimit(rl) + DebugClientOrigSetRecursionLimit(rl + 64) + +# use our own setrecursionlimit(). +if 'setrecursionlimit' in dir(sys): + DebugClientOrigSetRecursionLimit = sys.setrecursionlimit + sys.setrecursionlimit = DebugClientSetRecursionLimit + DebugClientSetRecursionLimit(sys.getrecursionlimit()) + +############################################################################### + + +class DebugClientBase(object): + """ + Class implementing the client side of the debugger. + + It provides access to the Python interpeter from a debugger running in + another process. + + The protocol between the debugger and the client is based on JSONRPC 2.0 + PDUs. Each one is sent on a single line, i.e. commands or responses are + separated by a linefeed character. + + If the debugger closes the session there is no response from the client. + The client may close the session at any time as a result of the script + being debugged closing or crashing. + + <b>Note</b>: This class is meant to be subclassed by individual + DebugClient classes. Do not instantiate it directly. + """ + clientCapabilities = DebugClientCapabilities.HasAll + + # keep these in sync with VariablesViewer.VariableItem.Indicators + Indicators = ("()", "[]", "{:}", "{}") # __IGNORE_WARNING__ + + def __init__(self): + """ + Constructor + """ + self.breakpoints = {} + self.redirect = True + self.__receiveBuffer = "" + + # special objects representing the main scripts thread and frame + self.mainThread = self + self.framenr = 0 + + # The context to run the debugged program in. + self.debugMod = imp.new_module('__main__') + self.debugMod.__dict__['__builtins__'] = __builtins__ + + # The list of complete lines to execute. + self.buffer = '' + + # The list of regexp objects to filter variables against + self.globalsFilterObjects = [] + self.localsFilterObjects = [] + + self._fncache = {} + self.dircache = [] + self.passive = False # used to indicate the passive mode + self.running = None + self.test = None + self.debugging = False + + self.fork_auto = False + self.fork_child = False + + self.readstream = None + self.writestream = None + self.errorstream = None + self.pollingDisabled = False + + self.callTraceEnabled = None + + self.variant = 'You should not see this' + + # commandline completion stuff + self.complete = Completer(self.debugMod.__dict__).complete + + self.compile_command = codeop.CommandCompiler() + + self.coding_re = re.compile(r"coding[:=]\s*([-\w_.]+)") + self.defaultCoding = 'utf-8' + self.__coding = self.defaultCoding + self.noencoding = False + + def getCoding(self): + """ + Public method to return the current coding. + + @return codec name (string) + """ + return self.__coding + + def __setCoding(self, filename): + """ + Private method to set the coding used by a python file. + + @param filename name of the file to inspect (string) + """ + if self.noencoding: + self.__coding = sys.getdefaultencoding() + else: + default = 'utf-8' + try: + f = open(filename, 'rb') + # read the first and second line + text = f.readline() + text = "{0}{1}".format(text, f.readline()) + f.close() + except IOError: + self.__coding = default + return + + for line in text.splitlines(): + m = self.coding_re.search(line) + if m: + self.__coding = m.group(1) + return + self.__coding = default + + def raw_input(self, prompt, echo): + """ + Public method to implement raw_input() / input() using the event loop. + + @param prompt the prompt to be shown (string) + @param echo Flag indicating echoing of the input (boolean) + @return the entered string + """ + self.sendJsonCommand("RequestRaw", { + "prompt": prompt, + "echo": echo, + }) + self.eventLoop(True) + return self.rawLine + + def input(self, prompt): + """ + Public method to implement input() (Python 2) using the event loop. + + @param prompt the prompt to be shown (string) + @return the entered string evaluated as a Python expresion + """ + return eval(self.raw_input(prompt, True)) + + def sessionClose(self, exit=True): + """ + Public method to close the session with the debugger and optionally + terminate. + + @param exit flag indicating to terminate (boolean) + """ + try: + self.set_quit() + except Exception: + pass + + self.debugging = False + + # make sure we close down our end of the socket + # might be overkill as normally stdin, stdout and stderr + # SHOULD be closed on exit, but it does not hurt to do it here + self.readstream.close(True) + self.writestream.close(True) + self.errorstream.close(True) + + if exit: + # Ok, go away. + sys.exit() + + def __compileFileSource(self, filename, mode='exec'): + """ + Private method to compile source code read from a file. + + @param filename name of the source file (string) + @param mode kind of code to be generated (string, exec or eval) + @return compiled code object (None in case of errors) + """ + with codecs.open(filename, encoding=self.__coding) as fp: + statement = fp.read() + + if sys.version_info[0] == 2: + lines = statement.splitlines(True) + for lineno, line in enumerate(lines[:2]): + lines[lineno] = self.coding_re.sub('', line) + + statement = unicode('').join(lines) # __IGNORE_WARNING__ + + try: + code = compile(statement + '\n', filename, mode) + except SyntaxError: + exctype, excval, exctb = sys.exc_info() + try: + message = str(excval) + filename = excval.filename + lineno = excval.lineno + charno = excval.offset + if charno is None: + charno = 0 + + except (AttributeError, ValueError): + message = "" + filename = "" + lineno = 0 + charno = 0 + + self.sendSyntaxError(message, filename, lineno, charno) + return None + + return code + + def handleLine(self, line): + """ + Public method to handle the receipt of a complete line. + + It first looks for a valid protocol token at the start of the line. + Thereafter it trys to execute the lines accumulated so far. + + @param line the received line + """ + # Remove any newline. + if line[-1] == '\n': + line = line[:-1] + +## printerr(line) ##debug + + self.handleJsonCommand(line) + + def handleJsonCommand(self, jsonStr): + """ + Public method to handle a command serialized as a JSON string. + + @param jsonStr string containing the command received from the IDE + @type str + """ + try: + commandDict = json.loads(jsonStr.strip()) + except (TypeError, ValueError) as err: + printerr(str(err)) + return + + method = commandDict["method"] + params = commandDict["params"] + if "filename" in params and sys.version_info[0] == 2: + params["filename"] = params["filename"].encode( + sys.getfilesystemencoding()) + + if method == "RequestVariables": + self.__dumpVariables( + params["frameNumber"], params["scope"], params["filters"]) + + elif method == "RequestVariable": + self.__dumpVariable( + params["variable"], params["frameNumber"], + params["scope"], params["filters"]) + + elif method == "RequestThreadList": + self.dumpThreadList() + + elif method == "RequestThreadSet": + if params["threadID"] in self.threads: + self.setCurrentThread(params["threadID"]) + self.sendJsonCommand("ResponseThreadSet", {}) + stack = self.currentThread.getStack() + self.sendJsonCommand("ResponseStack", { + "stack": stack, + }) + + elif method == "RequestCapabilities": + clientType = "Python2" if sys.version_info[0] == 2 else "Python3" + self.sendJsonCommand("ResponseCapabilities", { + "capabilities": self.__clientCapabilities(), + "clientType": clientType + }) + + elif method == "RequestBanner": + self.sendJsonCommand("ResponseBanner", { + "version": "Python {0}".format(sys.version), + "platform": socket.gethostname(), + "dbgclient": self.variant, + }) + + elif method == "RequestSetFilter": + self.__generateFilterObjects(params["scope"], params["filter"]) + + elif method == "RequestCallTrace": + if params["enable"]: + callTraceEnabled = self.profile + else: + callTraceEnabled = None + + if self.debugging: + sys.setprofile(callTraceEnabled) + else: + # remember for later + self.callTraceEnabled = callTraceEnabled + + elif method == "RequestEnvironment": + for key, value in params["environment"].items(): + if key.endswith("+"): + if key[:-1] in os.environ: + os.environ[key[:-1]] += value + else: + os.environ[key[:-1]] = value + else: + os.environ[key] = value + + elif method == "RequestLoad": + self._fncache = {} + self.dircache = [] + sys.argv = [] + self.__setCoding(params["filename"]) + sys.argv.append(params["filename"]) + sys.argv.extend(params["argv"]) + sys.path = self.__getSysPath(os.path.dirname(sys.argv[0])) + if params["workdir"] == '': + os.chdir(sys.path[1]) + else: + os.chdir(params["workdir"]) + + self.running = sys.argv[0] + self.debugging = True + + self.fork_auto = params["autofork"] + self.fork_child = params["forkChild"] + + self.threads.clear() + self.attachThread(mainThread=True) + + # set the system exception handling function to ensure, that + # we report on all unhandled exceptions + sys.excepthook = self.__unhandled_exception + self.__interceptSignals() + + # clear all old breakpoints, they'll get set after we have + # started + Breakpoint.clear_all_breaks() + Watch.clear_all_watches() + + self.mainThread.tracePythonLibs(params["traceInterpreter"]) + + # This will eventually enter a local event loop. + self.debugMod.__dict__['__file__'] = self.running + sys.modules['__main__'] = self.debugMod + code = self.__compileFileSource(self.running) + if code: + sys.setprofile(self.callTraceEnabled) + res = self.mainThread.run(code, self.debugMod.__dict__) + self.progTerminated(res) + + elif method == "RequestRun": + sys.argv = [] + self.__setCoding(params["filename"]) + sys.argv.append(params["filename"]) + sys.argv.extend(params["argv"]) + sys.path = self.__getSysPath(os.path.dirname(sys.argv[0])) + if params["workdir"] == '': + os.chdir(sys.path[1]) + else: + os.chdir(params["workdir"]) + + self.running = sys.argv[0] + self.botframe = None + + self.fork_auto = params["autofork"] + self.fork_child = params["forkChild"] + + self.threads.clear() + self.attachThread(mainThread=True) + + # set the system exception handling function to ensure, that + # we report on all unhandled exceptions + sys.excepthook = self.__unhandled_exception + self.__interceptSignals() + + self.mainThread.tracePythonLibs(False) + + self.debugMod.__dict__['__file__'] = sys.argv[0] + sys.modules['__main__'] = self.debugMod + res = 0 + code = self.__compileFileSource(self.running) + if code: + try: + exec(code, self.debugMod.__dict__) + except SystemExit as exc: + res = exc.code + atexit._run_exitfuncs() + self.writestream.flush() + self.progTerminated(res) + + elif method == "RequestCoverage": + from coverage import coverage + sys.argv = [] + self.__setCoding(params["filename"]) + sys.argv.append(params["filename"]) + sys.argv.extend(params["argv"]) + sys.path = self.__getSysPath(os.path.dirname(sys.argv[0])) + if params["workdir"] == '': + os.chdir(sys.path[1]) + else: + os.chdir(params["workdir"]) + + # set the system exception handling function to ensure, that + # we report on all unhandled exceptions + sys.excepthook = self.__unhandled_exception + self.__interceptSignals() + + # generate a coverage object + self.cover = coverage( + auto_data=True, + data_file="{0}.coverage".format( + os.path.splitext(sys.argv[0])[0])) + + if params["erase"]: + self.cover.erase() + sys.modules['__main__'] = self.debugMod + self.debugMod.__dict__['__file__'] = sys.argv[0] + code = self.__compileFileSource(sys.argv[0]) + if code: + self.running = sys.argv[0] + res = 0 + self.cover.start() + try: + exec(code, self.debugMod.__dict__) + except SystemExit as exc: + res = exc.code + atexit._run_exitfuncs() + self.cover.stop() + self.cover.save() + self.writestream.flush() + self.progTerminated(res) + + elif method == "RequestProfile": + sys.setprofile(None) + import PyProfile + sys.argv = [] + self.__setCoding(params["filename"]) + sys.argv.append(params["filename"]) + sys.argv.extend(params["argv"]) + sys.path = self.__getSysPath(os.path.dirname(sys.argv[0])) + if params["workdir"] == '': + os.chdir(sys.path[1]) + else: + os.chdir(params["workdir"]) + + # set the system exception handling function to ensure, that + # we report on all unhandled exceptions + sys.excepthook = self.__unhandled_exception + self.__interceptSignals() + + # generate a profile object + self.prof = PyProfile.PyProfile(sys.argv[0]) + + if params["erase"]: + self.prof.erase() + self.debugMod.__dict__['__file__'] = sys.argv[0] + sys.modules['__main__'] = self.debugMod + script = '' + if sys.version_info[0] == 2: + script = 'execfile({0!r})'.format(sys.argv[0]) + else: + with codecs.open(sys.argv[0], encoding=self.__coding) as fp: + script = fp.read() + if script and not script.endswith('\n'): + script += '\n' + + if script: + self.running = sys.argv[0] + res = 0 + try: + self.prof.run(script) + except SystemExit as exc: + res = exc.code + + atexit._run_exitfuncs() + self.prof.save() + self.writestream.flush() + self.progTerminated(res) + + elif method == "ExecuteStatement": + if self.buffer: + self.buffer = self.buffer + '\n' + params["statement"] + else: + self.buffer = params["statement"] + + try: + code = self.compile_command(self.buffer, self.readstream.name) + except (OverflowError, SyntaxError, ValueError): + # Report the exception + sys.last_type, sys.last_value, sys.last_traceback = \ + sys.exc_info() + self.sendJsonCommand("ClientOutput", { + "text": "".join(traceback.format_exception_only( + sys.last_type, sys.last_value)) + }) + self.buffer = '' + else: + if code is None: + self.sendJsonCommand("ResponseContinue", {}) + return + else: + self.buffer = '' + + try: + if self.running is None: + exec(code, self.debugMod.__dict__) + else: + if self.currentThread is None: + # program has terminated + self.running = None + _globals = self.debugMod.__dict__ + _locals = _globals + else: + cf = self.currentThread.getCurrentFrame() + # program has terminated + if cf is None: + self.running = None + _globals = self.debugMod.__dict__ + _locals = _globals + else: + frmnr = self.framenr + while cf is not None and frmnr > 0: + cf = cf.f_back + frmnr -= 1 + _globals = cf.f_globals + _locals = \ + self.currentThread.getFrameLocals( + self.framenr) + # reset sys.stdout to our redirector + # (unconditionally) + if "sys" in _globals: + __stdout = _globals["sys"].stdout + _globals["sys"].stdout = self.writestream + exec(code, _globals, _locals) + _globals["sys"].stdout = __stdout + elif "sys" in _locals: + __stdout = _locals["sys"].stdout + _locals["sys"].stdout = self.writestream + exec(code, _globals, _locals) + _locals["sys"].stdout = __stdout + else: + exec(code, _globals, _locals) + + self.currentThread.storeFrameLocals(self.framenr) + except SystemExit as exc: + self.progTerminated(exc.code) + except Exception: + # Report the exception and the traceback + tlist = [] + try: + exc_type, exc_value, exc_tb = sys.exc_info() + sys.last_type = exc_type + sys.last_value = exc_value + sys.last_traceback = exc_tb + tblist = traceback.extract_tb(exc_tb) + del tblist[:1] + tlist = traceback.format_list(tblist) + if tlist: + tlist.insert( + 0, "Traceback (innermost last):\n") + tlist.extend(traceback.format_exception_only( + exc_type, exc_value)) + finally: + tblist = exc_tb = None + + self.sendJsonCommand("ClientOutput", { + "text": "".join(tlist) + }) + + self.sendJsonCommand("ResponseOK", {}) + + elif method == "RequestStep": + self.currentThreadExec.step(True) + self.eventExit = True + + elif method == "RequestStepOver": + self.currentThreadExec.step(False) + self.eventExit = True + + elif method == "RequestStepOut": + self.currentThreadExec.stepOut() + self.eventExit = True + + elif method == "RequestStepQuit": + if self.passive: + self.progTerminated(42) + else: + self.set_quit() + self.eventExit = True + + elif method == "RequestContinue": + self.currentThread.go(params["special"]) + self.eventExit = True + + elif method == "RawInput": + # If we are handling raw mode input then break out of the current + # event loop. + self.rawLine = params["input"] + self.eventExit = True + + elif method == "RequestBreakpoint": + if params["setBreakpoint"]: + if params["condition"] in ['None', '']: + cond = None + elif params["condition"] is not None: + try: + cond = compile(params["condition"], '<string>', 'eval') + except SyntaxError: + self.sendJsonCommand("ResponseBPConditionError", { + "filename": params["filename"], + "line": params["line"], + }) + return + else: + cond = None + + Breakpoint( + params["filename"], params["line"], params["temporary"], + cond) + else: + Breakpoint.clear_break(params["filename"], params["line"]) + + elif method == "RequestBreakpointEnable": + bp = Breakpoint.get_break(params["filename"], params["line"]) + if bp is not None: + if params["enable"]: + bp.enable() + else: + bp.disable() + + elif method == "RequestBreakpointIgnore": + bp = Breakpoint.get_break(params["filename"], params["line"]) + if bp is not None: + bp.ignore = params["count"] + + elif method == "RequestWatch": + if params["setWatch"]: + if params["condition"].endswith( + ('??created??', '??changed??')): + compiledCond, flag = params["condition"].split() + else: + compiledCond = params["condition"] + flag = '' + + try: + compiledCond = compile(compiledCond, '<string>', 'eval') + except SyntaxError: + self.sendJsonCommand("ResponseWatchConditionError", { + "condition": params["condition"], + }) + return + Watch( + params["condition"], compiledCond, flag, + params["temporary"]) + else: + Watch.clear_watch(params["condition"]) + + elif method == "RequestWatchEnable": + wp = Watch.get_watch(params["condition"]) + if wp is not None: + if params["enable"]: + wp.enable() + else: + wp.disable() + + elif method == "RequestWatchIgnore": + wp = Watch.get_watch(params["condition"]) + if wp is not None: + wp.ignore = params["count"] + + elif method == "RequestShutdown": + self.sessionClose() + + elif method == "RequestCompletion": + self.__completionList(params["text"]) + + elif method == "RequestUTPrepare": + sys.path.insert( + 0, os.path.dirname(os.path.abspath(params["filename"]))) + os.chdir(sys.path[0]) + + # set the system exception handling function to ensure, that + # we report on all unhandled exceptions + sys.excepthook = self.__unhandled_exception + self.__interceptSignals() + + try: + import unittest + utModule = imp.load_source( + params["testname"], params["filename"]) + try: + if params["failed"]: + self.test = unittest.defaultTestLoader\ + .loadTestsFromNames(params["failed"], utModule) + else: + self.test = unittest.defaultTestLoader\ + .loadTestsFromName(params["testfunctionname"], + utModule) + except AttributeError: + self.test = unittest.defaultTestLoader\ + .loadTestsFromModule(utModule) + except Exception: + exc_type, exc_value, exc_tb = sys.exc_info() + self.sendJsonCommand("ResponseUTPrepared", { + "count": 0, + "exception": exc_type.__name__, + "message": str(exc_value), + }) + return + + # generate a coverage object + if params["coverage"]: + from coverage import coverage + self.cover = coverage( + auto_data=True, + data_file="{0}.coverage".format( + os.path.splitext(params["coveragefile"])[0])) + if params["coverageerase"]: + self.cover.erase() + else: + self.cover = None + + self.sendJsonCommand("ResponseUTPrepared", { + "count": self.test.countTestCases(), + "exception": "", + "message": "", + }) + + elif method == "RequestUTRun": + from DCTestResult import DCTestResult + self.testResult = DCTestResult(self) + if self.cover: + self.cover.start() + self.test.run(self.testResult) + if self.cover: + self.cover.stop() + self.cover.save() + self.sendJsonCommand("ResponseUTFinished", {}) + + elif method == "RequestUTStop": + self.testResult.stop() + + elif method == "ResponseForkTo": + # this results from a separate event loop + self.fork_child = (params["target"] == 'child') + self.eventExit = True + + def sendJsonCommand(self, method, params): + """ + Public method to send a single command or response to the IDE. + + @param method command or response command name to be sent + @type str + @param params dictionary of named parameters for the command or + response + @type dict + """ + cmd = prepareJsonCommand(method, params) + + self.writestream.write_p(cmd) + self.writestream.flush() + + def sendClearTemporaryBreakpoint(self, filename, lineno): + """ + Public method to signal the deletion of a temporary breakpoint. + + @param filename name of the file the bp belongs to + @type str + @param lineno linenumber of the bp + @type int + """ + self.sendJsonCommand("ResponseClearBreakpoint", { + "filename": filename, + "line": lineno + }) + + def sendClearTemporaryWatch(self, condition): + """ + Public method to signal the deletion of a temporary watch expression. + + @param condition condition of the watch expression to be cleared + @type str + """ + self.sendJsonCommand("ResponseClearWatch", { + "condition": condition, + }) + + def sendResponseLine(self, stack): + """ + Public method to send the current call stack. + + @param stack call stack + @type list + """ + self.sendJsonCommand("ResponseLine", { + "stack": stack, + }) + + def sendCallTrace(self, event, fromInfo, toInfo): + """ + Public method to send a call trace entry. + + @param event trace event (call or return) + @type str + @param fromInfo dictionary containing the origin info + @type dict with 'filename', 'linenumber' and 'codename' + as keys + @param toInfo dictionary containing the target info + @type dict with 'filename', 'linenumber' and 'codename' + as keys + """ + self.sendJsonCommand("CallTrace", { + "event": event[0], + "from": fromInfo, + "to": toInfo, + }) + + def sendException(self, exceptionType, exceptionMessage, stack): + """ + Public method to send information for an exception. + + @param exceptionType type of exception raised + @type str + @param exceptionMessage message of the exception + @type str + @param stack stack trace information + @type list + """ + self.sendJsonCommand("ResponseException", { + "type": exceptionType, + "message": exceptionMessage, + "stack": stack, + }) + + def sendSyntaxError(self, message, filename, lineno, charno): + """ + Public method to send information for a syntax error. + + @param message syntax error message + @type str + @param filename name of the faulty file + @type str + @param lineno line number info + @type int + @param charno character number info + @type int + """ + self.sendJsonCommand("ResponseSyntax", { + "message": message, + "filename": filename, + "linenumber": lineno, + "characternumber": charno, + }) + + def sendPassiveStartup(self, filename, exceptions): + """ + Public method to send the passive start information. + + @param filename name of the script + @type str + @param exceptions flag to enable exception reporting of the IDE + @type bool + """ + self.sendJsonCommand("PassiveStartup", { + "filename": filename, + "exceptions": exceptions, + }) + + def __clientCapabilities(self): + """ + Private method to determine the clients capabilities. + + @return client capabilities (integer) + """ + try: + import PyProfile # __IGNORE_WARNING__ + try: + del sys.modules['PyProfile'] + except KeyError: + pass + return self.clientCapabilities + except ImportError: + return ( + self.clientCapabilities & ~DebugClientCapabilities.HasProfiler) + + def readReady(self, stream): + """ + Public method called when there is data ready to be read. + + @param stream file like object that has data to be written + """ + try: + got = stream.readline_p() + except Exception: + return + + if len(got) == 0: + self.sessionClose() + return + + self.__receiveBuffer = self.__receiveBuffer + got + + # Call handleLine for the line if it is complete. + eol = self.__receiveBuffer.find('\n') + while eol >= 0: + line = self.__receiveBuffer[:eol + 1] + self.__receiveBuffer = self.__receiveBuffer[eol + 1:] + self.handleLine(line) + eol = self.__receiveBuffer.find('\n') + + def writeReady(self, stream): + """ + Public method called when we are ready to write data. + + @param stream file like object that has data to be written + """ + stream.write_p("") + stream.flush() + + def __interact(self): + """ + Private method to interact with the debugger. + """ + global DebugClientInstance + + DebugClientInstance = self + self.__receiveBuffer = "" + + if not self.passive: + # At this point simulate an event loop. + self.eventLoop() + + def eventLoop(self, disablePolling=False): + """ + Public method implementing our event loop. + + @param disablePolling flag indicating to enter an event loop with + polling disabled (boolean) + """ + self.eventExit = None + self.pollingDisabled = disablePolling + + while self.eventExit is None: + wrdy = [] + + if self.writestream.nWriteErrors > self.writestream.maxtries: + break + + if AsyncPendingWrite(self.writestream): + wrdy.append(self.writestream) + + if AsyncPendingWrite(self.errorstream): + wrdy.append(self.errorstream) + + try: + rrdy, wrdy, xrdy = select.select([self.readstream], wrdy, []) + except (select.error, KeyboardInterrupt, socket.error): + # just carry on + continue + + if self.readstream in rrdy: + self.readReady(self.readstream) + + if self.writestream in wrdy: + self.writeReady(self.writestream) + + if self.errorstream in wrdy: + self.writeReady(self.errorstream) + + self.eventExit = None + self.pollingDisabled = False + + def eventPoll(self): + """ + Public method to poll for events like 'set break point'. + """ + if self.pollingDisabled: + return + + wrdy = [] + if AsyncPendingWrite(self.writestream): + wrdy.append(self.writestream) + + if AsyncPendingWrite(self.errorstream): + wrdy.append(self.errorstream) + + # immediate return if nothing is ready. + try: + rrdy, wrdy, xrdy = select.select([self.readstream], wrdy, [], 0) + except (select.error, KeyboardInterrupt, socket.error): + return + + if self.readstream in rrdy: + self.readReady(self.readstream) + + if self.writestream in wrdy: + self.writeReady(self.writestream) + + if self.errorstream in wrdy: + self.writeReady(self.errorstream) + + def connectDebugger(self, port, remoteAddress=None, redirect=True): + """ + Public method to establish a session with the debugger. + + It opens a network connection to the debugger, connects it to stdin, + stdout and stderr and saves these file objects in case the application + being debugged redirects them itself. + + @param port the port number to connect to (int) + @param remoteAddress the network address of the debug server host + (string) + @param redirect flag indicating redirection of stdin, stdout and + stderr (boolean) + """ + if remoteAddress is None: + remoteAddress = "127.0.0.1" + elif "@@i" in remoteAddress: + remoteAddress = remoteAddress.split("@@i")[0] + sock = socket.create_connection((remoteAddress, port)) + + self.readstream = AsyncFile(sock, sys.stdin.mode, sys.stdin.name) + self.writestream = AsyncFile(sock, sys.stdout.mode, sys.stdout.name) + self.errorstream = AsyncFile(sock, sys.stderr.mode, sys.stderr.name) + + if redirect: + sys.stdin = self.readstream + sys.stdout = self.writestream + sys.stderr = self.errorstream + self.redirect = redirect + + # attach to the main thread here + self.attachThread(mainThread=True) + + def __unhandled_exception(self, exctype, excval, exctb): + """ + Private method called to report an uncaught exception. + + @param exctype the type of the exception + @param excval data about the exception + @param exctb traceback for the exception + """ + self.mainThread.user_exception(None, (exctype, excval, exctb), True) + + def __interceptSignals(self): + """ + Private method to intercept common signals. + """ + for signum in [ + signal.SIGABRT, # abnormal termination + signal.SIGFPE, # floating point exception + signal.SIGILL, # illegal instruction + signal.SIGSEGV, # segmentation violation + ]: + signal.signal(signum, self.__signalHandler) + + def __signalHandler(self, signalNumber, stackFrame): + """ + Private method to handle signals. + + @param signalNumber number of the signal to be handled + @type int + @param stackFrame current stack frame + @type frame object + """ + if signalNumber == signal.SIGABRT: + message = "Abnormal Termination" + elif signalNumber == signal.SIGFPE: + message = "Floating Point Exception" + elif signalNumber == signal.SIGILL: + message = "Illegal Instruction" + elif signalNumber == signal.SIGSEGV: + message = "Segmentation Violation" + else: + message = "Unknown Signal '{0}'".format(signalNumber) + + filename = self.absPath(stackFrame) + + linenr = stackFrame.f_lineno + ffunc = stackFrame.f_code.co_name + + if ffunc == '?': + ffunc = '' + + if ffunc and not ffunc.startswith("<"): + argInfo = getargvalues(stackFrame) + try: + fargs = formatargvalues( + argInfo.args, argInfo.varargs, + argInfo.keywords, argInfo.locals) + except Exception: + fargs = "" + else: + fargs = "" + + self.sendJsonCommand("ResponseSignal", { + "message": message, + "filename": filename, + "linenumber": linenr, + "function": ffunc, + "arguments": fargs, + }) + + def absPath(self, fn): + """ + Public method to convert a filename to an absolute name. + + sys.path is used as a set of possible prefixes. The name stays + relative if a file could not be found. + + @param fn filename (string) + @return the converted filename (string) + """ + if os.path.isabs(fn): + return fn + + # Check the cache. + if fn in self._fncache: + return self._fncache[fn] + + # Search sys.path. + for p in sys.path: + afn = os.path.abspath(os.path.join(p, fn)) + nafn = os.path.normcase(afn) + + if os.path.exists(nafn): + self._fncache[fn] = afn + d = os.path.dirname(afn) + if (d not in sys.path) and (d not in self.dircache): + self.dircache.append(d) + return afn + + # Search the additional directory cache + for p in self.dircache: + afn = os.path.abspath(os.path.join(p, fn)) + nafn = os.path.normcase(afn) + + if os.path.exists(nafn): + self._fncache[fn] = afn + return afn + + # Nothing found. + return fn + + def getRunning(self): + """ + Public method to return the main script we are currently running. + + @return flag indicating a running debug session (boolean) + """ + return self.running + + def progTerminated(self, status, message=""): + """ + Public method to tell the debugger that the program has terminated. + + @param status return status + @type int + @param message status message + @type str + """ + if status is None: + status = 0 + elif not isinstance(status, int): + message = str(status) + status = 1 + + if self.running: + self.set_quit() + self.running = None + self.sendJsonCommand("ResponseExit", { + "status": status, + "message": message, + }) + + # reset coding + self.__coding = self.defaultCoding + + def __dumpVariables(self, frmnr, scope, filter): + """ + Private method to return the variables of a frame to the debug server. + + @param frmnr distance of frame reported on. 0 is the current frame + (int) + @param scope 1 to report global variables, 0 for local variables (int) + @param filter the indices of variable types to be filtered + (list of int) + """ + if self.currentThread is None: + return + + frmnr += self.currentThread.skipFrames + if scope == 0: + self.framenr = frmnr + + f = self.currentThread.getCurrentFrame() + + while f is not None and frmnr > 0: + f = f.f_back + frmnr -= 1 + + if f is None: + if scope: + dict = self.debugMod.__dict__ + else: + scope = -1 + elif scope: + dict = f.f_globals + elif f.f_globals is f.f_locals: + scope = -1 + else: + dict = f.f_locals + + varlist = [] + + if scope != -1: + keylist = dict.keys() + + vlist = self.__formatVariablesList(keylist, dict, scope, filter) + varlist.extend(vlist) + + self.sendJsonCommand("ResponseVariables", { + "scope": scope, + "variables": varlist, + }) + + def __dumpVariable(self, var, frmnr, scope, filter): + """ + Private method to return the variables of a frame to the debug server. + + @param var list encoded name of the requested variable + (list of strings) + @param frmnr distance of frame reported on. 0 is the current frame + (int) + @param scope 1 to report global variables, 0 for local variables (int) + @param filter the indices of variable types to be filtered + (list of int) + """ + if self.currentThread is None: + return + + frmnr += self.currentThread.skipFrames + f = self.currentThread.getCurrentFrame() + + while f is not None and frmnr > 0: + f = f.f_back + frmnr -= 1 + + if f is None: + if scope: + dict = self.debugMod.__dict__ + else: + scope = -1 + elif scope: + dict = f.f_globals + elif f.f_globals is f.f_locals: + scope = -1 + else: + dict = f.f_locals + + varlist = [] + + if scope != -1: + variable = dict + for attribute in var: + attribute = self.__extractIndicators(attribute)[0] + typeObject, typeName, typeStr, resolver = \ + DebugVariables.getType(variable) + if resolver: + variable = resolver.resolve(variable, attribute) + if variable is None: + break + + else: + break + + if variable is not None: + typeObject, typeName, typeStr, resolver = \ + DebugVariables.getType(variable) + if typeStr.startswith(("PyQt5.", "PyQt4.")): + vlist = self.__formatQtVariable(variable, typeName) + varlist.extend(vlist) + elif resolver: + dict = resolver.getDictionary(variable) + vlist = self.__formatVariablesList( + list(dict.keys()), dict, scope, filter) + varlist.extend(vlist) + + self.sendJsonCommand("ResponseVariable", { + "scope": scope, + "variable": var, + "variables": varlist, + }) + + def __extractIndicators(self, var): + """ + Private method to extract the indicator string from a variable text. + + @param var variable text + @type str + @return tuple containing the variable text without indicators and the + indicator string + @rtype tuple of two str + """ + for indicator in DebugClientBase.Indicators: + if var.endswith(indicator): + return var[:-len(indicator)], indicator + + return var, "" + + def __formatQtVariable(self, value, qttype): + """ + Private method to produce a formatted output of a simple Qt4/Qt5 type. + + @param value variable to be formatted + @param qttype type of the Qt variable to be formatted (string) + @return A tuple consisting of a list of formatted variables. Each + variable entry is a tuple of three elements, the variable name, + its type and value. + """ + varlist = [] + if qttype == 'QChar': + varlist.append( + ("", "QChar", "{0}".format(unichr(value.unicode())))) + varlist.append(("", "int", "{0:d}".format(value.unicode()))) + elif qttype == 'QByteArray': + varlist.append( + ("bytes", "QByteArray", "{0}".format(bytes(value))[2:-1])) + varlist.append( + ("hex", "QByteArray", "{0}".format(value.toHex())[2:-1])) + varlist.append( + ("base64", "QByteArray", "{0}".format(value.toBase64())[2:-1])) + varlist.append(("percent encoding", "QByteArray", + "{0}".format(value.toPercentEncoding())[2:-1])) + elif qttype == 'QString': + varlist.append(("", "QString", "{0}".format(value))) + elif qttype == 'QStringList': + for i in range(value.count()): + varlist.append( + ("{0:d}".format(i), "QString", "{0}".format(value[i]))) + elif qttype == 'QPoint': + varlist.append(("x", "int", "{0:d}".format(value.x()))) + varlist.append(("y", "int", "{0:d}".format(value.y()))) + elif qttype == 'QPointF': + varlist.append(("x", "float", "{0:g}".format(value.x()))) + varlist.append(("y", "float", "{0:g}".format(value.y()))) + elif qttype == 'QRect': + varlist.append(("x", "int", "{0:d}".format(value.x()))) + varlist.append(("y", "int", "{0:d}".format(value.y()))) + varlist.append(("width", "int", "{0:d}".format(value.width()))) + varlist.append(("height", "int", "{0:d}".format(value.height()))) + elif qttype == 'QRectF': + varlist.append(("x", "float", "{0:g}".format(value.x()))) + varlist.append(("y", "float", "{0:g}".format(value.y()))) + varlist.append(("width", "float", "{0:g}".format(value.width()))) + varlist.append(("height", "float", "{0:g}".format(value.height()))) + elif qttype == 'QSize': + varlist.append(("width", "int", "{0:d}".format(value.width()))) + varlist.append(("height", "int", "{0:d}".format(value.height()))) + elif qttype == 'QSizeF': + varlist.append(("width", "float", "{0:g}".format(value.width()))) + varlist.append(("height", "float", "{0:g}".format(value.height()))) + elif qttype == 'QColor': + varlist.append(("name", "str", "{0}".format(value.name()))) + r, g, b, a = value.getRgb() + varlist.append( + ("rgba", "int", + "{0:d}, {1:d}, {2:d}, {3:d}".format(r, g, b, a))) + h, s, v, a = value.getHsv() + varlist.append( + ("hsva", "int", + "{0:d}, {1:d}, {2:d}, {3:d}".format(h, s, v, a))) + c, m, y, k, a = value.getCmyk() + varlist.append( + ("cmyka", "int", + "{0:d}, {1:d}, {2:d}, {3:d}, {4:d}".format(c, m, y, k, a))) + elif qttype == 'QDate': + varlist.append(("", "QDate", "{0}".format(value.toString()))) + elif qttype == 'QTime': + varlist.append(("", "QTime", "{0}".format(value.toString()))) + elif qttype == 'QDateTime': + varlist.append(("", "QDateTime", "{0}".format(value.toString()))) + elif qttype == 'QDir': + varlist.append(("path", "str", "{0}".format(value.path()))) + varlist.append(("absolutePath", "str", + "{0}".format(value.absolutePath()))) + varlist.append(("canonicalPath", "str", + "{0}".format(value.canonicalPath()))) + elif qttype == 'QFile': + varlist.append(("fileName", "str", "{0}".format(value.fileName()))) + elif qttype == 'QFont': + varlist.append(("family", "str", "{0}".format(value.family()))) + varlist.append( + ("pointSize", "int", "{0:d}".format(value.pointSize()))) + varlist.append(("weight", "int", "{0:d}".format(value.weight()))) + varlist.append(("bold", "bool", "{0}".format(value.bold()))) + varlist.append(("italic", "bool", "{0}".format(value.italic()))) + elif qttype == 'QUrl': + varlist.append(("url", "str", "{0}".format(value.toString()))) + varlist.append(("scheme", "str", "{0}".format(value.scheme()))) + varlist.append(("user", "str", "{0}".format(value.userName()))) + varlist.append(("password", "str", "{0}".format(value.password()))) + varlist.append(("host", "str", "{0}".format(value.host()))) + varlist.append(("port", "int", "{0:d}".format(value.port()))) + varlist.append(("path", "str", "{0}".format(value.path()))) + elif qttype == 'QModelIndex': + varlist.append(("valid", "bool", "{0}".format(value.isValid()))) + if value.isValid(): + varlist.append(("row", "int", "{0}".format(value.row()))) + varlist.append(("column", "int", "{0}".format(value.column()))) + varlist.append( + ("internalId", "int", "{0}".format(value.internalId()))) + varlist.append(("internalPointer", "void *", + "{0}".format(value.internalPointer()))) + elif qttype == 'QRegExp': + varlist.append(("pattern", "str", "{0}".format(value.pattern()))) + + # GUI stuff + elif qttype == 'QAction': + varlist.append(("name", "str", "{0}".format(value.objectName()))) + varlist.append(("text", "str", "{0}".format(value.text()))) + varlist.append( + ("icon text", "str", "{0}".format(value.iconText()))) + varlist.append(("tooltip", "str", "{0}".format(value.toolTip()))) + varlist.append( + ("whatsthis", "str", "{0}".format(value.whatsThis()))) + varlist.append( + ("shortcut", "str", + "{0}".format(value.shortcut().toString()))) + elif qttype == 'QKeySequence': + varlist.append(("value", "", "{0}".format(value.toString()))) + + # XML stuff + elif qttype == 'QDomAttr': + varlist.append(("name", "str", "{0}".format(value.name()))) + varlist.append(("value", "str", "{0}".format(value.value()))) + elif qttype == 'QDomCharacterData': + varlist.append(("data", "str", "{0}".format(value.data()))) + elif qttype == 'QDomComment': + varlist.append(("data", "str", "{0}".format(value.data()))) + elif qttype == "QDomDocument": + varlist.append(("text", "str", "{0}".format(value.toString()))) + elif qttype == 'QDomElement': + varlist.append(("tagName", "str", "{0}".format(value.tagName()))) + varlist.append(("text", "str", "{0}".format(value.text()))) + elif qttype == 'QDomText': + varlist.append(("data", "str", "{0}".format(value.data()))) + + # Networking stuff + elif qttype == 'QHostAddress': + varlist.append( + ("address", "QHostAddress", "{0}".format(value.toString()))) + + return varlist + + def __formatVariablesList(self, keylist, dict_, scope, filter=[], + formatSequences=False): + """ + Private method to produce a formated variables list. + + The dictionary passed in to it is scanned. Variables are + only added to the list, if their type is not contained + in the filter list and their name doesn't match any of the filter + expressions. The formated variables list (a list of tuples of 3 + values) is returned. + + @param keylist keys of the dictionary + @param dict_ the dictionary to be scanned + @param scope 1 to filter using the globals filter, 0 using the locals + filter (int). + Variables are only added to the list, if their name do not match + any of the filter expressions. + @param filter the indices of variable types to be filtered. Variables + are only added to the list, if their type is not contained in the + filter list. + @param formatSequences flag indicating, that sequence or dictionary + variables should be formatted. If it is 0 (or false), just the + number of items contained in these variables is returned. (boolean) + @return A tuple consisting of a list of formatted variables. Each + variable entry is a tuple of three elements, the variable name, + its type and value. + """ + varlist = [] + if scope: + patternFilterObjects = self.globalsFilterObjects + else: + patternFilterObjects = self.localsFilterObjects + + for key in keylist: + # filter based on the filter pattern + matched = False + for pat in patternFilterObjects: + if pat.match(str(key)): + matched = True + break + if matched: + continue + + # filter hidden attributes (filter #0) + if 0 in filter and str(key)[:2] == '__' and not ( + key == "___len___" and + DebugVariables.TooLargeAttribute in keylist): + continue + + # special handling for '__builtins__' (it's way too big) + if key == '__builtins__': + rvalue = '<module __builtin__ (built-in)>' + valtype = 'module' + else: + value = dict_[key] + valtypestr = str(type(value))[1:-1] + _, valtype = valtypestr.split(' ', 1) + valtype = valtype[1:-1] + valtypename = type(value).__name__ + if valtype not in ConfigVarTypeStrings: + if valtype in ["numpy.ndarray", "array.array"]: + if ConfigVarTypeStrings.index('list') in filter: + continue + elif valtypename == "MultiValueDict": + if ConfigVarTypeStrings.index('dict') in filter: + continue + elif valtype == "sip.methoddescriptor": + if ConfigVarTypeStrings.index( + 'method') in filter: + continue + elif valtype == "sip.enumtype": + if ConfigVarTypeStrings.index('class') in filter: + continue + elif ConfigVarTypeStrings.index('instance') in filter: + continue + + if valtypename not in ["ndarray", "MultiValueDict", + "array"]: + valtype = valtypestr + else: + try: + if ConfigVarTypeStrings.index(valtype) in filter: + continue + except ValueError: + if valtype == "classobj": + if ConfigVarTypeStrings.index( + 'instance') in filter: + continue + elif valtype == "sip.methoddescriptor": + if ConfigVarTypeStrings.index( + 'method') in filter: + continue + elif valtype == "sip.enumtype": + if ConfigVarTypeStrings.index('class') in filter: + continue + elif not valtype.startswith("PySide") and \ + ConfigVarTypeStrings.index('other') in filter: + continue + + try: + if valtype in ['list', 'tuple', 'dict', 'set', + 'frozenset', 'array.array']: + if valtype == 'dict': + rvalue = "{0:d}".format(len(value.keys())) + else: + rvalue = "{0:d}".format(len(value)) + elif valtype == "numpy.ndarray": + rvalue = "{0:d}".format(value.size) + elif valtypename == "MultiValueDict": + rvalue = "{0:d}".format(len(value.keys())) + valtype = "django.MultiValueDict" # shortened type + else: + rvalue = repr(value) + if valtype.startswith('class') and \ + rvalue[0] in ['{', '(', '[']: + rvalue = "" + except Exception: + rvalue = '' + + if formatSequences: + if str(key) == key: + key = "'{0!s}'".format(key) + else: + key = str(key) + varlist.append((key, valtype, rvalue)) + + return varlist + + def __generateFilterObjects(self, scope, filterString): + """ + Private slot to convert a filter string to a list of filter objects. + + @param scope 1 to generate filter for global variables, 0 for local + variables (int) + @param filterString string of filter patterns separated by ';' + """ + patternFilterObjects = [] + for pattern in filterString.split(';'): + patternFilterObjects.append(re.compile('^{0}$'.format(pattern))) + if scope: + self.globalsFilterObjects = patternFilterObjects[:] + else: + self.localsFilterObjects = patternFilterObjects[:] + + def __completionList(self, text): + """ + Private slot to handle the request for a commandline completion list. + + @param text the text to be completed (string) + """ + completerDelims = ' \t\n`~!@#$%^&*()-=+[{]}\\|;:\'",<>/?' + + completions = set() + # find position of last delim character + pos = -1 + while pos >= -len(text): + if text[pos] in completerDelims: + if pos == -1: + text = '' + else: + text = text[pos + 1:] + break + pos -= 1 + + # Get local and global completions + try: + localdict = self.currentThread.getFrameLocals(self.framenr) + localCompleter = Completer(localdict).complete + self.__getCompletionList(text, localCompleter, completions) + except AttributeError: + pass + self.__getCompletionList(text, self.complete, completions) + + self.sendJsonCommand("ResponseCompletion", { + "completions": list(completions), + "text": text, + }) + + def __getCompletionList(self, text, completer, completions): + """ + Private method to create a completions list. + + @param text text to complete (string) + @param completer completer methode + @param completions set where to add new completions strings (set) + """ + state = 0 + try: + comp = completer(text, state) + except Exception: + comp = None + while comp is not None: + completions.add(comp) + state += 1 + try: + comp = completer(text, state) + except Exception: + comp = None + + def startDebugger(self, filename=None, host=None, port=None, + enableTrace=True, exceptions=True, tracePython=False, + redirect=True): + """ + Public method used to start the remote debugger. + + @param filename the program to be debugged (string) + @param host hostname of the debug server (string) + @param port portnumber of the debug server (int) + @param enableTrace flag to enable the tracing function (boolean) + @param exceptions flag to enable exception reporting of the IDE + (boolean) + @param tracePython flag to enable tracing into the Python library + (boolean) + @param redirect flag indicating redirection of stdin, stdout and + stderr (boolean) + """ + global debugClient + if host is None: + host = os.getenv('ERICHOST', 'localhost') + if port is None: + port = os.getenv('ERICPORT', 42424) + + remoteAddress = self.__resolveHost(host) + self.connectDebugger(port, remoteAddress, redirect) + if filename is not None: + self.running = os.path.abspath(filename) + else: + try: + self.running = os.path.abspath(sys.argv[0]) + except IndexError: + self.running = None + if self.running: + self.__setCoding(self.running) + self.passive = True + self.sendPassiveStartup(self.running, exceptions) + self.__interact() + + # setup the debugger variables + self._fncache = {} + self.dircache = [] + self.debugging = True + + self.attachThread(mainThread=True) + self.mainThread.tracePythonLibs(tracePython) + + # set the system exception handling function to ensure, that + # we report on all unhandled exceptions + sys.excepthook = self.__unhandled_exception + self.__interceptSignals() + + # now start debugging + if enableTrace: + self.mainThread.set_trace() + + def startProgInDebugger(self, progargs, wd='', host=None, + port=None, exceptions=True, tracePython=False, + redirect=True): + """ + Public method used to start the remote debugger. + + @param progargs commandline for the program to be debugged + (list of strings) + @param wd working directory for the program execution (string) + @param host hostname of the debug server (string) + @param port portnumber of the debug server (int) + @param exceptions flag to enable exception reporting of the IDE + (boolean) + @param tracePython flag to enable tracing into the Python library + (boolean) + @param redirect flag indicating redirection of stdin, stdout and + stderr (boolean) + """ + if host is None: + host = os.getenv('ERICHOST', 'localhost') + if port is None: + port = os.getenv('ERICPORT', 42424) + + remoteAddress = self.__resolveHost(host) + self.connectDebugger(port, remoteAddress, redirect) + + self._fncache = {} + self.dircache = [] + sys.argv = progargs[:] + sys.argv[0] = os.path.abspath(sys.argv[0]) + sys.path = self.__getSysPath(os.path.dirname(sys.argv[0])) + if wd == '': + os.chdir(sys.path[1]) + else: + os.chdir(wd) + self.running = sys.argv[0] + self.__setCoding(self.running) + self.debugging = True + + self.passive = True + self.sendPassiveStartup(self.running, exceptions) + self.__interact() + + self.attachThread(mainThread=True) + self.mainThread.tracePythonLibs(tracePython) + + # set the system exception handling function to ensure, that + # we report on all unhandled exceptions + sys.excepthook = self.__unhandled_exception + self.__interceptSignals() + + # This will eventually enter a local event loop. + # Note the use of backquotes to cause a repr of self.running. The + # need for this is on Windows os where backslash is the path separator. + # They will get inadvertantly stripped away during the eval causing + # IOErrors if self.running is passed as a normal str. + self.debugMod.__dict__['__file__'] = self.running + sys.modules['__main__'] = self.debugMod + res = self.mainThread.run( + 'exec(open(' + repr(self.running) + ').read())', + self.debugMod.__dict__) + self.progTerminated(res) + + def run_call(self, scriptname, func, *args): + """ + Public method used to start the remote debugger and call a function. + + @param scriptname name of the script to be debugged (string) + @param func function to be called + @param *args arguments being passed to func + @return result of the function call + """ + self.startDebugger(scriptname, enableTrace=False) + res = self.mainThread.runcall(func, *args) + self.progTerminated(res) + return res + + def __resolveHost(self, host): + """ + Private method to resolve a hostname to an IP address. + + @param host hostname of the debug server (string) + @return IP address (string) + """ + try: + host, version = host.split("@@") + except ValueError: + version = 'v4' + if version == 'v4': + family = socket.AF_INET + else: + family = socket.AF_INET6 + return socket.getaddrinfo(host, None, family, + socket.SOCK_STREAM)[0][4][0] + + def main(self): + """ + Public method implementing the main method. + """ + if '--' in sys.argv: + args = sys.argv[1:] + host = None + port = None + wd = '' + tracePython = False + exceptions = True + redirect = True + while args[0]: + if args[0] == '-h': + host = args[1] + del args[0] + del args[0] + elif args[0] == '-p': + port = int(args[1]) + del args[0] + del args[0] + elif args[0] == '-w': + wd = args[1] + del args[0] + del args[0] + elif args[0] == '-t': + tracePython = True + del args[0] + elif args[0] == '-e': + exceptions = False + del args[0] + elif args[0] == '-n': + redirect = False + del args[0] + elif args[0] == '--no-encoding': + self.noencoding = True + del args[0] + elif args[0] == '--fork-child': + self.fork_auto = True + self.fork_child = True + del args[0] + elif args[0] == '--fork-parent': + self.fork_auto = True + self.fork_child = False + del args[0] + elif args[0] == '--': + del args[0] + break + else: # unknown option + del args[0] + if not args: + print("No program given. Aborting!") # __IGNORE_WARNING__ + else: + if not self.noencoding: + self.__coding = self.defaultCoding + self.startProgInDebugger(args, wd, host, port, + exceptions=exceptions, + tracePython=tracePython, + redirect=redirect) + else: + if sys.argv[1] == '--no-encoding': + self.noencoding = True + del sys.argv[1] + if sys.argv[1] == '': + del sys.argv[1] + try: + port = int(sys.argv[1]) + except (ValueError, IndexError): + port = -1 + try: + redirect = int(sys.argv[2]) + except (ValueError, IndexError): + redirect = True + try: + ipOrHost = sys.argv[3] + if ':' in ipOrHost: + remoteAddress = ipOrHost + elif ipOrHost[0] in '0123456789': + remoteAddress = ipOrHost + else: + remoteAddress = self.__resolveHost(ipOrHost) + except Exception: + remoteAddress = None + sys.argv = [''] + if '' not in sys.path: + sys.path.insert(0, '') + if port >= 0: + if not self.noencoding: + self.__coding = self.defaultCoding + self.connectDebugger(port, remoteAddress, redirect) + self.__interact() + else: + print("No network port given. Aborting...") # __IGNORE_WARNING__ + + def fork(self): + """ + Public method implementing a fork routine deciding which branch + to follow. + + @return process ID (integer) + """ + if not self.fork_auto: + self.sendJsonCommand("RequestForkTo", {}) + self.eventLoop(True) + pid = DebugClientOrigFork() + if pid == 0: + # child + if not self.fork_child: + sys.settrace(None) + sys.setprofile(None) + self.sessionClose(0) + else: + # parent + if self.fork_child: + sys.settrace(None) + sys.setprofile(None) + self.sessionClose(0) + return pid + + def close(self, fd): + """ + Public method implementing a close method as a replacement for + os.close(). + + It prevents the debugger connections from being closed. + + @param fd file descriptor to be closed (integer) + """ + if fd in [self.readstream.fileno(), self.writestream.fileno(), + self.errorstream.fileno()]: + return + + DebugClientOrigClose(fd) + + def __getSysPath(self, firstEntry): + """ + Private slot to calculate a path list including the PYTHONPATH + environment variable. + + @param firstEntry entry to be put first in sys.path (string) + @return path list for use as sys.path (list of strings) + """ + sysPath = [path for path in os.environ.get("PYTHONPATH", "") + .split(os.pathsep) + if path not in sys.path] + sys.path[:] + if "" in sysPath: + sysPath.remove("") + sysPath.insert(0, firstEntry) + sysPath.insert(0, '') + return sysPath + +# +# eflag: noqa = M702
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/DebugClients/Python/DebugClientCapabilities.py Sun Oct 16 12:18:42 2016 +0200 @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- + +# Copyright (c) 2005 - 2016 Detlev Offenbach <detlev@die-offenbachs.de> +# + +""" +Module defining the debug clients capabilities. +""" + +HasDebugger = 0x0001 +HasInterpreter = 0x0002 +HasProfiler = 0x0004 +HasCoverage = 0x0008 +HasCompleter = 0x0010 +HasUnittest = 0x0020 +HasShell = 0x0040 + +HasAll = HasDebugger | HasInterpreter | HasProfiler | \ + HasCoverage | HasCompleter | HasUnittest | HasShell + +# +# eflag: noqa = M702
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/DebugClients/Python/DebugConfig.py Sun Oct 16 12:18:42 2016 +0200 @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- + +# Copyright (c) 2005 - 2016 Detlev Offenbach <detlev@die-offenbachs.de> +# + +""" +Module defining type strings for the different Python types. +""" + +# +# Keep this list in sync with Debugger.Config.ConfigVarTypeFilters +# +ConfigVarTypeStrings = [ + '__', 'NoneType', 'type', + 'bool', 'int', 'long', 'float', 'complex', + 'str', 'unicode', 'tuple', 'list', + 'dict', 'dict-proxy', 'set', 'file', 'xrange', + 'slice', 'buffer', 'class', 'instance', + 'method', 'property', 'generator', + 'function', 'builtin_function_or_method', 'code', 'module', + 'ellipsis', 'traceback', 'frame', 'other', 'frozenset' +] + +# +# eflag: noqa = M702
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/DebugClients/Python/DebugUtilities.py Sun Oct 16 12:18:42 2016 +0200 @@ -0,0 +1,149 @@ +# -*- coding: utf-8 -*- + +# Copyright (c) 2015 - 2016 Detlev Offenbach <detlev@die-offenbachs.de> +# + +""" +Module implementing utilities functions for the debug client. +""" + +# +# Taken from inspect.py of Python 3.4 +# + +from collections import namedtuple +from inspect import iscode, isframe + +# Create constants for the compiler flags in Include/code.h +# We try to get them from dis to avoid duplication, but fall +# back to hardcoding so the dependency is optional +try: + from dis import COMPILER_FLAG_NAMES +except ImportError: + CO_OPTIMIZED, CO_NEWLOCALS = 0x1, 0x2 + CO_VARARGS, CO_VARKEYWORDS = 0x4, 0x8 + CO_NESTED, CO_GENERATOR, CO_NOFREE = 0x10, 0x20, 0x40 +else: + mod_dict = globals() + for k, v in COMPILER_FLAG_NAMES.items(): + mod_dict["CO_" + v] = k + +ArgInfo = namedtuple('ArgInfo', 'args varargs keywords locals') + + +def getargvalues(frame): + """ + Function to get information about arguments passed into a + particular frame. + + @param frame reference to a frame object to be processed + @type frame + @return tuple of four things, where 'args' is a list of the argument names, + 'varargs' and 'varkw' are the names of the * and ** arguments or None + and 'locals' is the locals dictionary of the given frame. + @exception TypeError raised if the input parameter is not a frame object + """ + if not isframe(frame): + raise TypeError('{0!r} is not a frame object'.format(frame)) + + args, varargs, kwonlyargs, varkw = _getfullargs(frame.f_code) + return ArgInfo(args + kwonlyargs, varargs, varkw, frame.f_locals) + + +def _getfullargs(co): + """ + Protected function to get information about the arguments accepted + by a code object. + + @param co reference to a code object to be processed + @type code + @return tuple of four things, where 'args' and 'kwonlyargs' are lists of + argument names, and 'varargs' and 'varkw' are the names of the + * and ** arguments or None. + @exception TypeError raised if the input parameter is not a code object + """ + if not iscode(co): + raise TypeError('{0!r} is not a code object'.format(co)) + + nargs = co.co_argcount + names = co.co_varnames + nkwargs = co.co_kwonlyargcount + args = list(names[:nargs]) + kwonlyargs = list(names[nargs:nargs + nkwargs]) + + nargs += nkwargs + varargs = None + if co.co_flags & CO_VARARGS: + varargs = co.co_varnames[nargs] + nargs = nargs + 1 + varkw = None + if co.co_flags & CO_VARKEYWORDS: + varkw = co.co_varnames[nargs] + return args, varargs, kwonlyargs, varkw + + +def formatargvalues(args, varargs, varkw, locals, + formatarg=str, + formatvarargs=lambda name: '*' + name, + formatvarkw=lambda name: '**' + name, + formatvalue=lambda value: '=' + repr(value)): + """ + Function to format an argument spec from the 4 values returned + by getargvalues. + + @param args list of argument names + @type list of str + @param varargs name of the variable arguments + @type str + @param varkw name of the keyword arguments + @type str + @param locals reference to the local variables dictionary + @type dict + @keyparam formatarg argument formatting function + @type func + @keyparam formatvarargs variable arguments formatting function + @type func + @keyparam formatvarkw keyword arguments formatting function + @type func + @keyparam formatvalue value formating functtion + @type func + @return formatted call signature + @rtype str + """ + specs = [] + for i in range(len(args)): + name = args[i] + specs.append(formatarg(name) + formatvalue(locals[name])) + if varargs: + specs.append(formatvarargs(varargs) + formatvalue(locals[varargs])) + if varkw: + specs.append(formatvarkw(varkw) + formatvalue(locals[varkw])) + argvalues = '(' + ', '.join(specs) + ')' + if '__return__' in locals: + argvalues += " -> " + formatvalue(locals['__return__']) + return argvalues + + +def prepareJsonCommand(method, params): + """ + Function to prepare a single command or response for transmission to + the IDE. + + @param method command or response name to be sent + @type str + @param params dictionary of named parameters for the command or response + @type dict + @return prepared JSON command or response string + @rtype str + """ + import json + + commandDict = { + "jsonrpc": "2.0", + "method": method, + "params": params, + } + return json.dumps(commandDict) + '\n' + +# +# eflag: noqa = M702
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/DebugClients/Python/DebugVariables.py Sun Oct 16 12:18:42 2016 +0200 @@ -0,0 +1,666 @@ +# -*- coding: utf-8 -*- + +# Copyright (c) 2016 Detlev Offenbach <detlev@die-offenbachs.de> +# + +""" +Module implementing classes and functions to dump variable contents. +""" + +# +# This code was inspired by pydevd. +# + +MaxItemsToHandle = 300 +TooLargeMessage = ("Too large to show contents. Max items to show: " + + str(MaxItemsToHandle)) +TooLargeAttribute = "Too large to be handled." + +############################################################ +## Classes implementing resolvers for various compund types +############################################################ + + +class BaseResolver(object): + """ + Base class of the resolver class tree. + """ + def resolve(self, var, attribute): + """ + Public method to get an attribute from a variable. + + @param var variable to extract an attribute or value from + @type any + @param attribute name of the attribute to extract + @type str + @return value of the attribute + @rtype any + @exception NotImplementedError raised to indicate a missing + implementation + """ # __IGNORE_WARNING_D235__ + raise NotImplementedError + + def getDictionary(self, var): + """ + Public method to get the attributes of a variable as a dictionary. + + @param var variable to be converted + @type any + @return dictionary containing the variable attributes + @rtype dict + @exception NotImplementedError raised to indicate a missing + implementation + """ # __IGNORE_WARNING_D235__ + raise NotImplementedError + + +############################################################ +## Default Resolver +############################################################ + + +class DefaultResolver(BaseResolver): + """ + Class used to resolve the default way. + """ + def resolve(self, var, attribute): + """ + Public method to get an attribute from a variable. + + @param var variable to extract an attribute or value from + @type any + @param attribute name of the attribute to extract + @type str + @return value of the attribute + @rtype any + """ + return getattr(var, attribute, None) + + def getDictionary(self, var): + """ + Public method to get the attributes of a variable as a dictionary. + + @param var variable to be converted + @type any + @return dictionary containing the variable attributes + @rtype dict + """ + names = dir(var) + if not names and hasattr(var, "__members__"): + names = var.__members__ + + d = {} + for name in names: + try: + attribute = getattr(var, name) + d[name] = attribute + except Exception: + pass # if we can't get it, simply ignore it + + return d + + +############################################################ +## Resolver for Dictionaries +############################################################ + + +class DictResolver(BaseResolver): + """ + Class used to resolve from a dictionary. + """ + def resolve(self, var, attribute): + """ + Public method to get an attribute from a variable. + + @param var variable to extract an attribute or value from + @type dict + @param attribute name of the attribute to extract + @type str + @return value of the attribute + @rtype any + """ + if attribute in ('___len___', TooLargeAttribute): + return None + + if "(ID:" not in attribute: + try: + return var[attribute] + except Exception: + return getattr(var, attribute, None) + + expectedID = int(attribute.split("(ID:")[-1][:-1]) + for key, value in var.items(): + if id(key) == expectedID: + return value + + return None + + def keyToStr(self, key): + """ + Public method to get a string representation for a key. + + @param key key to be converted + @type any + @return string representation of the given key + @rtype str + """ + if isinstance(key, str): + return repr(key) + else: + return key + + def getDictionary(self, var): + """ + Public method to get the attributes of a variable as a dictionary. + + @param var variable to be converted + @type any + @return dictionary containing the variable attributes + @rtype dict + """ + d = {} + count = 0 + for key, value in var.items(): + count += 1 + key = "{0} (ID:{1})".format(self.keyToStr(key), id(key)) + d[key] = value + if count > MaxItemsToHandle: + d[TooLargeAttribute] = TooLargeMessage + break + + d["___len___"] = len(var) + + # in case it has additional fields + additionals = defaultResolver.getDictionary(var) + d.update(additionals) + + return d + + +############################################################ +## Resolver for Lists and Tuples +############################################################ + + +class ListResolver(BaseResolver): + """ + Class used to resolve from a tuple or list. + """ + def resolve(self, var, attribute): + """ + Public method to get an attribute from a variable. + + @param var variable to extract an attribute or value from + @type tuple or list + @param attribute name of the attribute to extract + @type str + @return value of the attribute + @rtype any + """ + if attribute in ('___len___', TooLargeAttribute): + return None + + try: + return var[int(attribute)] + except Exception: + return getattr(var, attribute, None) + + def getDictionary(self, var): + """ + Public method to get the attributes of a variable as a dictionary. + + @param var variable to be converted + @type any + @return dictionary containing the variable attributes + @rtype dict + """ + d = {} + count = 0 + for value in var: + d[str(count)] = value + count += 1 + if count > MaxItemsToHandle: + d[TooLargeAttribute] = TooLargeMessage + break + + d["___len___"] = len(var) + + # in case it has additional fields + additionals = defaultResolver.getDictionary(var) + d.update(additionals) + + return d + + +############################################################ +## Resolver for Sets and Frozensets +############################################################ + + +class SetResolver(BaseResolver): + """ + Class used to resolve from a set or frozenset. + """ + def resolve(self, var, attribute): + """ + Public method to get an attribute from a variable. + + @param var variable to extract an attribute or value from + @type tuple or list + @param attribute id of the value to extract + @type str + @return value of the attribute + @rtype any + """ + if attribute in ('___len___', TooLargeAttribute): + return None + + if attribute.startswith("ID: "): + attribute = attribute.split(None, 1)[1] + try: + attribute = int(attribute) + except Exception: + return getattr(var, attribute, None) + + for v in var: + if id(v) == attribute: + return v + + return None + + def getDictionary(self, var): + """ + Public method to get the attributes of a variable as a dictionary. + + @param var variable to be converted + @type any + @return dictionary containing the variable attributes + @rtype dict + """ + d = {} + count = 0 + for value in var: + count += 1 + d["ID: " + str(id(value))] = value + if count > MaxItemsToHandle: + d[TooLargeAttribute] = TooLargeMessage + break + + d["___len___"] = len(var) + + # in case it has additional fields + additionals = defaultResolver.getDictionary(var) + d.update(additionals) + + return d + + +############################################################ +## Resolver for Numpy Arrays +############################################################ + + +class NdArrayResolver(BaseResolver): + """ + Class used to resolve from numpy ndarray including some meta data. + """ + def __isNumeric(self, arr): + """ + Private method to check, if an array is of a numeric type. + + @param arr array to check + @type ndarray + @return flag indicating a numeric array + @rtype bool + """ + try: + return arr.dtype.kind in 'biufc' + except AttributeError: + return False + + def resolve(self, var, attribute): + """ + Public method to get an attribute from a variable. + + @param var variable to extract an attribute or value from + @type tuple or list + @param attribute id of the value to extract + @type str + @return value of the attribute + @rtype any + """ + if attribute == '__internals__': + return defaultResolver.getDictionary(var) + + if attribute == 'min': + if self.__isNumeric(var): + return var.min() + else: + return None + + if attribute == 'max': + if self.__isNumeric(var): + return var.max() + else: + return None + + if attribute == 'mean': + if self.__isNumeric(var): + return var.mean() + else: + return None + + if attribute == 'shape': + return var.shape + + if attribute == 'dtype': + return var.dtype + + if attribute == 'size': + return var.size + + if attribute.startswith('['): + container = NdArrayItemsContainer() + count = 0 + for element in var: + setattr(container, str(count), element) + count += 1 + if count > MaxItemsToHandle: + setattr(container, TooLargeAttribute, TooLargeMessage) + break + return container + + return None + + def getDictionary(self, var): + """ + Public method to get the attributes of a variable as a dictionary. + + @param var variable to be converted + @type any + @return dictionary containing the variable attributes + @rtype dict + """ + d = {} + d['__internals__'] = defaultResolver.getDictionary(var) + if var.size > 1024 * 1024: + d['min'] = 'ndarray too big, calculating min would slow down' \ + ' debugging' + d['max'] = 'ndarray too big, calculating max would slow down' \ + ' debugging' + else: + if self.__isNumeric(var): + d['min'] = var.min() + d['max'] = var.max() + d['mean'] = var.mean() + else: + d['min'] = 'not a numeric object' + d['max'] = 'not a numeric object' + d['mean'] = 'not a numeric object' + d['shape'] = var.shape + d['dtype'] = var.dtype + d['size'] = var.size + d['[0:{0}]'.format(len(var) - 1)] = list(var[0:MaxItemsToHandle]) + return d + + +class NdArrayItemsContainer: + """ + Class to store ndarray items. + """ + pass + + +############################################################ +## Resolver for Django Multi Value Dictionaries +############################################################ + + +class MultiValueDictResolver(DictResolver): + """ + Class used to resolve from Django multi value dictionaries. + """ + def resolve(self, var, attribute): + """ + Public method to get an attribute from a variable. + + @param var variable to extract an attribute or value from + @type dict + @param attribute name of the attribute to extract + @type str + @return value of the attribute + @rtype any + """ + if attribute in ('___len___', TooLargeAttribute): + return None + + if "(ID:" not in attribute: + try: + return var[attribute] + except Exception: + return getattr(var, attribute, None) + + expectedID = int(attribute.split("(ID:")[-1][:-1]) + for key in var.keys(): + if id(key) == expectedID: + value = var.getlist(key) + return value + + return None + + def getDictionary(self, var): + """ + Public method to get the attributes of a variable as a dictionary. + + @param var variable to be converted + @type any + @return dictionary containing the variable attributes + @rtype dict + """ + d = {} + count = 0 + for key in var.keys(): + count += 1 + value = var.getlist(key) + key = "{0} (ID:{1})".format(self.keyToStr(key), id(key)) + d[key] = value + if count > MaxItemsToHandle: + d[TooLargeAttribute] = TooLargeMessage + break + + d["___len___"] = len(var) + + return d + + +############################################################ +## Resolver for array.array +############################################################ + + +class ArrayResolver(BaseResolver): + """ + Class used to resolve from array.array including some meta data. + """ + TypeCodeMap = { + "b": "int (signed char)", + "B": "int (unsigned char)", + "u": "Unicode character (Py_UNICODE)", + "h": "int (signed short)", + "H": "int (unsigned short)", + "i": "int (signed int)", + "I": "int (unsigned int)", + "l": "int (signed long)", + "L": "int (unsigned long)", + "q": "int (signed long long)", + "Q": "int (unsigned long long)", + "f": "float (float)", + "d": "float (double)", + } + def resolve(self, var, attribute): + """ + Public method to get an attribute from a variable. + + @param var variable to extract an attribute or value from + @type tuple or list + @param attribute id of the value to extract + @type str + @return value of the attribute + @rtype any + """ + if attribute == 'itemsize': + return var.itemsize + + if attribute == 'typecode': + return var.typecode + + if attribute == 'type': + if var.typecode in ArrayResolver.TypeCodeMap: + return ArrayResolver.TypeCodeMap[var.typecode] + else: + return 'illegal type' + + if attribute.startswith('['): + container = ArrayItemsContainer() + count = 0 + for element in var: + setattr(container, str(count), element) + count += 1 + if count > MaxItemsToHandle: + setattr(container, TooLargeAttribute, TooLargeMessage) + break + return container + + return None + + def getDictionary(self, var): + """ + Public method to get the attributes of a variable as a dictionary. + + @param var variable to be converted + @type any + @return dictionary containing the variable attributes + @rtype dict + """ + d = {} + d['typecode'] = var.typecode + if var.typecode in ArrayResolver.TypeCodeMap: + d['type'] = ArrayResolver.TypeCodeMap[var.typecode] + else: + d['type'] = 'illegal type' + d['itemsize'] = var.itemsize + d['[0:{0}]'.format(len(var) - 1)] = var.tolist()[0:MaxItemsToHandle] + return d + + +class ArrayItemsContainer: + """ + Class to store array.array items. + """ + pass + + +defaultResolver = DefaultResolver() +dictResolver = DictResolver() +listResolver = ListResolver() +setResolver = SetResolver() +ndarrayResolver = NdArrayResolver() +multiValueDictResolver = MultiValueDictResolver() +arrayResolver = ArrayResolver() + +############################################################ +## Methods to determine the type of a variable and the +## resolver class to use +############################################################ + +_TypeMap = None + + +def _initTypeMap(): + """ + Protected function to initialize the type map. + """ + global _TypeMap + + _TypeMap = [ + (type(None), None,), + (int, None), + (float, None), + (complex, None), + (str, None), + (tuple, listResolver), + (list, listResolver), + (dict, dictResolver), + ] + + try: + _TypeMap.append((long, None)) # __IGNORE_WARNING__ + except Exception: + pass # not available on all python versions + + try: + _TypeMap.append((unicode, None)) # __IGNORE_WARNING__ + except Exception: + pass # not available on all python versions + + try: + _TypeMap.append((set, setResolver)) # __IGNORE_WARNING__ + except Exception: + pass # not available on all python versions + + try: + _TypeMap.append((frozenset, setResolver)) # __IGNORE_WARNING__ + except Exception: + pass # not available on all python versions + + try: + import array + _TypeMap.append((array.array, arrayResolver)) + except ImportError: + pass # array.array may not be available + + try: + import numpy + _TypeMap.append((numpy.ndarray, ndarrayResolver)) + except ImportError: + pass # numpy may not be installed + + try: + from django.utils.datastructures import MultiValueDict + _TypeMap.insert(0, (MultiValueDict, multiValueDictResolver)) + # it should go before dict + except ImportError: + pass # django may not be installed + + +def getType(obj): + """ + Public method to get the type information for an object. + + @param obj object to get type information for + @type any + @return tuple containing the type, type name, type string and resolver + @rtype tuple of type, str, str, BaseResolver + """ + typeObject = type(obj) + typeName = typeObject.__name__ + typeStr = str(typeObject)[8:-2] + + if typeStr.startswith(("PyQt5.", "PyQt4.")): + resolver = None + else: + if _TypeMap is None: + _initTypeMap() + + for typeData in _TypeMap: + if isinstance(obj, typeData[0]): + resolver = typeData[1] + break + else: + resolver = defaultResolver + + return typeObject, typeName, typeStr, resolver + +# +# eflag: noqa = M702
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/DebugClients/Python/FlexCompleter.py Sun Oct 16 12:18:42 2016 +0200 @@ -0,0 +1,219 @@ +# -*- coding: utf-8 -*- + +""" +Word completion for the eric6 shell. + +<h4>NOTE for eric6 variant</h4> + + This version is a re-implementation of rlcompleter + as found in the Python3 library. It is modified to work with the eric6 + debug clients. + +<h4>Original rlcompleter documentation</h4> + + This requires the latest extension to the readline module. The completer + completes keywords, built-ins and globals in a selectable namespace (which + defaults to __main__); when completing NAME.NAME..., it evaluates (!) the + expression up to the last dot and completes its attributes. + + It's very cool to do "import sys" type "sys.", hit the + completion key (twice), and see the list of names defined by the + sys module! + + Tip: to use the tab key as the completion key, call + + readline.parse_and_bind("tab: complete") + + <b>Notes</b>: + <ul> + <li> + Exceptions raised by the completer function are *ignored* (and + generally cause the completion to fail). This is a feature -- since + readline sets the tty device in raw (or cbreak) mode, printing a + traceback wouldn't work well without some complicated hoopla to save, + reset and restore the tty state. + </li> + <li> + The evaluation of the NAME.NAME... form may cause arbitrary + application defined code to be executed if an object with a + __getattr__ hook is found. Since it is the responsibility of the + application (or the user) to enable this feature, I consider this an + acceptable risk. More complicated expressions (e.g. function calls or + indexing operations) are *not* evaluated. + </li> + <li> + When the original stdin is not a tty device, GNU readline is never + used, and this module (and the readline module) are silently inactive. + </li> + </ul> +""" + +try: + import __builtin__ as builtins +except ImportError: + import builtins + +import __main__ + +__all__ = ["Completer"] + + +class Completer(object): + """ + Class implementing the command line completer object. + """ + def __init__(self, namespace=None): + """ + Constructor + + Completer([namespace]) -> completer instance. + + If unspecified, the default namespace where completions are performed + is __main__ (technically, __main__.__dict__). Namespaces should be + given as dictionaries. + + Completer instances should be used as the completion mechanism of + readline via the set_completer() call: + + readline.set_completer(Completer(my_namespace).complete) + + @param namespace The namespace for the completer. + @exception TypeError raised to indicate a wrong data structure of + the namespace object + """ + if namespace and not isinstance(namespace, dict): + raise TypeError('namespace must be a dictionary') + + # Don't bind to namespace quite yet, but flag whether the user wants a + # specific namespace or to use __main__.__dict__. This will allow us + # to bind to __main__.__dict__ at completion time, not now. + if namespace is None: + self.use_main_ns = True + else: + self.use_main_ns = False + self.namespace = namespace + + def complete(self, text, state): + """ + Public method to return the next possible completion for 'text'. + + This is called successively with state == 0, 1, 2, ... until it + returns None. The completion should begin with 'text'. + + @param text The text to be completed. (string) + @param state The state of the completion. (integer) + @return The possible completions as a list of strings. + """ + if self.use_main_ns: + self.namespace = __main__.__dict__ + + if state == 0: + if "." in text: + self.matches = self.attr_matches(text) + else: + self.matches = self.global_matches(text) + try: + return self.matches[state] + except IndexError: + return None + + def _callable_postfix(self, val, word): + """ + Protected method to check for a callable. + + @param val value to check (object) + @param word word to ammend (string) + @return ammended word (string) + """ + if hasattr(val, '__call__'): + word = word + "(" + return word + + def global_matches(self, text): + """ + Public method to compute matches when text is a simple name. + + @param text The text to be completed. (string) + @return A list of all keywords, built-in functions and names currently + defined in self.namespace that match. + """ + import keyword + matches = [] + n = len(text) + for word in keyword.kwlist: + if word[:n] == text: + matches.append(word) + for nspace in [builtins.__dict__, self.namespace]: + for word, val in nspace.items(): + if word[:n] == text and word != "__builtins__": + matches.append(self._callable_postfix(val, word)) + return matches + + def attr_matches(self, text): + """ + Public method to compute matches when text contains a dot. + + Assuming the text is of the form NAME.NAME....[NAME], and is + evaluatable in self.namespace, it will be evaluated and its attributes + (as revealed by dir()) are used as possible completions. (For class + instances, class members are are also considered.) + + <b>WARNING</b>: this can still invoke arbitrary C code, if an object + with a __getattr__ hook is evaluated. + + @param text The text to be completed. (string) + @return A list of all matches. + """ + import re + + # Another option, seems to work great. Catches things like ''.<tab> + m = re.match(r"(\S+(\.\w+)*)\.(\w*)", text) + + if not m: + return + expr, attr = m.group(1, 3) + try: + thisobject = eval(expr, self.namespace) + except Exception: + return [] + + # get the content of the object, except __builtins__ + words = dir(thisobject) + if "__builtins__" in words: + words.remove("__builtins__") + + if hasattr(object, '__class__'): + words.append('__class__') + words = words + get_class_members(object.__class__) + matches = [] + n = len(attr) + for word in words: + try: + if word[:n] == attr and hasattr(thisobject, word): + val = getattr(thisobject, word) + word = self._callable_postfix( + val, "{0}.{1}".format(expr, word)) + matches.append(word) + except Exception: + # some badly behaved objects pollute dir() with non-strings, + # which cause the completion to fail. This way we skip the + # bad entries and can still continue processing the others. + pass + return matches + + +def get_class_members(klass): + """ + Module function to retrieve the class members. + + @param klass The class object to be analysed. + @return A list of all names defined in the class. + """ + ret = dir(klass) + if hasattr(klass, '__bases__'): + for base in klass.__bases__: + ret = ret + get_class_members(base) + return ret + +# +# eflag: noqa = M702, M111
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/DebugClients/Python/PyProfile.py Sun Oct 16 12:18:42 2016 +0200 @@ -0,0 +1,186 @@ +# -*- coding: utf-8 -*- + +# Copyright (c) 2002 - 2016 Detlev Offenbach <detlev@die-offenbachs.de> + +""" +Module defining additions to the standard Python profile.py. +""" + +import os +import marshal +import profile +import atexit +import pickle +import sys + + +class PyProfile(profile.Profile): + """ + Class extending the standard Python profiler with additional methods. + + This class extends the standard Python profiler by the functionality to + save the collected timing data in a timing cache, to restore these data + on subsequent calls, to store a profile dump to a standard filename and + to erase these caches. + """ + def __init__(self, basename, timer=None, bias=None): + """ + Constructor + + @param basename name of the script to be profiled (string) + @param timer function defining the timing calculation + @param bias calibration value (float) + """ + try: + profile.Profile.__init__(self, timer, bias) + except TypeError: + profile.Profile.__init__(self, timer) + + self.dispatch = self.__class__.dispatch + + basename = os.path.splitext(basename)[0] + self.profileCache = "{0}.profile".format(basename) + self.timingCache = "{0}.timings".format(basename) + + self.__restore() + atexit.register(self.save) + + def __restore(self): + """ + Private method to restore the timing data from the timing cache. + """ + if not os.path.exists(self.timingCache): + return + + try: + cache = open(self.timingCache, 'rb') + timings = marshal.load(cache) + if isinstance(timings, dict): + self.timings = timings + except Exception: + pass + finally: + cache.close() + + def save(self): + """ + Public method to store the collected profile data. + """ + # dump the raw timing data + try: + cache = open(self.timingCache, 'wb') + marshal.dump(self.timings, cache) + except Exception: + pass + finally: + cache.close() + + # dump the profile data + self.dump_stats(self.profileCache) + + def dump_stats(self, file): + """ + Public method to dump the statistics data. + + @param file name of the file to write to (string) + """ + try: + f = open(file, 'wb') + self.create_stats() + pickle.dump(self.stats, f, 2) + except (EnvironmentError, pickle.PickleError): + pass + finally: + f.close() + + def erase(self): + """ + Public method to erase the collected timing data. + """ + self.timings = {} + if os.path.exists(self.timingCache): + os.remove(self.timingCache) + + def fix_frame_filename(self, frame): + """ + Public method used to fixup the filename for a given frame. + + The logic employed here is that if a module was loaded + from a .pyc file, then the correct .py to operate with + should be in the same path as the .pyc. The reason this + logic is needed is that when a .pyc file is generated, the + filename embedded and thus what is readable in the code object + of the frame object is the fully qualified filepath when the + pyc is generated. If files are moved from machine to machine + this can break debugging as the .pyc will refer to the .py + on the original machine. Another case might be sharing + code over a network... This logic deals with that. + + @param frame the frame object + @return fixed up file name (string) + """ + if sys.version_info[0] == 2: + versionExt = '.py2' + else: + versionExt = '.py3' + + # get module name from __file__ + if not isinstance(frame, profile.Profile.fake_frame) and \ + '__file__' in frame.f_globals: + root, ext = os.path.splitext(frame.f_globals['__file__']) + if ext in ['.pyc', '.py', versionExt, '.pyo']: + fixedName = root + '.py' + if os.path.exists(fixedName): + return fixedName + + fixedName = root + versionExt + if os.path.exists(fixedName): + return fixedName + + return frame.f_code.co_filename + + def trace_dispatch_call(self, frame, t): + """ + Public method used to trace functions calls. + + This is a variant of the one found in the standard Python + profile.py calling fix_frame_filename above. + + @param frame reference to the call frame + @param t arguments + @return flag indicating a successful handling (boolean) + """ + if self.cur and frame.f_back is not self.cur[-2]: + rpt, rit, ret, rfn, rframe, rcur = self.cur + if not isinstance(rframe, profile.Profile.fake_frame): + assert rframe.f_back is frame.f_back, ("Bad call", rfn, + rframe, rframe.f_back, + frame, frame.f_back) + self.trace_dispatch_return(rframe, 0) + assert (self.cur is None or + frame.f_back is self.cur[-2]), ("Bad call", + self.cur[-3]) + fcode = frame.f_code + fn = (self.fix_frame_filename(frame), + fcode.co_firstlineno, fcode.co_name) + self.cur = (t, 0, 0, fn, frame, self.cur) + timings = self.timings + if fn in timings: + cc, ns, tt, ct, callers = timings[fn] + timings[fn] = cc, ns + 1, tt, ct, callers + else: + timings[fn] = 0, 0, 0, 0, {} + return 1 + + dispatch = { + "call": trace_dispatch_call, + "exception": profile.Profile.trace_dispatch_exception, + "return": profile.Profile.trace_dispatch_return, + "c_call": profile.Profile.trace_dispatch_c_call, + "c_exception": profile.Profile.trace_dispatch_return, + # the C function returned + "c_return": profile.Profile.trace_dispatch_return, + } + +# +# eflag: noqa = M702
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/DebugClients/Python/ThreadExtension.py Sun Oct 16 12:18:42 2016 +0200 @@ -0,0 +1,325 @@ +# -*- coding: utf-8 -*- + +# Copyright (c) 2014 - 2016 Detlev Offenbach <detlev@die-offenbachs.de> +# + +""" +Module implementing an import hook patching thread modules to get debugged too. +""" + +import os.path +import sys +import importlib + +if sys.version_info[0] == 2: + import thread as _thread +else: + import _thread + +import threading + +from DebugBase import DebugBase + + +class ThreadExtension(object): + """ + Class implementing the thread support for the debugger. + + Provides methods for intercepting thread creation, retriving the running + threads and their name and state. + """ + def __init__(self): + """ + Constructor + """ + self.threadNumber = 1 + self.enableImportHooks = True + self._original_start_new_thread = None + self._qtThread = None + + self.clientLock = threading.RLock() + + # dictionary of all threads running {id: DebugBase} + self.threads = {_thread.get_ident(): self} + + # the "current" thread, basically for variables view + self.currentThread = self + # the thread we are at a breakpoint continuing at next command + self.currentThreadExec = self + + # special objects representing the main scripts thread and frame + self.mainThread = self + + if sys.version_info[0] == 2: + self.threadModName = 'thread' + else: + self.threadModName = '_thread' + + # reset already imported thread module to apply hooks at next import + del sys.modules[self.threadModName] + del sys.modules['threading'] + + sys.meta_path.insert(0, self) + + # provide a hook to perform a hard breakpoint + # Use it like this: + # if hasattr(sys, 'breakpoint): sys.breakpoint() + sys.breakpoint = self.set_trace + + def attachThread(self, target=None, args=None, kwargs={}, + mainThread=False): + """ + Public method to setup a standard thread for DebugClient to debug. + + If mainThread is True, then we are attaching to the already + started mainthread of the app and the rest of the args are ignored. + + @param target the start function of the target thread (i.e. the user + code) + @param args arguments to pass to target + @param kwargs keyword arguments to pass to target + @param mainThread True, if we are attaching to the already + started mainthread of the app + @return identifier of the created thread + """ + if mainThread: + ident = _thread.get_ident() + name = 'MainThread' + newThread = self.mainThread + newThread.isMainThread = True + if self.debugging: + sys.setprofile(newThread.profile) + + else: + newThread = DebugBase(self) + ident = self._original_start_new_thread( + newThread.bootstrap, (target, args, kwargs)) + name = 'Thread-{0}'.format(self.threadNumber) + self.threadNumber += 1 + + newThread.id = ident + newThread.name = name + + self.threads[ident] = newThread + + return ident + + def threadTerminated(self, threadId): + """ + Public method called when a DebugThread has exited. + + @param threadId id of the DebugThread that has exited + @type int + """ + self.lockClient() + try: + del self.threads[threadId] + except KeyError: + pass + finally: + self.unlockClient() + + def lockClient(self, blocking=True): + """ + Public method to acquire the lock for this client. + + @param blocking flag to indicating a blocking lock + @type bool + @return flag indicating successful locking + @rtype bool + """ + if blocking: + self.clientLock.acquire() + else: + return self.clientLock.acquire(blocking) + + def unlockClient(self): + """ + Public method to release the lock for this client. + """ + try: + self.clientLock.release() + except AssertionError: + pass + + def setCurrentThread(self, id): + """ + Public method to set the current thread. + + @param id the id the current thread should be set to. + @type int + """ + try: + self.lockClient() + if id is None: + self.currentThread = None + else: + self.currentThread = self.threads.get(id) + finally: + self.unlockClient() + + def dumpThreadList(self): + """ + Public method to send the list of threads. + """ + self.updateThreadList() + threadList = [] + if len(self.threads) > 1: + currentId = _thread.get_ident() + # update thread names set by user (threading.setName) + threadNames = {t.ident: t.getName() for t in threading.enumerate()} + + for id, thd in self.threads.items(): + d = {"id": id} + try: + d["name"] = threadNames.get(id, thd.name) + d["broken"] = thd.isBroken + except Exception: + d["name"] = 'UnknownThread' + d["broken"] = False + + threadList.append(d) + else: + currentId = -1 + d = {"id": -1} + d["name"] = "MainThread" + d["broken"] = self.isBroken + threadList.append(d) + + self.sendJsonCommand("ResponseThreadList", { + "currentID": currentId, + "threadList": threadList, + }) + + def getExecutedFrame(self, frame): + """ + Public method to return the currently executed frame. + + @param frame the current frame + @type frame object + @return the frame which is excecuted (without debugger frames) + @rtype frame object + """ + # to get the currently executed frame, skip all frames belonging to the + # debugger + while frame is not None: + baseName = os.path.basename(frame.f_code.co_filename) + if not baseName.startswith( + ('DebugClientBase.py', 'DebugBase.py', 'AsyncIO.py', + 'ThreadExtension.py', 'threading.py')): + break + frame = frame.f_back + + return frame + + def updateThreadList(self): + """ + Public method to update the list of running threads. + """ + frames = sys._current_frames() + for id, frame in frames.items(): + # skip our own timer thread + if frame.f_code.co_name == '__eventPollTimer': + continue + + # Unknown thread + if id not in self.threads: + newThread = DebugBase(self) + name = 'Thread-{0}'.format(self.threadNumber) + self.threadNumber += 1 + + newThread.id = id + newThread.name = name + self.threads[id] = newThread + + # adjust current frame + self.threads[id].currentFrame = self.getExecutedFrame(frame) + + # Clean up obsolet because terminated threads + self.threads = {id_: thrd for id_, thrd in self.threads.items() + if id_ in frames} + + def find_module(self, fullname, path=None): + """ + Public method returning the module loader. + + @param fullname name of the module to be loaded + @type str + @param path path to resolve the module name + @type str + @return module loader object + @rtype object + """ + if fullname in sys.modules or not self.debugging: + return None + + if fullname in [self.threadModName, 'PyQt4.QtCore', 'PyQt5.QtCore', + 'PySide.QtCore'] and self.enableImportHooks: + # Disable hook to be able to import original module + self.enableImportHooks = False + return self + + return None + + def load_module(self, fullname): + """ + Public method to load a module. + + @param fullname name of the module to be loaded + @type str + @return reference to the loaded module + @rtype module + """ + module = importlib.import_module(fullname) + sys.modules[fullname] = module + if (fullname == self.threadModName and + self._original_start_new_thread is None): + # make thread hooks available to system + self._original_start_new_thread = module.start_new_thread + module.start_new_thread = self.attachThread + elif fullname in ['PyQt4.QtCore', 'PyQt5.QtCore', + 'PySide.QtCore'] and self._qtThread is None: + self._qtThread = module.QThread + # _debugClient as a class attribute can't be accessed in following + # class. Therefore we need a global variable. + _debugClient = self + + class QThreadWrapper(module.QThread): + __qtThreadNumber = 1 + + def __init__(self, *args, **kwargs): + # Overwrite the provided run method with our own, to + # intercept the thread creation by Qt + self._ApplicationRun = self.run + self.run = self.__bootstrapQThread + + super(QThreadWrapper, self).__init__(*args, **kwargs) + + def __bootstrapQThread(self): + newThread = DebugBase(_debugClient) + ident = _thread.get_ident() + name = 'QtThread-{0}'.format(self.__qtThreadNumber) + self.__qtThreadNumber += 1 + + newThread.id = ident + newThread.name = name + + _debugClient.threads[ident] = newThread + + frame = sys._getframe() + newThread.botframe = frame + frame.f_trace = newThread.trace_dispatch + # see DebugBase.bootstrap + sys.settrace( + lambda frame, event, arg: newThread.trace_dispatch) + + return self._ApplicationRun() + + module.QThread = QThreadWrapper + + self.enableImportHooks = True + return module + + +# +# eflag: noqa = M702
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/DebugClients/Python/__init__.py Sun Oct 16 12:18:42 2016 +0200 @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- + +# Copyright (c) 2005 - 2016 Detlev Offenbach <detlev@die-offenbachs.de> +# + +""" +Package implementing the standard Python debugger. + +It consists of different kinds of debug clients. +"""
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/DebugClients/Python/coverage/__init__.py Sun Oct 16 12:18:42 2016 +0200 @@ -0,0 +1,35 @@ +# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt + +"""Code coverage measurement for Python. + +Ned Batchelder +http://nedbatchelder.com/code/coverage + +""" + +from coverage.version import __version__, __url__, version_info + +from coverage.control import Coverage, process_startup +from coverage.data import CoverageData +from coverage.misc import CoverageException +from coverage.plugin import CoveragePlugin, FileTracer, FileReporter +from coverage.pytracer import PyTracer + +# Backward compatibility. +coverage = Coverage + +# On Windows, we encode and decode deep enough that something goes wrong and +# the encodings.utf_8 module is loaded and then unloaded, I don't know why. +# Adding a reference here prevents it from being unloaded. Yuk. +import encodings.utf_8 + +# Because of the "from coverage.control import fooey" lines at the top of the +# file, there's an entry for coverage.coverage in sys.modules, mapped to None. +# This makes some inspection tools (like pydoc) unable to find the class +# coverage.coverage. So remove that entry. +import sys +try: + del sys.modules['coverage.coverage'] +except KeyError: + pass
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/DebugClients/Python/coverage/__main__.py Sun Oct 16 12:18:42 2016 +0200 @@ -0,0 +1,8 @@ +# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt + +"""Coverage.py's main entry point.""" + +import sys +from coverage.cmdline import main +sys.exit(main())
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/DebugClients/Python/coverage/annotate.py Sun Oct 16 12:18:42 2016 +0200 @@ -0,0 +1,103 @@ +# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt + +"""Source file annotation for coverage.py.""" + +import io +import os +import re + +from coverage.files import flat_rootname +from coverage.misc import isolate_module +from coverage.report import Reporter + +os = isolate_module(os) + + +class AnnotateReporter(Reporter): + """Generate annotated source files showing line coverage. + + This reporter creates annotated copies of the measured source files. Each + .py file is copied as a .py,cover file, with a left-hand margin annotating + each line:: + + > def h(x): + - if 0: #pragma: no cover + - pass + > if x == 1: + ! a = 1 + > else: + > a = 2 + + > h(2) + + Executed lines use '>', lines not executed use '!', lines excluded from + consideration use '-'. + + """ + + def __init__(self, coverage, config): + super(AnnotateReporter, self).__init__(coverage, config) + self.directory = None + + blank_re = re.compile(r"\s*(#|$)") + else_re = re.compile(r"\s*else\s*:\s*(#|$)") + + def report(self, morfs, directory=None): + """Run the report. + + See `coverage.report()` for arguments. + + """ + self.report_files(self.annotate_file, morfs, directory) + + def annotate_file(self, fr, analysis): + """Annotate a single file. + + `fr` is the FileReporter for the file to annotate. + + """ + statements = sorted(analysis.statements) + missing = sorted(analysis.missing) + excluded = sorted(analysis.excluded) + + if self.directory: + dest_file = os.path.join(self.directory, flat_rootname(fr.relative_filename())) + if dest_file.endswith("_py"): + dest_file = dest_file[:-3] + ".py" + dest_file += ",cover" + else: + dest_file = fr.filename + ",cover" + + with io.open(dest_file, 'w', encoding='utf8') as dest: + i = 0 + j = 0 + covered = True + source = fr.source() + for lineno, line in enumerate(source.splitlines(True), start=1): + while i < len(statements) and statements[i] < lineno: + i += 1 + while j < len(missing) and missing[j] < lineno: + j += 1 + if i < len(statements) and statements[i] == lineno: + covered = j >= len(missing) or missing[j] > lineno + if self.blank_re.match(line): + dest.write(u' ') + elif self.else_re.match(line): + # Special logic for lines containing only 'else:'. + if i >= len(statements) and j >= len(missing): + dest.write(u'! ') + elif i >= len(statements) or j >= len(missing): + dest.write(u'> ') + elif statements[i] == missing[j]: + dest.write(u'! ') + else: + dest.write(u'> ') + elif lineno in excluded: + dest.write(u'- ') + elif covered: + dest.write(u'> ') + else: + dest.write(u'! ') + + dest.write(line)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/DebugClients/Python/coverage/backunittest.py Sun Oct 16 12:18:42 2016 +0200 @@ -0,0 +1,42 @@ +# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt + +"""Implementations of unittest features from the future.""" + +# Use unittest2 if it's available, otherwise unittest. This gives us +# back-ported features for 2.6. +try: + import unittest2 as unittest +except ImportError: + import unittest + + +def unittest_has(method): + """Does `unittest.TestCase` have `method` defined?""" + return hasattr(unittest.TestCase, method) + + +class TestCase(unittest.TestCase): + """Just like unittest.TestCase, but with assert methods added. + + Designed to be compatible with 3.1 unittest. Methods are only defined if + `unittest` doesn't have them. + + """ + # pylint: disable=missing-docstring + + # Many Pythons have this method defined. But PyPy3 has a bug with it + # somehow (https://bitbucket.org/pypy/pypy/issues/2092), so always use our + # own implementation that works everywhere, at least for the ways we're + # calling it. + def assertCountEqual(self, s1, s2): + """Assert these have the same elements, regardless of order.""" + self.assertEqual(sorted(s1), sorted(s2)) + + if not unittest_has('assertRaisesRegex'): + def assertRaisesRegex(self, *args, **kwargs): + return self.assertRaisesRegexp(*args, **kwargs) + + if not unittest_has('assertRegex'): + def assertRegex(self, *args, **kwargs): + return self.assertRegexpMatches(*args, **kwargs)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/DebugClients/Python/coverage/backward.py Sun Oct 16 12:18:42 2016 +0200 @@ -0,0 +1,172 @@ +# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt + +"""Add things to old Pythons so I can pretend they are newer.""" + +# This file does lots of tricky stuff, so disable a bunch of pylint warnings. +# pylint: disable=redefined-builtin +# pylint: disable=unused-import +# pylint: disable=no-name-in-module + +import sys + +from coverage import env + + +# Pythons 2 and 3 differ on where to get StringIO. +try: + from cStringIO import StringIO +except ImportError: + from io import StringIO + +# In py3, ConfigParser was renamed to the more-standard configparser +try: + import configparser +except ImportError: + import ConfigParser as configparser + +# What's a string called? +try: + string_class = basestring +except NameError: + string_class = str + +# What's a Unicode string called? +try: + unicode_class = unicode +except NameError: + unicode_class = str + +# Where do pickles come from? +try: + import cPickle as pickle +except ImportError: + import pickle + +# range or xrange? +try: + range = xrange +except NameError: + range = range + +# shlex.quote is new, but there's an undocumented implementation in "pipes", +# who knew!? +try: + from shlex import quote as shlex_quote +except ImportError: + # Useful function, available under a different (undocumented) name + # in Python versions earlier than 3.3. + from pipes import quote as shlex_quote + +# A function to iterate listlessly over a dict's items. +try: + {}.iteritems +except AttributeError: + def iitems(d): + """Produce the items from dict `d`.""" + return d.items() +else: + def iitems(d): + """Produce the items from dict `d`.""" + return d.iteritems() + +# Getting the `next` function from an iterator is different in 2 and 3. +try: + iter([]).next +except AttributeError: + def iternext(seq): + """Get the `next` function for iterating over `seq`.""" + return iter(seq).__next__ +else: + def iternext(seq): + """Get the `next` function for iterating over `seq`.""" + return iter(seq).next + +# Python 3.x is picky about bytes and strings, so provide methods to +# get them right, and make them no-ops in 2.x +if env.PY3: + def to_bytes(s): + """Convert string `s` to bytes.""" + return s.encode('utf8') + + def binary_bytes(byte_values): + """Produce a byte string with the ints from `byte_values`.""" + return bytes(byte_values) + + def bytes_to_ints(bytes_value): + """Turn a bytes object into a sequence of ints.""" + # In Python 3, iterating bytes gives ints. + return bytes_value + +else: + def to_bytes(s): + """Convert string `s` to bytes (no-op in 2.x).""" + return s + + def binary_bytes(byte_values): + """Produce a byte string with the ints from `byte_values`.""" + return "".join(chr(b) for b in byte_values) + + def bytes_to_ints(bytes_value): + """Turn a bytes object into a sequence of ints.""" + for byte in bytes_value: + yield ord(byte) + + +try: + # In Python 2.x, the builtins were in __builtin__ + BUILTINS = sys.modules['__builtin__'] +except KeyError: + # In Python 3.x, they're in builtins + BUILTINS = sys.modules['builtins'] + + +# imp was deprecated in Python 3.3 +try: + import importlib + import importlib.util + imp = None +except ImportError: + importlib = None + +# We only want to use importlib if it has everything we need. +try: + importlib_util_find_spec = importlib.util.find_spec +except Exception: + import imp + importlib_util_find_spec = None + +# What is the .pyc magic number for this version of Python? +try: + PYC_MAGIC_NUMBER = importlib.util.MAGIC_NUMBER +except AttributeError: + PYC_MAGIC_NUMBER = imp.get_magic() + + +def import_local_file(modname, modfile=None): + """Import a local file as a module. + + Opens a file in the current directory named `modname`.py, imports it + as `modname`, and returns the module object. `modfile` is the file to + import if it isn't in the current directory. + + """ + try: + from importlib.machinery import SourceFileLoader + except ImportError: + SourceFileLoader = None + + if modfile is None: + modfile = modname + '.py' + if SourceFileLoader: + mod = SourceFileLoader(modname, modfile).load_module() + else: + for suff in imp.get_suffixes(): # pragma: part covered + if suff[0] == '.py': + break + + with open(modfile, 'r') as f: + # pylint: disable=undefined-loop-variable + mod = imp.load_module(modname, f, modfile, suff) + + return mod
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/DebugClients/Python/coverage/bytecode.py Sun Oct 16 12:18:42 2016 +0200 @@ -0,0 +1,22 @@ +# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt + +"""Bytecode manipulation for coverage.py""" + +import types + + +class CodeObjects(object): + """Iterate over all the code objects in `code`.""" + def __init__(self, code): + self.stack = [code] + + def __iter__(self): + while self.stack: + # We're going to return the code object on the stack, but first + # push its children for later returning. + code = self.stack.pop() + for c in code.co_consts: + if isinstance(c, types.CodeType): + self.stack.append(c) + yield code
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/DebugClients/Python/coverage/cmdline.py Sun Oct 16 12:18:42 2016 +0200 @@ -0,0 +1,763 @@ +# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt + +"""Command-line support for coverage.py.""" + +import glob +import optparse +import os.path +import sys +import textwrap +import traceback + +from coverage import env +from coverage.collector import CTracer +from coverage.execfile import run_python_file, run_python_module +from coverage.misc import CoverageException, ExceptionDuringRun, NoSource +from coverage.debug import info_formatter, info_header + + +class Opts(object): + """A namespace class for individual options we'll build parsers from.""" + + append = optparse.make_option( + '-a', '--append', action='store_true', + help="Append coverage data to .coverage, otherwise it is started clean with each run.", + ) + branch = optparse.make_option( + '', '--branch', action='store_true', + help="Measure branch coverage in addition to statement coverage.", + ) + CONCURRENCY_CHOICES = [ + "thread", "gevent", "greenlet", "eventlet", "multiprocessing", + ] + concurrency = optparse.make_option( + '', '--concurrency', action='store', metavar="LIB", + choices=CONCURRENCY_CHOICES, + help=( + "Properly measure code using a concurrency library. " + "Valid values are: %s." + ) % ", ".join(CONCURRENCY_CHOICES), + ) + debug = optparse.make_option( + '', '--debug', action='store', metavar="OPTS", + help="Debug options, separated by commas", + ) + directory = optparse.make_option( + '-d', '--directory', action='store', metavar="DIR", + help="Write the output files to DIR.", + ) + fail_under = optparse.make_option( + '', '--fail-under', action='store', metavar="MIN", type="int", + help="Exit with a status of 2 if the total coverage is less than MIN.", + ) + help = optparse.make_option( + '-h', '--help', action='store_true', + help="Get help on this command.", + ) + ignore_errors = optparse.make_option( + '-i', '--ignore-errors', action='store_true', + help="Ignore errors while reading source files.", + ) + include = optparse.make_option( + '', '--include', action='store', + metavar="PAT1,PAT2,...", + help=( + "Include only files whose paths match one of these patterns. " + "Accepts shell-style wildcards, which must be quoted." + ), + ) + pylib = optparse.make_option( + '-L', '--pylib', action='store_true', + help=( + "Measure coverage even inside the Python installed library, " + "which isn't done by default." + ), + ) + show_missing = optparse.make_option( + '-m', '--show-missing', action='store_true', + help="Show line numbers of statements in each module that weren't executed.", + ) + skip_covered = optparse.make_option( + '--skip-covered', action='store_true', + help="Skip files with 100% coverage.", + ) + omit = optparse.make_option( + '', '--omit', action='store', + metavar="PAT1,PAT2,...", + help=( + "Omit files whose paths match one of these patterns. " + "Accepts shell-style wildcards, which must be quoted." + ), + ) + output_xml = optparse.make_option( + '-o', '', action='store', dest="outfile", + metavar="OUTFILE", + help="Write the XML report to this file. Defaults to 'coverage.xml'", + ) + parallel_mode = optparse.make_option( + '-p', '--parallel-mode', action='store_true', + help=( + "Append the machine name, process id and random number to the " + ".coverage data file name to simplify collecting data from " + "many processes." + ), + ) + module = optparse.make_option( + '-m', '--module', action='store_true', + help=( + "<pyfile> is an importable Python module, not a script path, " + "to be run as 'python -m' would run it." + ), + ) + rcfile = optparse.make_option( + '', '--rcfile', action='store', + help="Specify configuration file. Defaults to '.coveragerc'", + ) + source = optparse.make_option( + '', '--source', action='store', metavar="SRC1,SRC2,...", + help="A list of packages or directories of code to be measured.", + ) + timid = optparse.make_option( + '', '--timid', action='store_true', + help=( + "Use a simpler but slower trace method. Try this if you get " + "seemingly impossible results!" + ), + ) + title = optparse.make_option( + '', '--title', action='store', metavar="TITLE", + help="A text string to use as the title on the HTML.", + ) + version = optparse.make_option( + '', '--version', action='store_true', + help="Display version information and exit.", + ) + + +class CoverageOptionParser(optparse.OptionParser, object): + """Base OptionParser for coverage.py. + + Problems don't exit the program. + Defaults are initialized for all options. + + """ + + def __init__(self, *args, **kwargs): + super(CoverageOptionParser, self).__init__( + add_help_option=False, *args, **kwargs + ) + self.set_defaults( + action=None, + append=None, + branch=None, + concurrency=None, + debug=None, + directory=None, + fail_under=None, + help=None, + ignore_errors=None, + include=None, + module=None, + omit=None, + parallel_mode=None, + pylib=None, + rcfile=True, + show_missing=None, + skip_covered=None, + source=None, + timid=None, + title=None, + version=None, + ) + + self.disable_interspersed_args() + self.help_fn = self.help_noop + + def help_noop(self, error=None, topic=None, parser=None): + """No-op help function.""" + pass + + class OptionParserError(Exception): + """Used to stop the optparse error handler ending the process.""" + pass + + def parse_args_ok(self, args=None, options=None): + """Call optparse.parse_args, but return a triple: + + (ok, options, args) + + """ + try: + options, args = \ + super(CoverageOptionParser, self).parse_args(args, options) + except self.OptionParserError: + return False, None, None + return True, options, args + + def error(self, msg): + """Override optparse.error so sys.exit doesn't get called.""" + self.help_fn(msg) + raise self.OptionParserError + + +class GlobalOptionParser(CoverageOptionParser): + """Command-line parser for coverage.py global option arguments.""" + + def __init__(self): + super(GlobalOptionParser, self).__init__() + + self.add_options([ + Opts.help, + Opts.version, + ]) + + +class CmdOptionParser(CoverageOptionParser): + """Parse one of the new-style commands for coverage.py.""" + + def __init__(self, action, options=None, defaults=None, usage=None, description=None): + """Create an OptionParser for a coverage.py command. + + `action` is the slug to put into `options.action`. + `options` is a list of Option's for the command. + `defaults` is a dict of default value for options. + `usage` is the usage string to display in help. + `description` is the description of the command, for the help text. + + """ + if usage: + usage = "%prog " + usage + super(CmdOptionParser, self).__init__( + usage=usage, + description=description, + ) + self.set_defaults(action=action, **(defaults or {})) + if options: + self.add_options(options) + self.cmd = action + + def __eq__(self, other): + # A convenience equality, so that I can put strings in unit test + # results, and they will compare equal to objects. + return (other == "<CmdOptionParser:%s>" % self.cmd) + + def get_prog_name(self): + """Override of an undocumented function in optparse.OptionParser.""" + program_name = super(CmdOptionParser, self).get_prog_name() + + # Include the sub-command for this parser as part of the command. + return "%(command)s %(subcommand)s" % {'command': program_name, 'subcommand': self.cmd} + + +GLOBAL_ARGS = [ + Opts.debug, + Opts.help, + Opts.rcfile, + ] + +CMDS = { + 'annotate': CmdOptionParser( + "annotate", + [ + Opts.directory, + Opts.ignore_errors, + Opts.include, + Opts.omit, + ] + GLOBAL_ARGS, + usage="[options] [modules]", + description=( + "Make annotated copies of the given files, marking statements that are executed " + "with > and statements that are missed with !." + ), + ), + + 'combine': CmdOptionParser( + "combine", + GLOBAL_ARGS, + usage="<path1> <path2> ... <pathN>", + description=( + "Combine data from multiple coverage files collected " + "with 'run -p'. The combined results are written to a single " + "file representing the union of the data. The positional " + "arguments are data files or directories containing data files. " + "If no paths are provided, data files in the default data file's " + "directory are combined." + ), + ), + + 'debug': CmdOptionParser( + "debug", GLOBAL_ARGS, + usage="<topic>", + description=( + "Display information on the internals of coverage.py, " + "for diagnosing problems. " + "Topics are 'data' to show a summary of the collected data, " + "or 'sys' to show installation information." + ), + ), + + 'erase': CmdOptionParser( + "erase", GLOBAL_ARGS, + usage=" ", + description="Erase previously collected coverage data.", + ), + + 'help': CmdOptionParser( + "help", GLOBAL_ARGS, + usage="[command]", + description="Describe how to use coverage.py", + ), + + 'html': CmdOptionParser( + "html", + [ + Opts.directory, + Opts.fail_under, + Opts.ignore_errors, + Opts.include, + Opts.omit, + Opts.title, + ] + GLOBAL_ARGS, + usage="[options] [modules]", + description=( + "Create an HTML report of the coverage of the files. " + "Each file gets its own page, with the source decorated to show " + "executed, excluded, and missed lines." + ), + ), + + 'report': CmdOptionParser( + "report", + [ + Opts.fail_under, + Opts.ignore_errors, + Opts.include, + Opts.omit, + Opts.show_missing, + Opts.skip_covered, + ] + GLOBAL_ARGS, + usage="[options] [modules]", + description="Report coverage statistics on modules." + ), + + 'run': CmdOptionParser( + "run", + [ + Opts.append, + Opts.branch, + Opts.concurrency, + Opts.include, + Opts.module, + Opts.omit, + Opts.pylib, + Opts.parallel_mode, + Opts.source, + Opts.timid, + ] + GLOBAL_ARGS, + usage="[options] <pyfile> [program options]", + description="Run a Python program, measuring code execution." + ), + + 'xml': CmdOptionParser( + "xml", + [ + Opts.fail_under, + Opts.ignore_errors, + Opts.include, + Opts.omit, + Opts.output_xml, + ] + GLOBAL_ARGS, + usage="[options] [modules]", + description="Generate an XML report of coverage results." + ), +} + + +OK, ERR, FAIL_UNDER = 0, 1, 2 + + +class CoverageScript(object): + """The command-line interface to coverage.py.""" + + def __init__(self, _covpkg=None, _run_python_file=None, + _run_python_module=None, _help_fn=None, _path_exists=None): + # _covpkg is for dependency injection, so we can test this code. + if _covpkg: + self.covpkg = _covpkg + else: + import coverage + self.covpkg = coverage + + # For dependency injection: + self.run_python_file = _run_python_file or run_python_file + self.run_python_module = _run_python_module or run_python_module + self.help_fn = _help_fn or self.help + self.path_exists = _path_exists or os.path.exists + self.global_option = False + + self.coverage = None + + self.program_name = os.path.basename(sys.argv[0]) + if env.WINDOWS: + # entry_points={'console_scripts':...} on Windows makes files + # called coverage.exe, coverage3.exe, and coverage-3.5.exe. These + # invoke coverage-script.py, coverage3-script.py, and + # coverage-3.5-script.py. argv[0] is the .py file, but we want to + # get back to the original form. + auto_suffix = "-script.py" + if self.program_name.endswith(auto_suffix): + self.program_name = self.program_name[:-len(auto_suffix)] + + def command_line(self, argv): + """The bulk of the command line interface to coverage.py. + + `argv` is the argument list to process. + + Returns 0 if all is well, 1 if something went wrong. + + """ + # Collect the command-line options. + if not argv: + self.help_fn(topic='minimum_help') + return OK + + # The command syntax we parse depends on the first argument. Global + # switch syntax always starts with an option. + self.global_option = argv[0].startswith('-') + if self.global_option: + parser = GlobalOptionParser() + else: + parser = CMDS.get(argv[0]) + if not parser: + self.help_fn("Unknown command: '%s'" % argv[0]) + return ERR + argv = argv[1:] + + parser.help_fn = self.help_fn + ok, options, args = parser.parse_args_ok(argv) + if not ok: + return ERR + + # Handle help and version. + if self.do_help(options, args, parser): + return OK + + # Check for conflicts and problems in the options. + if not self.args_ok(options, args): + return ERR + + # We need to be able to import from the current directory, because + # plugins may try to, for example, to read Django settings. + sys.path[0] = '' + + # Listify the list options. + source = unshell_list(options.source) + omit = unshell_list(options.omit) + include = unshell_list(options.include) + debug = unshell_list(options.debug) + + # Do something. + self.coverage = self.covpkg.coverage( + data_suffix=options.parallel_mode, + cover_pylib=options.pylib, + timid=options.timid, + branch=options.branch, + config_file=options.rcfile, + source=source, + omit=omit, + include=include, + debug=debug, + concurrency=options.concurrency, + ) + + if options.action == "debug": + return self.do_debug(args) + + elif options.action == "erase": + self.coverage.erase() + return OK + + elif options.action == "run": + return self.do_run(options, args) + + elif options.action == "combine": + self.coverage.load() + data_dirs = args or None + self.coverage.combine(data_dirs) + self.coverage.save() + return OK + + # Remaining actions are reporting, with some common options. + report_args = dict( + morfs=unglob_args(args), + ignore_errors=options.ignore_errors, + omit=omit, + include=include, + ) + + self.coverage.load() + + total = None + if options.action == "report": + total = self.coverage.report( + show_missing=options.show_missing, + skip_covered=options.skip_covered, **report_args) + elif options.action == "annotate": + self.coverage.annotate( + directory=options.directory, **report_args) + elif options.action == "html": + total = self.coverage.html_report( + directory=options.directory, title=options.title, + **report_args) + elif options.action == "xml": + outfile = options.outfile + total = self.coverage.xml_report(outfile=outfile, **report_args) + + if total is not None: + # Apply the command line fail-under options, and then use the config + # value, so we can get fail_under from the config file. + if options.fail_under is not None: + self.coverage.set_option("report:fail_under", options.fail_under) + + if self.coverage.get_option("report:fail_under"): + + # Total needs to be rounded, but be careful of 0 and 100. + if 0 < total < 1: + total = 1 + elif 99 < total < 100: + total = 99 + else: + total = round(total) + + if total >= self.coverage.get_option("report:fail_under"): + return OK + else: + return FAIL_UNDER + + return OK + + def help(self, error=None, topic=None, parser=None): + """Display an error message, or the named topic.""" + assert error or topic or parser + if error: + print(error) + print("Use '%s help' for help." % (self.program_name,)) + elif parser: + print(parser.format_help().strip()) + else: + help_params = dict(self.covpkg.__dict__) + help_params['program_name'] = self.program_name + if CTracer is not None: + help_params['extension_modifier'] = 'with C extension' + else: + help_params['extension_modifier'] = 'without C extension' + help_msg = textwrap.dedent(HELP_TOPICS.get(topic, '')).strip() + if help_msg: + print(help_msg.format(**help_params)) + else: + print("Don't know topic %r" % topic) + + def do_help(self, options, args, parser): + """Deal with help requests. + + Return True if it handled the request, False if not. + + """ + # Handle help. + if options.help: + if self.global_option: + self.help_fn(topic='help') + else: + self.help_fn(parser=parser) + return True + + if options.action == "help": + if args: + for a in args: + parser = CMDS.get(a) + if parser: + self.help_fn(parser=parser) + else: + self.help_fn(topic=a) + else: + self.help_fn(topic='help') + return True + + # Handle version. + if options.version: + self.help_fn(topic='version') + return True + + return False + + def args_ok(self, options, args): + """Check for conflicts and problems in the options. + + Returns True if everything is OK, or False if not. + + """ + if options.action == "run" and not args: + self.help_fn("Nothing to do.") + return False + + return True + + def do_run(self, options, args): + """Implementation of 'coverage run'.""" + + if options.append and self.coverage.get_option("run:parallel"): + self.help_fn("Can't append to data files in parallel mode.") + return ERR + + if not self.coverage.get_option("run:parallel"): + if not options.append: + self.coverage.erase() + + # Run the script. + self.coverage.start() + code_ran = True + try: + if options.module: + self.run_python_module(args[0], args) + else: + filename = args[0] + self.run_python_file(filename, args) + except NoSource: + code_ran = False + raise + finally: + self.coverage.stop() + if code_ran: + if options.append: + data_file = self.coverage.get_option("run:data_file") + if self.path_exists(data_file): + self.coverage.combine(data_paths=[data_file]) + self.coverage.save() + + return OK + + def do_debug(self, args): + """Implementation of 'coverage debug'.""" + + if not args: + self.help_fn("What information would you like: data, sys?") + return ERR + + for info in args: + if info == 'sys': + sys_info = self.coverage.sys_info() + print(info_header("sys")) + for line in info_formatter(sys_info): + print(" %s" % line) + elif info == 'data': + self.coverage.load() + data = self.coverage.data + print(info_header("data")) + print("path: %s" % self.coverage.data_files.filename) + if data: + print("has_arcs: %r" % data.has_arcs()) + summary = data.line_counts(fullpath=True) + filenames = sorted(summary.keys()) + print("\n%d files:" % len(filenames)) + for f in filenames: + line = "%s: %d lines" % (f, summary[f]) + plugin = data.file_tracer(f) + if plugin: + line += " [%s]" % plugin + print(line) + else: + print("No data collected") + else: + self.help_fn("Don't know what you mean by %r" % info) + return ERR + + return OK + + +def unshell_list(s): + """Turn a command-line argument into a list.""" + if not s: + return None + if env.WINDOWS: + # When running coverage.py as coverage.exe, some of the behavior + # of the shell is emulated: wildcards are expanded into a list of + # file names. So you have to single-quote patterns on the command + # line, but (not) helpfully, the single quotes are included in the + # argument, so we have to strip them off here. + s = s.strip("'") + return s.split(',') + + +def unglob_args(args): + """Interpret shell wildcards for platforms that need it.""" + if env.WINDOWS: + globbed = [] + for arg in args: + if '?' in arg or '*' in arg: + globbed.extend(glob.glob(arg)) + else: + globbed.append(arg) + args = globbed + return args + + +HELP_TOPICS = { + 'help': """\ + Coverage.py, version {__version__} {extension_modifier} + Measure, collect, and report on code coverage in Python programs. + + usage: {program_name} <command> [options] [args] + + Commands: + annotate Annotate source files with execution information. + combine Combine a number of data files. + erase Erase previously collected coverage data. + help Get help on using coverage.py. + html Create an HTML report. + report Report coverage stats on modules. + run Run a Python program and measure code execution. + xml Create an XML report of coverage results. + + Use "{program_name} help <command>" for detailed help on any command. + For full documentation, see {__url__} + """, + + 'minimum_help': """\ + Code coverage for Python. Use '{program_name} help' for help. + """, + + 'version': """\ + Coverage.py, version {__version__} {extension_modifier} + Documentation at {__url__} + """, +} + + +def main(argv=None): + """The main entry point to coverage.py. + + This is installed as the script entry point. + + """ + if argv is None: + argv = sys.argv[1:] + try: + status = CoverageScript().command_line(argv) + except ExceptionDuringRun as err: + # An exception was caught while running the product code. The + # sys.exc_info() return tuple is packed into an ExceptionDuringRun + # exception. + traceback.print_exception(*err.args) + status = ERR + except CoverageException as err: + # A controlled error inside coverage.py: print the message to the user. + print(err) + status = ERR + except SystemExit as err: + # The user called `sys.exit()`. Exit with their argument, if any. + if err.args: + status = err.args[0] + else: + status = None + return status
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/DebugClients/Python/coverage/collector.py Sun Oct 16 12:18:42 2016 +0200 @@ -0,0 +1,361 @@ +# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt + +"""Raw data collector for coverage.py.""" + +import os +import sys + +from coverage import env +from coverage.backward import iitems +from coverage.files import abs_file +from coverage.misc import CoverageException, isolate_module +from coverage.pytracer import PyTracer + +os = isolate_module(os) + + +try: + # Use the C extension code when we can, for speed. + from coverage.tracer import CTracer, CFileDisposition # pylint: disable=no-name-in-module +except ImportError: + # Couldn't import the C extension, maybe it isn't built. + if os.getenv('COVERAGE_TEST_TRACER') == 'c': + # During testing, we use the COVERAGE_TEST_TRACER environment variable + # to indicate that we've fiddled with the environment to test this + # fallback code. If we thought we had a C tracer, but couldn't import + # it, then exit quickly and clearly instead of dribbling confusing + # errors. I'm using sys.exit here instead of an exception because an + # exception here causes all sorts of other noise in unittest. + sys.stderr.write("*** COVERAGE_TEST_TRACER is 'c' but can't import CTracer!\n") + sys.exit(1) + CTracer = None + + +class FileDisposition(object): + """A simple value type for recording what to do with a file.""" + pass + + +def should_start_context(frame): + """Who-Tests-What hack: Determine whether this frame begins a new who-context.""" + fn_name = frame.f_code.co_name + if fn_name.startswith("test"): + return fn_name + + +class Collector(object): + """Collects trace data. + + Creates a Tracer object for each thread, since they track stack + information. Each Tracer points to the same shared data, contributing + traced data points. + + When the Collector is started, it creates a Tracer for the current thread, + and installs a function to create Tracers for each new thread started. + When the Collector is stopped, all active Tracers are stopped. + + Threads started while the Collector is stopped will never have Tracers + associated with them. + + """ + + # The stack of active Collectors. Collectors are added here when started, + # and popped when stopped. Collectors on the stack are paused when not + # the top, and resumed when they become the top again. + _collectors = [] + + def __init__(self, should_trace, check_include, timid, branch, warn, concurrency): + """Create a collector. + + `should_trace` is a function, taking a file name, and returning a + `coverage.FileDisposition object`. + + `check_include` is a function taking a file name and a frame. It returns + a boolean: True if the file should be traced, False if not. + + If `timid` is true, then a slower simpler trace function will be + used. This is important for some environments where manipulation of + tracing functions make the faster more sophisticated trace function not + operate properly. + + If `branch` is true, then branches will be measured. This involves + collecting data on which statements followed each other (arcs). Use + `get_arc_data` to get the arc data. + + `warn` is a warning function, taking a single string message argument, + to be used if a warning needs to be issued. + + `concurrency` is a string indicating the concurrency library in use. + Valid values are "greenlet", "eventlet", "gevent", or "thread" (the + default). + + """ + self.should_trace = should_trace + self.check_include = check_include + self.warn = warn + self.branch = branch + self.threading = None + self.concurrency = concurrency + + self.concur_id_func = None + + try: + if concurrency == "greenlet": + import greenlet + self.concur_id_func = greenlet.getcurrent + elif concurrency == "eventlet": + import eventlet.greenthread # pylint: disable=import-error,useless-suppression + self.concur_id_func = eventlet.greenthread.getcurrent + elif concurrency == "gevent": + import gevent # pylint: disable=import-error,useless-suppression + self.concur_id_func = gevent.getcurrent + elif concurrency == "thread" or not concurrency: + # It's important to import threading only if we need it. If + # it's imported early, and the program being measured uses + # gevent, then gevent's monkey-patching won't work properly. + import threading + self.threading = threading + else: + raise CoverageException("Don't understand concurrency=%s" % concurrency) + except ImportError: + raise CoverageException( + "Couldn't trace with concurrency=%s, the module isn't installed." % concurrency + ) + + # Who-Tests-What is just a hack at the moment, so turn it on with an + # environment variable. + self.wtw = int(os.getenv('COVERAGE_WTW', 0)) + + self.reset() + + if timid: + # Being timid: use the simple Python trace function. + self._trace_class = PyTracer + else: + # Being fast: use the C Tracer if it is available, else the Python + # trace function. + self._trace_class = CTracer or PyTracer + + if self._trace_class is CTracer: + self.file_disposition_class = CFileDisposition + self.supports_plugins = True + else: + self.file_disposition_class = FileDisposition + self.supports_plugins = False + + def __repr__(self): + return "<Collector at 0x%x: %s>" % (id(self), self.tracer_name()) + + def tracer_name(self): + """Return the class name of the tracer we're using.""" + return self._trace_class.__name__ + + def reset(self): + """Clear collected data, and prepare to collect more.""" + # A dictionary mapping file names to dicts with line number keys (if not + # branch coverage), or mapping file names to dicts with line number + # pairs as keys (if branch coverage). + self.data = {} + + # A dict mapping contexts to data dictionaries. + self.contexts = {} + self.contexts[None] = self.data + + # A dictionary mapping file names to file tracer plugin names that will + # handle them. + self.file_tracers = {} + + # The .should_trace_cache attribute is a cache from file names to + # coverage.FileDisposition objects, or None. When a file is first + # considered for tracing, a FileDisposition is obtained from + # Coverage.should_trace. Its .trace attribute indicates whether the + # file should be traced or not. If it should be, a plugin with dynamic + # file names can decide not to trace it based on the dynamic file name + # being excluded by the inclusion rules, in which case the + # FileDisposition will be replaced by None in the cache. + if env.PYPY: + import __pypy__ # pylint: disable=import-error + # Alex Gaynor said: + # should_trace_cache is a strictly growing key: once a key is in + # it, it never changes. Further, the keys used to access it are + # generally constant, given sufficient context. That is to say, at + # any given point _trace() is called, pypy is able to know the key. + # This is because the key is determined by the physical source code + # line, and that's invariant with the call site. + # + # This property of a dict with immutable keys, combined with + # call-site-constant keys is a match for PyPy's module dict, + # which is optimized for such workloads. + # + # This gives a 20% benefit on the workload described at + # https://bitbucket.org/pypy/pypy/issue/1871/10x-slower-than-cpython-under-coverage + self.should_trace_cache = __pypy__.newdict("module") + else: + self.should_trace_cache = {} + + # Our active Tracers. + self.tracers = [] + + def _start_tracer(self): + """Start a new Tracer object, and store it in self.tracers.""" + tracer = self._trace_class() + tracer.data = self.data + tracer.trace_arcs = self.branch + tracer.should_trace = self.should_trace + tracer.should_trace_cache = self.should_trace_cache + tracer.warn = self.warn + + if hasattr(tracer, 'concur_id_func'): + tracer.concur_id_func = self.concur_id_func + elif self.concur_id_func: + raise CoverageException( + "Can't support concurrency=%s with %s, only threads are supported" % ( + self.concurrency, self.tracer_name(), + ) + ) + + if hasattr(tracer, 'file_tracers'): + tracer.file_tracers = self.file_tracers + if hasattr(tracer, 'threading'): + tracer.threading = self.threading + if hasattr(tracer, 'check_include'): + tracer.check_include = self.check_include + if self.wtw: + if hasattr(tracer, 'should_start_context'): + tracer.should_start_context = should_start_context + if hasattr(tracer, 'switch_context'): + tracer.switch_context = self.switch_context + + fn = tracer.start() + self.tracers.append(tracer) + + return fn + + # The trace function has to be set individually on each thread before + # execution begins. Ironically, the only support the threading module has + # for running code before the thread main is the tracing function. So we + # install this as a trace function, and the first time it's called, it does + # the real trace installation. + + def _installation_trace(self, frame, event, arg): + """Called on new threads, installs the real tracer.""" + # Remove ourselves as the trace function. + sys.settrace(None) + # Install the real tracer. + fn = self._start_tracer() + # Invoke the real trace function with the current event, to be sure + # not to lose an event. + if fn: + fn = fn(frame, event, arg) + # Return the new trace function to continue tracing in this scope. + return fn + + def start(self): + """Start collecting trace information.""" + if self._collectors: + self._collectors[-1].pause() + + # Check to see whether we had a fullcoverage tracer installed. If so, + # get the stack frames it stashed away for us. + traces0 = [] + fn0 = sys.gettrace() + if fn0: + tracer0 = getattr(fn0, '__self__', None) + if tracer0: + traces0 = getattr(tracer0, 'traces', []) + + try: + # Install the tracer on this thread. + fn = self._start_tracer() + except: + if self._collectors: + self._collectors[-1].resume() + raise + + # If _start_tracer succeeded, then we add ourselves to the global + # stack of collectors. + self._collectors.append(self) + + # Replay all the events from fullcoverage into the new trace function. + for args in traces0: + (frame, event, arg), lineno = args + try: + fn(frame, event, arg, lineno=lineno) + except TypeError: + raise Exception("fullcoverage must be run with the C trace function.") + + # Install our installation tracer in threading, to jump start other + # threads. + if self.threading: + self.threading.settrace(self._installation_trace) + + def stop(self): + """Stop collecting trace information.""" + assert self._collectors + assert self._collectors[-1] is self, ( + "Expected current collector to be %r, but it's %r" % (self, self._collectors[-1]) + ) + + self.pause() + self.tracers = [] + + # Remove this Collector from the stack, and resume the one underneath + # (if any). + self._collectors.pop() + if self._collectors: + self._collectors[-1].resume() + + def pause(self): + """Pause tracing, but be prepared to `resume`.""" + for tracer in self.tracers: + tracer.stop() + stats = tracer.get_stats() + if stats: + print("\nCoverage.py tracer stats:") + for k in sorted(stats.keys()): + print("%20s: %s" % (k, stats[k])) + if self.threading: + self.threading.settrace(None) + + def resume(self): + """Resume tracing after a `pause`.""" + for tracer in self.tracers: + tracer.start() + if self.threading: + self.threading.settrace(self._installation_trace) + else: + self._start_tracer() + + def switch_context(self, new_context): + """Who-Tests-What hack: switch to a new who-context.""" + # Make a new data dict, or find the existing one, and switch all the + # tracers to use it. + data = self.contexts.setdefault(new_context, {}) + for tracer in self.tracers: + tracer.data = data + + def save_data(self, covdata): + """Save the collected data to a `CoverageData`. + + Also resets the collector. + + """ + def abs_file_dict(d): + """Return a dict like d, but with keys modified by `abs_file`.""" + return dict((abs_file(k), v) for k, v in iitems(d)) + + if self.branch: + covdata.add_arcs(abs_file_dict(self.data)) + else: + covdata.add_lines(abs_file_dict(self.data)) + covdata.add_file_tracers(abs_file_dict(self.file_tracers)) + + if self.wtw: + # Just a hack, so just hack it. + import pprint + out_file = "coverage_wtw_{:06}.py".format(os.getpid()) + with open(out_file, "w") as wtw_out: + pprint.pprint(self.contexts, wtw_out) + + self.reset()
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/DebugClients/Python/coverage/config.py Sun Oct 16 12:18:42 2016 +0200 @@ -0,0 +1,365 @@ +# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt + +"""Config file for coverage.py""" + +import collections +import os +import re +import sys + +from coverage.backward import configparser, iitems, string_class +from coverage.misc import CoverageException, isolate_module + +os = isolate_module(os) + + +class HandyConfigParser(configparser.RawConfigParser): + """Our specialization of ConfigParser.""" + + def __init__(self, section_prefix): + configparser.RawConfigParser.__init__(self) + self.section_prefix = section_prefix + + def read(self, filename): + """Read a file name as UTF-8 configuration data.""" + kwargs = {} + if sys.version_info >= (3, 2): + kwargs['encoding'] = "utf-8" + return configparser.RawConfigParser.read(self, filename, **kwargs) + + def has_option(self, section, option): + section = self.section_prefix + section + return configparser.RawConfigParser.has_option(self, section, option) + + def has_section(self, section): + section = self.section_prefix + section + return configparser.RawConfigParser.has_section(self, section) + + def options(self, section): + section = self.section_prefix + section + return configparser.RawConfigParser.options(self, section) + + def get_section(self, section): + """Get the contents of a section, as a dictionary.""" + d = {} + for opt in self.options(section): + d[opt] = self.get(section, opt) + return d + + def get(self, section, *args, **kwargs): + """Get a value, replacing environment variables also. + + The arguments are the same as `RawConfigParser.get`, but in the found + value, ``$WORD`` or ``${WORD}`` are replaced by the value of the + environment variable ``WORD``. + + Returns the finished value. + + """ + section = self.section_prefix + section + v = configparser.RawConfigParser.get(self, section, *args, **kwargs) + def dollar_replace(m): + """Called for each $replacement.""" + # Only one of the groups will have matched, just get its text. + word = next(w for w in m.groups() if w is not None) # pragma: part covered + if word == "$": + return "$" + else: + return os.environ.get(word, '') + + dollar_pattern = r"""(?x) # Use extended regex syntax + \$(?: # A dollar sign, then + (?P<v1>\w+) | # a plain word, + {(?P<v2>\w+)} | # or a {-wrapped word, + (?P<char>[$]) # or a dollar sign. + ) + """ + v = re.sub(dollar_pattern, dollar_replace, v) + return v + + def getlist(self, section, option): + """Read a list of strings. + + The value of `section` and `option` is treated as a comma- and newline- + separated list of strings. Each value is stripped of whitespace. + + Returns the list of strings. + + """ + value_list = self.get(section, option) + values = [] + for value_line in value_list.split('\n'): + for value in value_line.split(','): + value = value.strip() + if value: + values.append(value) + return values + + def getregexlist(self, section, option): + """Read a list of full-line regexes. + + The value of `section` and `option` is treated as a newline-separated + list of regexes. Each value is stripped of whitespace. + + Returns the list of strings. + + """ + line_list = self.get(section, option) + value_list = [] + for value in line_list.splitlines(): + value = value.strip() + try: + re.compile(value) + except re.error as e: + raise CoverageException( + "Invalid [%s].%s value %r: %s" % (section, option, value, e) + ) + if value: + value_list.append(value) + return value_list + + +# The default line exclusion regexes. +DEFAULT_EXCLUDE = [ + r'(?i)#\s*pragma[:\s]?\s*no\s*cover', +] + +# The default partial branch regexes, to be modified by the user. +DEFAULT_PARTIAL = [ + r'(?i)#\s*pragma[:\s]?\s*no\s*branch', +] + +# The default partial branch regexes, based on Python semantics. +# These are any Python branching constructs that can't actually execute all +# their branches. +DEFAULT_PARTIAL_ALWAYS = [ + 'while (True|1|False|0):', + 'if (True|1|False|0):', +] + + +class CoverageConfig(object): + """Coverage.py configuration. + + The attributes of this class are the various settings that control the + operation of coverage.py. + + """ + def __init__(self): + """Initialize the configuration attributes to their defaults.""" + # Metadata about the config. + self.attempted_config_files = [] + self.config_files = [] + + # Defaults for [run] + self.branch = False + self.concurrency = None + self.cover_pylib = False + self.data_file = ".coverage" + self.debug = [] + self.note = None + self.parallel = False + self.plugins = [] + self.source = None + self.timid = False + + # Defaults for [report] + self.exclude_list = DEFAULT_EXCLUDE[:] + self.fail_under = 0 + self.ignore_errors = False + self.include = None + self.omit = None + self.partial_always_list = DEFAULT_PARTIAL_ALWAYS[:] + self.partial_list = DEFAULT_PARTIAL[:] + self.precision = 0 + self.show_missing = False + self.skip_covered = False + + # Defaults for [html] + self.extra_css = None + self.html_dir = "htmlcov" + self.html_title = "Coverage report" + + # Defaults for [xml] + self.xml_output = "coverage.xml" + self.xml_package_depth = 99 + + # Defaults for [paths] + self.paths = {} + + # Options for plugins + self.plugin_options = {} + + MUST_BE_LIST = ["omit", "include", "debug", "plugins"] + + def from_args(self, **kwargs): + """Read config values from `kwargs`.""" + for k, v in iitems(kwargs): + if v is not None: + if k in self.MUST_BE_LIST and isinstance(v, string_class): + v = [v] + setattr(self, k, v) + + def from_file(self, filename, section_prefix=""): + """Read configuration from a .rc file. + + `filename` is a file name to read. + + Returns True or False, whether the file could be read. + + """ + self.attempted_config_files.append(filename) + + cp = HandyConfigParser(section_prefix) + try: + files_read = cp.read(filename) + except configparser.Error as err: + raise CoverageException("Couldn't read config file %s: %s" % (filename, err)) + if not files_read: + return False + + self.config_files.extend(files_read) + + try: + for option_spec in self.CONFIG_FILE_OPTIONS: + self._set_attr_from_config_option(cp, *option_spec) + except ValueError as err: + raise CoverageException("Couldn't read config file %s: %s" % (filename, err)) + + # Check that there are no unrecognized options. + all_options = collections.defaultdict(set) + for option_spec in self.CONFIG_FILE_OPTIONS: + section, option = option_spec[1].split(":") + all_options[section].add(option) + + for section, options in iitems(all_options): + if cp.has_section(section): + for unknown in set(cp.options(section)) - options: + if section_prefix: + section = section_prefix + section + raise CoverageException( + "Unrecognized option '[%s] %s=' in config file %s" % ( + section, unknown, filename + ) + ) + + # [paths] is special + if cp.has_section('paths'): + for option in cp.options('paths'): + self.paths[option] = cp.getlist('paths', option) + + # plugins can have options + for plugin in self.plugins: + if cp.has_section(plugin): + self.plugin_options[plugin] = cp.get_section(plugin) + + return True + + CONFIG_FILE_OPTIONS = [ + # These are *args for _set_attr_from_config_option: + # (attr, where, type_="") + # + # attr is the attribute to set on the CoverageConfig object. + # where is the section:name to read from the configuration file. + # type_ is the optional type to apply, by using .getTYPE to read the + # configuration value from the file. + + # [run] + ('branch', 'run:branch', 'boolean'), + ('concurrency', 'run:concurrency'), + ('cover_pylib', 'run:cover_pylib', 'boolean'), + ('data_file', 'run:data_file'), + ('debug', 'run:debug', 'list'), + ('include', 'run:include', 'list'), + ('note', 'run:note'), + ('omit', 'run:omit', 'list'), + ('parallel', 'run:parallel', 'boolean'), + ('plugins', 'run:plugins', 'list'), + ('source', 'run:source', 'list'), + ('timid', 'run:timid', 'boolean'), + + # [report] + ('exclude_list', 'report:exclude_lines', 'regexlist'), + ('fail_under', 'report:fail_under', 'int'), + ('ignore_errors', 'report:ignore_errors', 'boolean'), + ('include', 'report:include', 'list'), + ('omit', 'report:omit', 'list'), + ('partial_always_list', 'report:partial_branches_always', 'regexlist'), + ('partial_list', 'report:partial_branches', 'regexlist'), + ('precision', 'report:precision', 'int'), + ('show_missing', 'report:show_missing', 'boolean'), + ('skip_covered', 'report:skip_covered', 'boolean'), + + # [html] + ('extra_css', 'html:extra_css'), + ('html_dir', 'html:directory'), + ('html_title', 'html:title'), + + # [xml] + ('xml_output', 'xml:output'), + ('xml_package_depth', 'xml:package_depth', 'int'), + ] + + def _set_attr_from_config_option(self, cp, attr, where, type_=''): + """Set an attribute on self if it exists in the ConfigParser.""" + section, option = where.split(":") + if cp.has_option(section, option): + method = getattr(cp, 'get' + type_) + setattr(self, attr, method(section, option)) + + def get_plugin_options(self, plugin): + """Get a dictionary of options for the plugin named `plugin`.""" + return self.plugin_options.get(plugin, {}) + + def set_option(self, option_name, value): + """Set an option in the configuration. + + `option_name` is a colon-separated string indicating the section and + option name. For example, the ``branch`` option in the ``[run]`` + section of the config file would be indicated with `"run:branch"`. + + `value` is the new value for the option. + + """ + + # Check all the hard-coded options. + for option_spec in self.CONFIG_FILE_OPTIONS: + attr, where = option_spec[:2] + if where == option_name: + setattr(self, attr, value) + return + + # See if it's a plugin option. + plugin_name, _, key = option_name.partition(":") + if key and plugin_name in self.plugins: + self.plugin_options.setdefault(plugin_name, {})[key] = value + return + + # If we get here, we didn't find the option. + raise CoverageException("No such option: %r" % option_name) + + def get_option(self, option_name): + """Get an option from the configuration. + + `option_name` is a colon-separated string indicating the section and + option name. For example, the ``branch`` option in the ``[run]`` + section of the config file would be indicated with `"run:branch"`. + + Returns the value of the option. + + """ + + # Check all the hard-coded options. + for option_spec in self.CONFIG_FILE_OPTIONS: + attr, where = option_spec[:2] + if where == option_name: + return getattr(self, attr) + + # See if it's a plugin option. + plugin_name, _, key = option_name.partition(":") + if key and plugin_name in self.plugins: + return self.plugin_options.get(plugin_name, {}).get(key) + + # If we get here, we didn't find the option. + raise CoverageException("No such option: %r" % option_name)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/DebugClients/Python/coverage/control.py Sun Oct 16 12:18:42 2016 +0200 @@ -0,0 +1,1199 @@ +# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt + +"""Core control stuff for coverage.py.""" + +import atexit +import inspect +import os +import platform +import re +import sys +import traceback + +from coverage import env, files +from coverage.annotate import AnnotateReporter +from coverage.backward import string_class, iitems +from coverage.collector import Collector +from coverage.config import CoverageConfig +from coverage.data import CoverageData, CoverageDataFiles +from coverage.debug import DebugControl +from coverage.files import TreeMatcher, FnmatchMatcher +from coverage.files import PathAliases, find_python_files, prep_patterns +from coverage.files import ModuleMatcher, abs_file +from coverage.html import HtmlReporter +from coverage.misc import CoverageException, bool_or_none, join_regex +from coverage.misc import file_be_gone, isolate_module +from coverage.monkey import patch_multiprocessing +from coverage.plugin import FileReporter +from coverage.plugin_support import Plugins +from coverage.python import PythonFileReporter +from coverage.results import Analysis, Numbers +from coverage.summary import SummaryReporter +from coverage.xmlreport import XmlReporter + +os = isolate_module(os) + +# Pypy has some unusual stuff in the "stdlib". Consider those locations +# when deciding where the stdlib is. +try: + import _structseq +except ImportError: + _structseq = None + + +class Coverage(object): + """Programmatic access to coverage.py. + + To use:: + + from coverage import Coverage + + cov = Coverage() + cov.start() + #.. call your code .. + cov.stop() + cov.html_report(directory='covhtml') + + """ + def __init__( + self, data_file=None, data_suffix=None, cover_pylib=None, + auto_data=False, timid=None, branch=None, config_file=True, + source=None, omit=None, include=None, debug=None, + concurrency=None, + ): + """ + `data_file` is the base name of the data file to use, defaulting to + ".coverage". `data_suffix` is appended (with a dot) to `data_file` to + create the final file name. If `data_suffix` is simply True, then a + suffix is created with the machine and process identity included. + + `cover_pylib` is a boolean determining whether Python code installed + with the Python interpreter is measured. This includes the Python + standard library and any packages installed with the interpreter. + + If `auto_data` is true, then any existing data file will be read when + coverage measurement starts, and data will be saved automatically when + measurement stops. + + If `timid` is true, then a slower and simpler trace function will be + used. This is important for some environments where manipulation of + tracing functions breaks the faster trace function. + + If `branch` is true, then branch coverage will be measured in addition + to the usual statement coverage. + + `config_file` determines what configuration file to read: + + * If it is ".coveragerc", it is interpreted as if it were True, + for backward compatibility. + + * If it is a string, it is the name of the file to read. If the + file can't be read, it is an error. + + * If it is True, then a few standard files names are tried + (".coveragerc", "setup.cfg"). It is not an error for these files + to not be found. + + * If it is False, then no configuration file is read. + + `source` is a list of file paths or package names. Only code located + in the trees indicated by the file paths or package names will be + measured. + + `include` and `omit` are lists of file name patterns. Files that match + `include` will be measured, files that match `omit` will not. Each + will also accept a single string argument. + + `debug` is a list of strings indicating what debugging information is + desired. + + `concurrency` is a string indicating the concurrency library being used + in the measured code. Without this, coverage.py will get incorrect + results. Valid strings are "greenlet", "eventlet", "gevent", + "multiprocessing", or "thread" (the default). + + .. versionadded:: 4.0 + The `concurrency` parameter. + + """ + # Build our configuration from a number of sources: + # 1: defaults: + self.config = CoverageConfig() + + # 2: from the rcfile, .coveragerc or setup.cfg file: + if config_file: + did_read_rc = False + # Some API users were specifying ".coveragerc" to mean the same as + # True, so make it so. + if config_file == ".coveragerc": + config_file = True + specified_file = (config_file is not True) + if not specified_file: + config_file = ".coveragerc" + + did_read_rc = self.config.from_file(config_file) + + if not did_read_rc: + if specified_file: + raise CoverageException( + "Couldn't read '%s' as a config file" % config_file + ) + self.config.from_file("setup.cfg", section_prefix="coverage:") + + # 3: from environment variables: + env_data_file = os.environ.get('COVERAGE_FILE') + if env_data_file: + self.config.data_file = env_data_file + debugs = os.environ.get('COVERAGE_DEBUG') + if debugs: + self.config.debug.extend(debugs.split(",")) + + # 4: from constructor arguments: + self.config.from_args( + data_file=data_file, cover_pylib=cover_pylib, timid=timid, + branch=branch, parallel=bool_or_none(data_suffix), + source=source, omit=omit, include=include, debug=debug, + concurrency=concurrency, + ) + + self._debug_file = None + self._auto_data = auto_data + self._data_suffix = data_suffix + + # The matchers for _should_trace. + self.source_match = None + self.source_pkgs_match = None + self.pylib_match = self.cover_match = None + self.include_match = self.omit_match = None + + # Is it ok for no data to be collected? + self._warn_no_data = True + self._warn_unimported_source = True + + # A record of all the warnings that have been issued. + self._warnings = [] + + # Other instance attributes, set later. + self.omit = self.include = self.source = None + self.source_pkgs = None + self.data = self.data_files = self.collector = None + self.plugins = None + self.pylib_dirs = self.cover_dirs = None + self.data_suffix = self.run_suffix = None + self._exclude_re = None + self.debug = None + + # State machine variables: + # Have we initialized everything? + self._inited = False + # Have we started collecting and not stopped it? + self._started = False + # Have we measured some data and not harvested it? + self._measured = False + + def _init(self): + """Set all the initial state. + + This is called by the public methods to initialize state. This lets us + construct a :class:`Coverage` object, then tweak its state before this + function is called. + + """ + if self._inited: + return + + # Create and configure the debugging controller. COVERAGE_DEBUG_FILE + # is an environment variable, the name of a file to append debug logs + # to. + if self._debug_file is None: + debug_file_name = os.environ.get("COVERAGE_DEBUG_FILE") + if debug_file_name: + self._debug_file = open(debug_file_name, "a") + else: + self._debug_file = sys.stderr + self.debug = DebugControl(self.config.debug, self._debug_file) + + # Load plugins + self.plugins = Plugins.load_plugins(self.config.plugins, self.config, self.debug) + + # _exclude_re is a dict that maps exclusion list names to compiled + # regexes. + self._exclude_re = {} + self._exclude_regex_stale() + + files.set_relative_directory() + + # The source argument can be directories or package names. + self.source = [] + self.source_pkgs = [] + for src in self.config.source or []: + if os.path.exists(src): + self.source.append(files.canonical_filename(src)) + else: + self.source_pkgs.append(src) + + self.omit = prep_patterns(self.config.omit) + self.include = prep_patterns(self.config.include) + + concurrency = self.config.concurrency + if concurrency == "multiprocessing": + patch_multiprocessing() + concurrency = None + + self.collector = Collector( + should_trace=self._should_trace, + check_include=self._check_include_omit_etc, + timid=self.config.timid, + branch=self.config.branch, + warn=self._warn, + concurrency=concurrency, + ) + + # Early warning if we aren't going to be able to support plugins. + if self.plugins.file_tracers and not self.collector.supports_plugins: + self._warn( + "Plugin file tracers (%s) aren't supported with %s" % ( + ", ".join( + plugin._coverage_plugin_name + for plugin in self.plugins.file_tracers + ), + self.collector.tracer_name(), + ) + ) + for plugin in self.plugins.file_tracers: + plugin._coverage_enabled = False + + # Suffixes are a bit tricky. We want to use the data suffix only when + # collecting data, not when combining data. So we save it as + # `self.run_suffix` now, and promote it to `self.data_suffix` if we + # find that we are collecting data later. + if self._data_suffix or self.config.parallel: + if not isinstance(self._data_suffix, string_class): + # if data_suffix=True, use .machinename.pid.random + self._data_suffix = True + else: + self._data_suffix = None + self.data_suffix = None + self.run_suffix = self._data_suffix + + # Create the data file. We do this at construction time so that the + # data file will be written into the directory where the process + # started rather than wherever the process eventually chdir'd to. + self.data = CoverageData(debug=self.debug) + self.data_files = CoverageDataFiles(basename=self.config.data_file, warn=self._warn) + + # The directories for files considered "installed with the interpreter". + self.pylib_dirs = set() + if not self.config.cover_pylib: + # Look at where some standard modules are located. That's the + # indication for "installed with the interpreter". In some + # environments (virtualenv, for example), these modules may be + # spread across a few locations. Look at all the candidate modules + # we've imported, and take all the different ones. + for m in (atexit, inspect, os, platform, re, _structseq, traceback): + if m is not None and hasattr(m, "__file__"): + self.pylib_dirs.add(self._canonical_dir(m)) + if _structseq and not hasattr(_structseq, '__file__'): + # PyPy 2.4 has no __file__ in the builtin modules, but the code + # objects still have the file names. So dig into one to find + # the path to exclude. + structseq_new = _structseq.structseq_new + try: + structseq_file = structseq_new.func_code.co_filename + except AttributeError: + structseq_file = structseq_new.__code__.co_filename + self.pylib_dirs.add(self._canonical_dir(structseq_file)) + + # To avoid tracing the coverage.py code itself, we skip anything + # located where we are. + self.cover_dirs = [self._canonical_dir(__file__)] + if env.TESTING: + # When testing, we use PyContracts, which should be considered + # part of coverage.py, and it uses six. Exclude those directories + # just as we exclude ourselves. + import contracts, six + for mod in [contracts, six]: + self.cover_dirs.append(self._canonical_dir(mod)) + + # Set the reporting precision. + Numbers.set_precision(self.config.precision) + + atexit.register(self._atexit) + + self._inited = True + + # Create the matchers we need for _should_trace + if self.source or self.source_pkgs: + self.source_match = TreeMatcher(self.source) + self.source_pkgs_match = ModuleMatcher(self.source_pkgs) + else: + if self.cover_dirs: + self.cover_match = TreeMatcher(self.cover_dirs) + if self.pylib_dirs: + self.pylib_match = TreeMatcher(self.pylib_dirs) + if self.include: + self.include_match = FnmatchMatcher(self.include) + if self.omit: + self.omit_match = FnmatchMatcher(self.omit) + + # The user may want to debug things, show info if desired. + wrote_any = False + if self.debug.should('config'): + config_info = sorted(self.config.__dict__.items()) + self.debug.write_formatted_info("config", config_info) + wrote_any = True + + if self.debug.should('sys'): + self.debug.write_formatted_info("sys", self.sys_info()) + for plugin in self.plugins: + header = "sys: " + plugin._coverage_plugin_name + info = plugin.sys_info() + self.debug.write_formatted_info(header, info) + wrote_any = True + + if wrote_any: + self.debug.write_formatted_info("end", ()) + + def _canonical_dir(self, morf): + """Return the canonical directory of the module or file `morf`.""" + morf_filename = PythonFileReporter(morf, self).filename + return os.path.split(morf_filename)[0] + + def _source_for_file(self, filename): + """Return the source file for `filename`. + + Given a file name being traced, return the best guess as to the source + file to attribute it to. + + """ + if filename.endswith(".py"): + # .py files are themselves source files. + return filename + + elif filename.endswith((".pyc", ".pyo")): + # Bytecode files probably have source files near them. + py_filename = filename[:-1] + if os.path.exists(py_filename): + # Found a .py file, use that. + return py_filename + if env.WINDOWS: + # On Windows, it could be a .pyw file. + pyw_filename = py_filename + "w" + if os.path.exists(pyw_filename): + return pyw_filename + # Didn't find source, but it's probably the .py file we want. + return py_filename + + elif filename.endswith("$py.class"): + # Jython is easy to guess. + return filename[:-9] + ".py" + + # No idea, just use the file name as-is. + return filename + + def _name_for_module(self, module_globals, filename): + """Get the name of the module for a set of globals and file name. + + For configurability's sake, we allow __main__ modules to be matched by + their importable name. + + If loaded via runpy (aka -m), we can usually recover the "original" + full dotted module name, otherwise, we resort to interpreting the + file name to get the module's name. In the case that the module name + can't be determined, None is returned. + + """ + dunder_name = module_globals.get('__name__', None) + + if isinstance(dunder_name, str) and dunder_name != '__main__': + # This is the usual case: an imported module. + return dunder_name + + loader = module_globals.get('__loader__', None) + for attrname in ('fullname', 'name'): # attribute renamed in py3.2 + if hasattr(loader, attrname): + fullname = getattr(loader, attrname) + else: + continue + + if isinstance(fullname, str) and fullname != '__main__': + # Module loaded via: runpy -m + return fullname + + # Script as first argument to Python command line. + inspectedname = inspect.getmodulename(filename) + if inspectedname is not None: + return inspectedname + else: + return dunder_name + + def _should_trace_internal(self, filename, frame): + """Decide whether to trace execution in `filename`, with a reason. + + This function is called from the trace function. As each new file name + is encountered, this function determines whether it is traced or not. + + Returns a FileDisposition object. + + """ + original_filename = filename + disp = _disposition_init(self.collector.file_disposition_class, filename) + + def nope(disp, reason): + """Simple helper to make it easy to return NO.""" + disp.trace = False + disp.reason = reason + return disp + + # Compiled Python files have two file names: frame.f_code.co_filename is + # the file name at the time the .pyc was compiled. The second name is + # __file__, which is where the .pyc was actually loaded from. Since + # .pyc files can be moved after compilation (for example, by being + # installed), we look for __file__ in the frame and prefer it to the + # co_filename value. + dunder_file = frame.f_globals.get('__file__') + if dunder_file: + filename = self._source_for_file(dunder_file) + if original_filename and not original_filename.startswith('<'): + orig = os.path.basename(original_filename) + if orig != os.path.basename(filename): + # Files shouldn't be renamed when moved. This happens when + # exec'ing code. If it seems like something is wrong with + # the frame's file name, then just use the original. + filename = original_filename + + if not filename: + # Empty string is pretty useless. + return nope(disp, "empty string isn't a file name") + + if filename.startswith('memory:'): + return nope(disp, "memory isn't traceable") + + if filename.startswith('<'): + # Lots of non-file execution is represented with artificial + # file names like "<string>", "<doctest readme.txt[0]>", or + # "<exec_function>". Don't ever trace these executions, since we + # can't do anything with the data later anyway. + return nope(disp, "not a real file name") + + # pyexpat does a dumb thing, calling the trace function explicitly from + # C code with a C file name. + if re.search(r"[/\\]Modules[/\\]pyexpat.c", filename): + return nope(disp, "pyexpat lies about itself") + + # Jython reports the .class file to the tracer, use the source file. + if filename.endswith("$py.class"): + filename = filename[:-9] + ".py" + + canonical = files.canonical_filename(filename) + disp.canonical_filename = canonical + + # Try the plugins, see if they have an opinion about the file. + plugin = None + for plugin in self.plugins.file_tracers: + if not plugin._coverage_enabled: + continue + + try: + file_tracer = plugin.file_tracer(canonical) + if file_tracer is not None: + file_tracer._coverage_plugin = plugin + disp.trace = True + disp.file_tracer = file_tracer + if file_tracer.has_dynamic_source_filename(): + disp.has_dynamic_filename = True + else: + disp.source_filename = files.canonical_filename( + file_tracer.source_filename() + ) + break + except Exception: + self._warn( + "Disabling plugin %r due to an exception:" % ( + plugin._coverage_plugin_name + ) + ) + traceback.print_exc() + plugin._coverage_enabled = False + continue + else: + # No plugin wanted it: it's Python. + disp.trace = True + disp.source_filename = canonical + + if not disp.has_dynamic_filename: + if not disp.source_filename: + raise CoverageException( + "Plugin %r didn't set source_filename for %r" % + (plugin, disp.original_filename) + ) + reason = self._check_include_omit_etc_internal( + disp.source_filename, frame, + ) + if reason: + nope(disp, reason) + + return disp + + def _check_include_omit_etc_internal(self, filename, frame): + """Check a file name against the include, omit, etc, rules. + + Returns a string or None. String means, don't trace, and is the reason + why. None means no reason found to not trace. + + """ + modulename = self._name_for_module(frame.f_globals, filename) + + # If the user specified source or include, then that's authoritative + # about the outer bound of what to measure and we don't have to apply + # any canned exclusions. If they didn't, then we have to exclude the + # stdlib and coverage.py directories. + if self.source_match: + if self.source_pkgs_match.match(modulename): + if modulename in self.source_pkgs: + self.source_pkgs.remove(modulename) + return None # There's no reason to skip this file. + + if not self.source_match.match(filename): + return "falls outside the --source trees" + elif self.include_match: + if not self.include_match.match(filename): + return "falls outside the --include trees" + else: + # If we aren't supposed to trace installed code, then check if this + # is near the Python standard library and skip it if so. + if self.pylib_match and self.pylib_match.match(filename): + return "is in the stdlib" + + # We exclude the coverage.py code itself, since a little of it + # will be measured otherwise. + if self.cover_match and self.cover_match.match(filename): + return "is part of coverage.py" + + # Check the file against the omit pattern. + if self.omit_match and self.omit_match.match(filename): + return "is inside an --omit pattern" + + # No reason found to skip this file. + return None + + def _should_trace(self, filename, frame): + """Decide whether to trace execution in `filename`. + + Calls `_should_trace_internal`, and returns the FileDisposition. + + """ + disp = self._should_trace_internal(filename, frame) + if self.debug.should('trace'): + self.debug.write(_disposition_debug_msg(disp)) + return disp + + def _check_include_omit_etc(self, filename, frame): + """Check a file name against the include/omit/etc, rules, verbosely. + + Returns a boolean: True if the file should be traced, False if not. + + """ + reason = self._check_include_omit_etc_internal(filename, frame) + if self.debug.should('trace'): + if not reason: + msg = "Including %r" % (filename,) + else: + msg = "Not including %r: %s" % (filename, reason) + self.debug.write(msg) + + return not reason + + def _warn(self, msg): + """Use `msg` as a warning.""" + self._warnings.append(msg) + if self.debug.should('pid'): + msg = "[%d] %s" % (os.getpid(), msg) + sys.stderr.write("Coverage.py warning: %s\n" % msg) + + def get_option(self, option_name): + """Get an option from the configuration. + + `option_name` is a colon-separated string indicating the section and + option name. For example, the ``branch`` option in the ``[run]`` + section of the config file would be indicated with `"run:branch"`. + + Returns the value of the option. + + .. versionadded:: 4.0 + + """ + return self.config.get_option(option_name) + + def set_option(self, option_name, value): + """Set an option in the configuration. + + `option_name` is a colon-separated string indicating the section and + option name. For example, the ``branch`` option in the ``[run]`` + section of the config file would be indicated with ``"run:branch"``. + + `value` is the new value for the option. This should be a Python + value where appropriate. For example, use True for booleans, not the + string ``"True"``. + + As an example, calling:: + + cov.set_option("run:branch", True) + + has the same effect as this configuration file:: + + [run] + branch = True + + .. versionadded:: 4.0 + + """ + self.config.set_option(option_name, value) + + def use_cache(self, usecache): + """Obsolete method.""" + self._init() + if not usecache: + self._warn("use_cache(False) is no longer supported.") + + def load(self): + """Load previously-collected coverage data from the data file.""" + self._init() + self.collector.reset() + self.data_files.read(self.data) + + def start(self): + """Start measuring code coverage. + + Coverage measurement actually occurs in functions called after + :meth:`start` is invoked. Statements in the same scope as + :meth:`start` won't be measured. + + Once you invoke :meth:`start`, you must also call :meth:`stop` + eventually, or your process might not shut down cleanly. + + """ + self._init() + if self.run_suffix: + # Calling start() means we're running code, so use the run_suffix + # as the data_suffix when we eventually save the data. + self.data_suffix = self.run_suffix + if self._auto_data: + self.load() + + self.collector.start() + self._started = True + self._measured = True + + def stop(self): + """Stop measuring code coverage.""" + if self._started: + self.collector.stop() + self._started = False + + def _atexit(self): + """Clean up on process shutdown.""" + if self._started: + self.stop() + if self._auto_data: + self.save() + + def erase(self): + """Erase previously-collected coverage data. + + This removes the in-memory data collected in this session as well as + discarding the data file. + + """ + self._init() + self.collector.reset() + self.data.erase() + self.data_files.erase(parallel=self.config.parallel) + + def clear_exclude(self, which='exclude'): + """Clear the exclude list.""" + self._init() + setattr(self.config, which + "_list", []) + self._exclude_regex_stale() + + def exclude(self, regex, which='exclude'): + """Exclude source lines from execution consideration. + + A number of lists of regular expressions are maintained. Each list + selects lines that are treated differently during reporting. + + `which` determines which list is modified. The "exclude" list selects + lines that are not considered executable at all. The "partial" list + indicates lines with branches that are not taken. + + `regex` is a regular expression. The regex is added to the specified + list. If any of the regexes in the list is found in a line, the line + is marked for special treatment during reporting. + + """ + self._init() + excl_list = getattr(self.config, which + "_list") + excl_list.append(regex) + self._exclude_regex_stale() + + def _exclude_regex_stale(self): + """Drop all the compiled exclusion regexes, a list was modified.""" + self._exclude_re.clear() + + def _exclude_regex(self, which): + """Return a compiled regex for the given exclusion list.""" + if which not in self._exclude_re: + excl_list = getattr(self.config, which + "_list") + self._exclude_re[which] = join_regex(excl_list) + return self._exclude_re[which] + + def get_exclude_list(self, which='exclude'): + """Return a list of excluded regex patterns. + + `which` indicates which list is desired. See :meth:`exclude` for the + lists that are available, and their meaning. + + """ + self._init() + return getattr(self.config, which + "_list") + + def save(self): + """Save the collected coverage data to the data file.""" + self._init() + self.get_data() + self.data_files.write(self.data, suffix=self.data_suffix) + + def combine(self, data_paths=None): + """Combine together a number of similarly-named coverage data files. + + All coverage data files whose name starts with `data_file` (from the + coverage() constructor) will be read, and combined together into the + current measurements. + + `data_paths` is a list of files or directories from which data should + be combined. If no list is passed, then the data files from the + directory indicated by the current data file (probably the current + directory) will be combined. + + .. versionadded:: 4.0 + The `data_paths` parameter. + + """ + self._init() + self.get_data() + + aliases = None + if self.config.paths: + aliases = PathAliases() + for paths in self.config.paths.values(): + result = paths[0] + for pattern in paths[1:]: + aliases.add(pattern, result) + + self.data_files.combine_parallel_data(self.data, aliases=aliases, data_paths=data_paths) + + def get_data(self): + """Get the collected data and reset the collector. + + Also warn about various problems collecting data. + + Returns a :class:`coverage.CoverageData`, the collected coverage data. + + .. versionadded:: 4.0 + + """ + self._init() + if not self._measured: + return self.data + + self.collector.save_data(self.data) + + # If there are still entries in the source_pkgs list, then we never + # encountered those packages. + if self._warn_unimported_source: + for pkg in self.source_pkgs: + if pkg not in sys.modules: + self._warn("Module %s was never imported." % pkg) + elif not ( + hasattr(sys.modules[pkg], '__file__') and + os.path.exists(sys.modules[pkg].__file__) + ): + self._warn("Module %s has no Python source." % pkg) + else: + self._warn("Module %s was previously imported, but not measured." % pkg) + + # Find out if we got any data. + if not self.data and self._warn_no_data: + self._warn("No data was collected.") + + # Find files that were never executed at all. + for src in self.source: + for py_file in find_python_files(src): + py_file = files.canonical_filename(py_file) + + if self.omit_match and self.omit_match.match(py_file): + # Turns out this file was omitted, so don't pull it back + # in as unexecuted. + continue + + self.data.touch_file(py_file) + + if self.config.note: + self.data.add_run_info(note=self.config.note) + + self._measured = False + return self.data + + # Backward compatibility with version 1. + def analysis(self, morf): + """Like `analysis2` but doesn't return excluded line numbers.""" + f, s, _, m, mf = self.analysis2(morf) + return f, s, m, mf + + def analysis2(self, morf): + """Analyze a module. + + `morf` is a module or a file name. It will be analyzed to determine + its coverage statistics. The return value is a 5-tuple: + + * The file name for the module. + * A list of line numbers of executable statements. + * A list of line numbers of excluded statements. + * A list of line numbers of statements not run (missing from + execution). + * A readable formatted string of the missing line numbers. + + The analysis uses the source file itself and the current measured + coverage data. + + """ + self._init() + analysis = self._analyze(morf) + return ( + analysis.filename, + sorted(analysis.statements), + sorted(analysis.excluded), + sorted(analysis.missing), + analysis.missing_formatted(), + ) + + def _analyze(self, it): + """Analyze a single morf or code unit. + + Returns an `Analysis` object. + + """ + self.get_data() + if not isinstance(it, FileReporter): + it = self._get_file_reporter(it) + + return Analysis(self.data, it) + + def _get_file_reporter(self, morf): + """Get a FileReporter for a module or file name.""" + plugin = None + file_reporter = "python" + + if isinstance(morf, string_class): + abs_morf = abs_file(morf) + plugin_name = self.data.file_tracer(abs_morf) + if plugin_name: + plugin = self.plugins.get(plugin_name) + + if plugin: + file_reporter = plugin.file_reporter(abs_morf) + if file_reporter is None: + raise CoverageException( + "Plugin %r did not provide a file reporter for %r." % ( + plugin._coverage_plugin_name, morf + ) + ) + + if file_reporter == "python": + file_reporter = PythonFileReporter(morf, self) + + return file_reporter + + def _get_file_reporters(self, morfs=None): + """Get a list of FileReporters for a list of modules or file names. + + For each module or file name in `morfs`, find a FileReporter. Return + the list of FileReporters. + + If `morfs` is a single module or file name, this returns a list of one + FileReporter. If `morfs` is empty or None, then the list of all files + measured is used to find the FileReporters. + + """ + if not morfs: + morfs = self.data.measured_files() + + # Be sure we have a list. + if not isinstance(morfs, (list, tuple)): + morfs = [morfs] + + file_reporters = [] + for morf in morfs: + file_reporter = self._get_file_reporter(morf) + file_reporters.append(file_reporter) + + return file_reporters + + def report( + self, morfs=None, show_missing=None, ignore_errors=None, + file=None, # pylint: disable=redefined-builtin + omit=None, include=None, skip_covered=None, + ): + """Write a summary report to `file`. + + Each module in `morfs` is listed, with counts of statements, executed + statements, missing statements, and a list of lines missed. + + `include` is a list of file name patterns. Files that match will be + included in the report. Files matching `omit` will not be included in + the report. + + Returns a float, the total percentage covered. + + """ + self.get_data() + self.config.from_args( + ignore_errors=ignore_errors, omit=omit, include=include, + show_missing=show_missing, skip_covered=skip_covered, + ) + reporter = SummaryReporter(self, self.config) + return reporter.report(morfs, outfile=file) + + def annotate( + self, morfs=None, directory=None, ignore_errors=None, + omit=None, include=None, + ): + """Annotate a list of modules. + + Each module in `morfs` is annotated. The source is written to a new + file, named with a ",cover" suffix, with each line prefixed with a + marker to indicate the coverage of the line. Covered lines have ">", + excluded lines have "-", and missing lines have "!". + + See :meth:`report` for other arguments. + + """ + self.get_data() + self.config.from_args( + ignore_errors=ignore_errors, omit=omit, include=include + ) + reporter = AnnotateReporter(self, self.config) + reporter.report(morfs, directory=directory) + + def html_report(self, morfs=None, directory=None, ignore_errors=None, + omit=None, include=None, extra_css=None, title=None): + """Generate an HTML report. + + The HTML is written to `directory`. The file "index.html" is the + overview starting point, with links to more detailed pages for + individual modules. + + `extra_css` is a path to a file of other CSS to apply on the page. + It will be copied into the HTML directory. + + `title` is a text string (not HTML) to use as the title of the HTML + report. + + See :meth:`report` for other arguments. + + Returns a float, the total percentage covered. + + """ + self.get_data() + self.config.from_args( + ignore_errors=ignore_errors, omit=omit, include=include, + html_dir=directory, extra_css=extra_css, html_title=title, + ) + reporter = HtmlReporter(self, self.config) + return reporter.report(morfs) + + def xml_report( + self, morfs=None, outfile=None, ignore_errors=None, + omit=None, include=None, + ): + """Generate an XML report of coverage results. + + The report is compatible with Cobertura reports. + + Each module in `morfs` is included in the report. `outfile` is the + path to write the file to, "-" will write to stdout. + + See :meth:`report` for other arguments. + + Returns a float, the total percentage covered. + + """ + self.get_data() + self.config.from_args( + ignore_errors=ignore_errors, omit=omit, include=include, + xml_output=outfile, + ) + file_to_close = None + delete_file = False + if self.config.xml_output: + if self.config.xml_output == '-': + outfile = sys.stdout + else: + # Ensure that the output directory is created; done here + # because this report pre-opens the output file. + # HTMLReport does this using the Report plumbing because + # its task is more complex, being multiple files. + output_dir = os.path.dirname(self.config.xml_output) + if output_dir and not os.path.isdir(output_dir): + os.makedirs(output_dir) + open_kwargs = {} + if env.PY3: + open_kwargs['encoding'] = 'utf8' + outfile = open(self.config.xml_output, "w", **open_kwargs) + file_to_close = outfile + try: + reporter = XmlReporter(self, self.config) + return reporter.report(morfs, outfile=outfile) + except CoverageException: + delete_file = True + raise + finally: + if file_to_close: + file_to_close.close() + if delete_file: + file_be_gone(self.config.xml_output) + + def sys_info(self): + """Return a list of (key, value) pairs showing internal information.""" + + import coverage as covmod + + self._init() + + ft_plugins = [] + for ft in self.plugins.file_tracers: + ft_name = ft._coverage_plugin_name + if not ft._coverage_enabled: + ft_name += " (disabled)" + ft_plugins.append(ft_name) + + info = [ + ('version', covmod.__version__), + ('coverage', covmod.__file__), + ('cover_dirs', self.cover_dirs), + ('pylib_dirs', self.pylib_dirs), + ('tracer', self.collector.tracer_name()), + ('plugins.file_tracers', ft_plugins), + ('config_files', self.config.attempted_config_files), + ('configs_read', self.config.config_files), + ('data_path', self.data_files.filename), + ('python', sys.version.replace('\n', '')), + ('platform', platform.platform()), + ('implementation', platform.python_implementation()), + ('executable', sys.executable), + ('cwd', os.getcwd()), + ('path', sys.path), + ('environment', sorted( + ("%s = %s" % (k, v)) + for k, v in iitems(os.environ) + if k.startswith(("COV", "PY")) + )), + ('command_line', " ".join(getattr(sys, 'argv', ['???']))), + ] + + matcher_names = [ + 'source_match', 'source_pkgs_match', + 'include_match', 'omit_match', + 'cover_match', 'pylib_match', + ] + + for matcher_name in matcher_names: + matcher = getattr(self, matcher_name) + if matcher: + matcher_info = matcher.info() + else: + matcher_info = '-none-' + info.append((matcher_name, matcher_info)) + + return info + + +# FileDisposition "methods": FileDisposition is a pure value object, so it can +# be implemented in either C or Python. Acting on them is done with these +# functions. + +def _disposition_init(cls, original_filename): + """Construct and initialize a new FileDisposition object.""" + disp = cls() + disp.original_filename = original_filename + disp.canonical_filename = original_filename + disp.source_filename = None + disp.trace = False + disp.reason = "" + disp.file_tracer = None + disp.has_dynamic_filename = False + return disp + + +def _disposition_debug_msg(disp): + """Make a nice debug message of what the FileDisposition is doing.""" + if disp.trace: + msg = "Tracing %r" % (disp.original_filename,) + if disp.file_tracer: + msg += ": will be traced by %r" % disp.file_tracer + else: + msg = "Not tracing %r: %s" % (disp.original_filename, disp.reason) + return msg + + +def process_startup(): + """Call this at Python start-up to perhaps measure coverage. + + If the environment variable COVERAGE_PROCESS_START is defined, coverage + measurement is started. The value of the variable is the config file + to use. + + There are two ways to configure your Python installation to invoke this + function when Python starts: + + #. Create or append to sitecustomize.py to add these lines:: + + import coverage + coverage.process_startup() + + #. Create a .pth file in your Python installation containing:: + + import coverage; coverage.process_startup() + + Returns the :class:`Coverage` instance that was started, or None if it was + not started by this call. + + """ + cps = os.environ.get("COVERAGE_PROCESS_START") + if not cps: + # No request for coverage, nothing to do. + return None + + # This function can be called more than once in a process. This happens + # because some virtualenv configurations make the same directory visible + # twice in sys.path. This means that the .pth file will be found twice, + # and executed twice, executing this function twice. We set a global + # flag (an attribute on this function) to indicate that coverage.py has + # already been started, so we can avoid doing it twice. + # + # https://bitbucket.org/ned/coveragepy/issue/340/keyerror-subpy has more + # details. + + if hasattr(process_startup, "done"): + # We've annotated this function before, so we must have already + # started coverage.py in this process. Nothing to do. + return None + + process_startup.done = True + cov = Coverage(config_file=cps, auto_data=True) + cov.start() + cov._warn_no_data = False + cov._warn_unimported_source = False + + return cov
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/DebugClients/Python/coverage/data.py Sun Oct 16 12:18:42 2016 +0200 @@ -0,0 +1,768 @@ +# Licensed under the Apache License: http://www.apache.org/licenses/LICENSE-2.0 +# For details: https://bitbucket.org/ned/coveragepy/src/default/NOTICE.txt + +"""Coverage data for coverage.py.""" + +import glob +import itertools +import json +import optparse +import os +import os.path +import random +import re +import socket + +from coverage import env +from coverage.backward import iitems, string_class +from coverage.debug import _TEST_NAME_FILE +from coverage.files import PathAliases +from coverage.misc import CoverageException, file_be_gone, isolate_module + +os = isolate_module(os) + + +class CoverageData(object): + """Manages collected coverage data, including file storage. + + This class is the public supported API to the data coverage.py collects + during program execution. It includes information about what code was + executed. It does not include information from the analysis phase, to + determine what lines could have been executed, or what lines were not + executed. + + .. note:: + + The file format is not documented or guaranteed. It will change in + the future, in possibly complicated ways. Do not read coverage.py + data files directly. Use this API to avoid disruption. + + There are a number of kinds of data that can be collected: + + * **lines**: the line numbers of source lines that were executed. + These are always available. + + * **arcs**: pairs of source and destination line numbers for transitions + between source lines. These are only available if branch coverage was + used. + + * **file tracer names**: the module names of the file tracer plugins that + handled each file in the data. + + * **run information**: information about the program execution. This is + written during "coverage run", and then accumulated during "coverage + combine". + + Lines, arcs, and file tracer names are stored for each source file. File + names in this API are case-sensitive, even on platforms with + case-insensitive file systems. + + To read a coverage.py data file, use :meth:`read_file`, or + :meth:`read_fileobj` if you have an already-opened file. You can then + access the line, arc, or file tracer data with :meth:`lines`, :meth:`arcs`, + or :meth:`file_tracer`. Run information is available with + :meth:`run_infos`. + + The :meth:`has_arcs` method indicates whether arc data is available. You + can get a list of the files in the data with :meth:`measured_files`. + A summary of the line data is available from :meth:`line_counts`. As with + most Python containers, you can determine if there is any data at all by + using this object as a boolean value. + + + Most data files will be created by coverage.py itself, but you can use + methods here to create data files if you like. The :meth:`add_lines`, + :meth:`add_arcs`, and :meth:`add_file_tracers` methods add data, in ways + that are convenient for coverage.py. The :meth:`add_run_info` method adds + key-value pairs to the run information. + + To add a file without any measured data, use :meth:`touch_file`. + + You write to a named file with :meth:`write_file`, or to an already opened + file with :meth:`write_fileobj`. + + You can clear the data in memory with :meth:`erase`. Two data collections + can be combined by using :meth:`update` on one :class:`CoverageData`, + passing it the other. + + """ + + # The data file format is JSON, with these keys: + # + # * lines: a dict mapping file names to lists of line numbers + # executed:: + # + # { "file1": [17,23,45], "file2": [1,2,3], ... } + # + # * arcs: a dict mapping file names to lists of line number pairs:: + # + # { "file1": [[17,23], [17,25], [25,26]], ... } + # + # * file_tracers: a dict mapping file names to plugin names:: + # + # { "file1": "django.coverage", ... } + # + # * runs: a list of dicts of information about the coverage.py runs + # contributing to the data:: + # + # [ { "brief_sys": "CPython 2.7.10 Darwin" }, ... ] + # + # Only one of `lines` or `arcs` will be present: with branch coverage, data + # is stored as arcs. Without branch coverage, it is stored as lines. The + # line data is easily recovered from the arcs: it is all the first elements + # of the pairs that are greater than zero. + + def __init__(self, debug=None): + """Create a CoverageData. + + `debug` is a `DebugControl` object for writing debug messages. + + """ + self._debug = debug + + # A map from canonical Python source file name to a dictionary in + # which there's an entry for each line number that has been + # executed: + # + # { 'filename1.py': [12, 47, 1001], ... } + # + self._lines = None + + # A map from canonical Python source file name to a dictionary with an + # entry for each pair of line numbers forming an arc: + # + # { 'filename1.py': [(12,14), (47,48), ... ], ... } + # + self._arcs = None + + # A map from canonical source file name to a plugin module name: + # + # { 'filename1.py': 'django.coverage', ... } + # + self._file_tracers = {} + + # A list of dicts of information about the coverage.py runs. + self._runs = [] + + def __repr__(self): + return "<{klass} lines={lines} arcs={arcs} tracers={tracers} runs={runs}>".format( + klass=self.__class__.__name__, + lines="None" if self._lines is None else "{{{0}}}".format(len(self._lines)), + arcs="None" if self._arcs is None else "{{{0}}}".format(len(self._arcs)), + tracers="{{{0}}}".format(len(self._file_tracers)), + runs="[{0}]".format(len(self._runs)), + ) + + ## + ## Reading data + ## + + def has_arcs(self): + """Does this data have arcs? + + Arc data is only available if branch coverage was used during + collection. + + Returns a boolean. + + """ + return self._has_arcs() + + def lines(self, filename): + """Get the list of lines executed for a file. + + If the file was not measured, returns None. A file might be measured, + and have no lines executed, in which case an empty list is returned. + + If the file was executed, returns a list of integers, the line numbers + executed in the file. The list is in no particular order. + + """ + if self._arcs is not None: + arcs = self._arcs.get(filename) + if arcs is not None: + all_lines = itertools.chain.from_iterable(arcs) + return list(set(l for l in all_lines if l > 0)) + elif self._lines is not None: + return self._lines.get(filename) + return None + + def arcs(self, filename): + """Get the list of arcs executed for a file. + + If the file was not measured, returns None. A file might be measured, + and have no arcs executed, in which case an empty list is returned. + + If the file was executed, returns a list of 2-tuples of integers. Each + pair is a starting line number and an ending line number for a + transition from one line to another. The list is in no particular + order. + + Negative numbers have special meaning. If the starting line number is + -N, it represents an entry to the code object that starts at line N. + If the ending ling number is -N, it's an exit from the code object that + starts at line N. + + """ + if self._arcs is not None: + if filename in self._arcs: + return self._arcs[filename] + return None + + def file_tracer(self, filename): + """Get the plugin name of the file tracer for a file. + + Returns the name of the plugin that handles this file. If the file was + measured, but didn't use a plugin, then "" is returned. If the file + was not measured, then None is returned. + + """ + # Because the vast majority of files involve no plugin, we don't store + # them explicitly in self._file_tracers. Check the measured data + # instead to see if it was a known file with no plugin. + if filename in (self._arcs or self._lines or {}): + return self._file_tracers.get(filename, "") + return None + + def run_infos(self): + """Return the list of dicts of run information. + + For data collected during a single run, this will be a one-element + list. If data has been combined, there will be one element for each + original data file. + + """ + return self._runs + + def measured_files(self): + """A list of all files that had been measured.""" + return list(self._arcs or self._lines or {}) + + def line_counts(self, fullpath=False): + """Return a dict summarizing the line coverage data. + + Keys are based on the file names, and values are the number of executed + lines. If `fullpath` is true, then the keys are the full pathnames of + the files, otherwise they are the basenames of the files. + + Returns a dict mapping file names to counts of lines. + + """ + summ = {} + if fullpath: + filename_fn = lambda f: f + else: + filename_fn = os.path.basename + for filename in self.measured_files(): + summ[filename_fn(filename)] = len(self.lines(filename)) + return summ + + def __nonzero__(self): + return bool(self._lines or self._arcs) + + __bool__ = __nonzero__ + + def read_fileobj(self, file_obj): + """Read the coverage data from the given file object. + + Should only be used on an empty CoverageData object. + + """ + data = self._read_raw_data(file_obj) + + self._lines = self._arcs = None + + if 'lines' in data: + self._lines = data['lines'] + if 'arcs' in data: + self._arcs = dict( + (fname, [tuple(pair) for pair in arcs]) + for fname, arcs in iitems(data['arcs']) + ) + self._file_tracers = data.get('file_tracers', {}) + self._runs = data.get('runs', []) + + self._validate() + + def read_file(self, filename): + """Read the coverage data from `filename` into this object.""" + if self._debug and self._debug.should('dataio'): + self._debug.write("Reading data from %r" % (filename,)) + try: + with self._open_for_reading(filename) as f: + self.read_fileobj(f) + except Exception as exc: + raise CoverageException( + "Couldn't read data from '%s': %s: %s" % ( + filename, exc.__class__.__name__, exc, + ) + ) + + _GO_AWAY = "!coverage.py: This is a private format, don't read it directly!" + + @classmethod + def _open_for_reading(cls, filename): + """Open a file appropriately for reading data.""" + return open(filename, "r") + + @classmethod + def _read_raw_data(cls, file_obj): + """Read the raw data from a file object.""" + go_away = file_obj.read(len(cls._GO_AWAY)) + if go_away != cls._GO_AWAY: + raise CoverageException("Doesn't seem to be a coverage.py data file") + return json.load(file_obj) + + @classmethod + def _read_raw_data_file(cls, filename): + """Read the raw data from a file, for debugging.""" + with cls._open_for_reading(filename) as f: + return cls._read_raw_data(f) + + ## + ## Writing data + ## + + def add_lines(self, line_data): + """Add measured line data. + + `line_data` is a dictionary mapping file names to dictionaries:: + + { filename: { lineno: None, ... }, ...} + + """ + if self._debug and self._debug.should('dataop'): + self._debug.write("Adding lines: %d files, %d lines total" % ( + len(line_data), sum(len(lines) for lines in line_data.values()) + )) + if self._has_arcs(): + raise CoverageException("Can't add lines to existing arc data") + + if self._lines is None: + self._lines = {} + for filename, linenos in iitems(line_data): + if filename in self._lines: + new_linenos = set(self._lines[filename]) + new_linenos.update(linenos) + linenos = new_linenos + self._lines[filename] = list(linenos) + + self._validate() + + def add_arcs(self, arc_data): + """Add measured arc data. + + `arc_data` is a dictionary mapping file names to dictionaries:: + + { filename: { (l1,l2): None, ... }, ...} + + """ + if self._debug and self._debug.should('dataop'): + self._debug.write("Adding arcs: %d files, %d arcs total" % ( + len(arc_data), sum(len(arcs) for arcs in arc_data.values()) + )) + if self._has_lines(): + raise CoverageException("Can't add arcs to existing line data") + + if self._arcs is None: + self._arcs = {} + for filename, arcs in iitems(arc_data): + if filename in self._arcs: + new_arcs = set(self._arcs[filename]) + new_arcs.update(arcs) + arcs = new_arcs + self._arcs[filename] = list(arcs) + + self._validate() + + def add_file_tracers(self, file_tracers): + """Add per-file plugin information. + + `file_tracers` is { filename: plugin_name, ... } + + """ + if self._debug and self._debug.should('dataop'): + self._debug.write("Adding file tracers: %d files" % (len(file_tracers),)) + + existing_files = self._arcs or self._lines or {} + for filename, plugin_name in iitems(file_tracers): + if filename not in existing_files: + raise CoverageException( + "Can't add file tracer data for unmeasured file '%s'" % (filename,) + ) + existing_plugin = self._file_tracers.get(filename) + if existing_plugin is not None and plugin_name != existing_plugin: + raise CoverageException( + "Conflicting file tracer name for '%s': %r vs %r" % ( + filename, existing_plugin, plugin_name, + ) + ) + self._file_tracers[filename] = plugin_name + + self._validate() + + def add_run_info(self, **kwargs): + """Add information about the run. + + Keywords are arbitrary, and are stored in the run dictionary. Values + must be JSON serializable. You may use this function more than once, + but repeated keywords overwrite each other. + + """ + if self._debug and self._debug.should('dataop'): + self._debug.write("Adding run info: %r" % (kwargs,)) + if not self._runs: + self._runs = [{}] + self._runs[0].update(kwargs) + self._validate() + + def touch_file(self, filename): + """Ensure that `filename` appears in the data, empty if needed.""" + if self._debug and self._debug.should('dataop'): + self._debug.write("Touching %r" % (filename,)) + if not self._has_arcs() and not self._has_lines(): + raise CoverageException("Can't touch files in an empty CoverageData") + + if self._has_arcs(): + where = self._arcs + else: + where = self._lines + where.setdefault(filename, []) + + self._validate() + + def write_fileobj(self, file_obj): + """Write the coverage data to `file_obj`.""" + + # Create the file data. + file_data = {} + + if self._has_arcs(): + file_data['arcs'] = self._arcs + + if self._has_lines(): + file_data['lines'] = self._lines + + if self._file_tracers: + file_data['file_tracers'] = self._file_tracers + + if self._runs: + file_data['runs'] = self._runs + + # Write the data to the file. + file_obj.write(self._GO_AWAY) + json.dump(file_data, file_obj) + + def write_file(self, filename): + """Write the coverage data to `filename`.""" + if self._debug and self._debug.should('dataio'): + self._debug.write("Writing data to %r" % (filename,)) + with open(filename, 'w') as fdata: + self.write_fileobj(fdata) + + def erase(self): + """Erase the data in this object.""" + self._lines = None + self._arcs = None + self._file_tracers = {} + self._runs = [] + self._validate() + + def update(self, other_data, aliases=None): + """Update this data with data from another `CoverageData`. + + If `aliases` is provided, it's a `PathAliases` object that is used to + re-map paths to match the local machine's. + + """ + if self._has_lines() and other_data._has_arcs(): + raise CoverageException("Can't combine arc data with line data") + if self._has_arcs() and other_data._has_lines(): + raise CoverageException("Can't combine line data with arc data") + + aliases = aliases or PathAliases() + + # _file_tracers: only have a string, so they have to agree. + # Have to do these first, so that our examination of self._arcs and + # self._lines won't be confused by data updated from other_data. + for filename in other_data.measured_files(): + other_plugin = other_data.file_tracer(filename) + filename = aliases.map(filename) + this_plugin = self.file_tracer(filename) + if this_plugin is None: + if other_plugin: + self._file_tracers[filename] = other_plugin + elif this_plugin != other_plugin: + raise CoverageException( + "Conflicting file tracer name for '%s': %r vs %r" % ( + filename, this_plugin, other_plugin, + ) + ) + + # _runs: add the new runs to these runs. + self._runs.extend(other_data._runs) + + # _lines: merge dicts. + if other_data._has_lines(): + if self._lines is None: + self._lines = {} + for filename, file_lines in iitems(other_data._lines): + filename = aliases.map(filename) + if filename in self._lines: + lines = set(self._lines[filename]) + lines.update(file_lines) + file_lines = list(lines) + self._lines[filename] = file_lines + + # _arcs: merge dicts. + if other_data._has_arcs(): + if self._arcs is None: + self._arcs = {} + for filename, file_arcs in iitems(other_data._arcs): + filename = aliases.map(filename) + if filename in self._arcs: + arcs = set(self._arcs[filename]) + arcs.update(file_arcs) + file_arcs = list(arcs) + self._arcs[filename] = file_arcs + + self._validate() + + ## + ## Miscellaneous + ## + + def _validate(self): + """If we are in paranoid mode, validate that everything is right.""" + if env.TESTING: + self._validate_invariants() + + def _validate_invariants(self): + """Validate internal invariants.""" + # Only one of _lines or _arcs should exist. + assert not(self._has_lines() and self._has_arcs()), ( + "Shouldn't have both _lines and _arcs" + ) + + # _lines should be a dict of lists of ints. + if self._has_lines(): + for fname, lines in iitems(self._lines): + assert isinstance(fname, string_class), "Key in _lines shouldn't be %r" % (fname,) + assert all(isinstance(x, int) for x in lines), ( + "_lines[%r] shouldn't be %r" % (fname, lines) + ) + + # _arcs should be a dict of lists of pairs of ints. + if self._has_arcs(): + for fname, arcs in iitems(self._arcs): + assert isinstance(fname, string_class), "Key in _arcs shouldn't be %r" % (fname,) + assert all(isinstance(x, int) and isinstance(y, int) for x, y in arcs), ( + "_arcs[%r] shouldn't be %r" % (fname, arcs) + ) + + # _file_tracers should have only non-empty strings as values. + for fname, plugin in iitems(self._file_tracers): + assert isinstance(fname, string_class), ( + "Key in _file_tracers shouldn't be %r" % (fname,) + ) + assert plugin and isinstance(plugin, string_class), ( + "_file_tracers[%r] shoudn't be %r" % (fname, plugin) + ) + + # _runs should be a list of dicts. + for val in self._runs: + assert isinstance(val, dict) + for key in val: + assert isinstance(key, string_class), "Key in _runs shouldn't be %r" % (key,) + + def add_to_hash(self, filename, hasher): + """Contribute `filename`'s data to the `hasher`. + + `hasher` is a `coverage.misc.Hasher` instance to be updated with + the file's data. It should only get the results data, not the run + data. + + """ + if self._has_arcs(): + hasher.update(sorted(self.arcs(filename) or [])) + else: + hasher.update(sorted(self.lines(filename) or [])) + hasher.update(self.file_tracer(filename)) + + ## + ## Internal + ## + + def _has_lines(self): + """Do we have data in self._lines?""" + return self._lines is not None + + def _has_arcs(self): + """Do we have data in self._arcs?""" + return self._arcs is not None + + +class CoverageDataFiles(object): + """Manage the use of coverage data files.""" + + def __init__(self, basename=None, warn=None): + """Create a CoverageDataFiles to manage data files. + + `warn` is the warning function to use. + + `basename` is the name of the file to use for storing data. + + """ + self.warn = warn + # Construct the file name that will be used for data storage. + self.filename = os.path.abspath(basename or ".coverage") + + def erase(self, parallel=False): + """Erase the data from the file storage. + + If `parallel` is true, then also deletes data files created from the + basename by parallel-mode. + + """ + file_be_gone(self.filename) + if parallel: + data_dir, local = os.path.split(self.filename) + localdot = local + '.*' + pattern = os.path.join(os.path.abspath(data_dir), localdot) + for filename in glob.glob(pattern): + file_be_gone(filename) + + def read(self, data): + """Read the coverage data.""" + if os.path.exists(self.filename): + data.read_file(self.filename) + + def write(self, data, suffix=None): + """Write the collected coverage data to a file. + + `suffix` is a suffix to append to the base file name. This can be used + for multiple or parallel execution, so that many coverage data files + can exist simultaneously. A dot will be used to join the base name and + the suffix. + + """