AssistantEric/APIsManager.py

# -*- coding: utf-8 -*-

# Copyright (c) 2008 - 2010 Detlev Offenbach <detlev@die-offenbachs.de>
#

"""
Module implementing the APIsManager.
"""

import os

from PyQt4.QtCore import *
from PyQt4.QtSql import QSqlDatabase, QSqlQuery

from E5Gui.E5Application import e5App

import QScintilla.Lexers

from DocumentationTools.APIGenerator import APIGenerator
import Utilities.ModuleParser
import Utilities
import Preferences

WorkerStarted = QEvent.User + 2001
WorkerFinished = QEvent.User + 2002
WorkerAborted = QEvent.User + 2003

ApisNameProject = "__Project__"

class DbAPIsWorker(QThread):
    """
    Class implementing a worker thread to prepare the API database.
    """
    populate_api_stmt = """
        INSERT INTO api (acWord, context, fullContext, signature, fileId, pictureId) 
        VALUES (:acWord, :context, :fullContext, :signature, :fileId, :pictureId)
    """
    populate_del_api_stmt = """
        DELETE FROM api WHERE fileId = :fileId
    """
    populate_file_stmt = """
        INSERT INTO file (file) VALUES (:file)
    """
    update_file_stmt = """
        UPDATE file SET lastRead = :lastRead WHERE file = :file
    """

    file_loaded_stmt = """
        SELECT lastRead from file WHERE file = :file
    """
    file_id_stmt = """
        SELECT id FROM file WHERE file = :file
    """
    file_delete_id_stmt = """
        DELETE FROM file WHERE id = :id
    """
    
    def __init__(self, proxy, language, apiFiles, projectPath = ""):
        """
        Constructor
        
        @param proxy reference to the object that is proxied (DbAPIs)
        @param language language of the APIs object (string)
        @param apiFiles list of API files to process (list of strings)
        @param projectPath path of the project. Only needed, if the APIs
            are extracted out of the sources of a project. (string)
        """
        QThread.__init__(self)
        
        self.setTerminationEnabled(True)
        
        # Get the AC word separators for all of the languages that the editor supports.
        # This has to be before we create a new thread, because access to GUI elements 
        # is not allowed from non-gui threads.
        self.__wseps = {}
        for lang in QScintilla.Lexers.getSupportedLanguages():
            lexer = QScintilla.Lexers.getLexer(lang)
            if lexer is not None:
                self.__wseps[lang] = lexer.autoCompletionWordSeparators()
        
        self.__proxy = proxy
        self.__language = language
        self.__apiFiles = apiFiles[:]
        self.__aborted = False
        self.__projectPath = projectPath
    
    def __autoCompletionWordSeparators(self, language):
        """
        Private method to get the word separator characters for a language.
        
        @param language language of the APIs object (string)
        @return word separator characters (list of strings)
        """
        return self.__wseps.get(language, None)
    
    def abort(self):
        """
        Public method to ask the thread to stop.
        """
        self.__aborted = True
    
    def __loadApiFileIfNewer(self, apiFile):
        """
        Private method to load an API file, if it is newer than the one read
        into the database.
        
        @param apiFile filename of the raw API file (string)
        """
        db = QSqlDatabase.database(self.__language)
        db.transaction()
        try:
            query = QSqlQuery(db)
            query.prepare(self.file_loaded_stmt)
            query.bindValue(":file", apiFile)
            query.exec_()
            if query.next() and query.isValid():
                loadTime = QDateTime.fromString(query.value(0), Qt.ISODate)
            else:
                loadTime = QDateTime(1970, 1, 1, 0, 0)
            del query
        finally:
            db.commit()
        if self.__projectPath:
            modTime = QFileInfo(os.path.join(self.__projectPath, apiFile)).lastModified()
        else:
            modTime = QFileInfo(apiFile).lastModified()
        if loadTime < modTime:
            self.__loadApiFile(apiFile)
    
    def __loadApiFile(self, apiFile):
        """
        Private method to read a raw API file into the database.
        
        @param apiFile filename of the raw API file (string)
        """
        if self.__language == ApisNameProject:
            try:
                module = Utilities.ModuleParser.readModule(
                    os.path.join(self.__projectPath, apiFile), 
                    basename = self.__projectPath + os.sep, 
                    caching = False)
                language = module.getType()
                if language:
                    apiGenerator = APIGenerator(module)
                    apis = apiGenerator.genAPI(True, "", True)
                else:
                    apis = []
            except (IOError, ImportError):
                apis = []
        else:
            try:
                apis = Utilities.readEncodedFile(apiFile)[0].splitlines(True)
            except (IOError, UnicodeError):
                apis = []
            language = None
        
        if len(apis) > 0:
            self.__storeApis(apis, apiFile, language)
    
    def __storeApis(self, apis, apiFile, language):
        """
        Private method to put the API entries into the database.
        
        @param apis list of api entries (list of strings)
        @param apiFile filename of the file read to get the APIs (string)
        @param language programming language of the file of the APIs (string)
        """
        if language:
            wseps = self.__autoCompletionWordSeparators(language)
        else:
            wseps = self.__proxy.autoCompletionWordSeparators()
        if wseps is None:
            return

        db = QSqlDatabase.database(self.__language)
        db.transaction()
        try:
            query = QSqlQuery(db)
            # step 1: create entry in file table and get the ID
            query.prepare(self.populate_file_stmt)
            query.bindValue(":file", apiFile)
            query.exec_()
            query.prepare(self.file_id_stmt)
            query.bindValue(":file", apiFile)
            query.exec_()
            query.next()
            id = int(query.value(0))
            
            # step 2: delete all entries belonging to this file
            query.prepare(self.populate_del_api_stmt)
            query.bindValue(":fileId", id)
            query.exec_()
            
            # step 3: load the given api file
            query.prepare(self.populate_api_stmt)
            for api in apis:
                if self.__aborted:
                    break
                
                api = api.strip()
                if len(api) == 0:
                    continue
                
                b = api.find('(')
                if b == -1:
                    path = api
                    sig = ""
                else:
                    path = api[:b]
                    sig = api[b:]
                
                while len(path) > 0:
                    acWord = ""
                    context = ""
                    fullContext = ""
                    pictureId = ""
                    
                    # search for word separators
                    index = len(path)
                    while index > 0:
                        index -= 1
                        found = False
                        for wsep in wseps:
                            if path[:index].endswith(wsep):
                                found = True
                                break
                        if found:
                            if acWord == "":
                                # completion found
                                acWord = path[index:]
                                path = path[:(index - len(wsep))]
                                index = len(path)
                                fullContext = path
                                context = path
                                try:
                                    acWord, pictureId = acWord.split("?", 1)
                                except ValueError:
                                    pass
                            else:
                                context = path[index:]
                                break
                    # none found?
                    if acWord == "":
                        acWord = path
                        path = ""
                    
                    query.bindValue(":acWord", acWord)
                    query.bindValue(":context", context)
                    query.bindValue(":fullContext", fullContext)
                    query.bindValue(":signature", sig)
                    query.bindValue(":fileId", id)
                    query.bindValue(":pictureId", pictureId)
                    query.exec_()
                    
                    sig = ""
            
            if not self.__aborted:
                # step 4: update the file entry
                query.prepare(self.update_file_stmt)
                query.bindValue(":lastRead", QDateTime.currentDateTime())
                query.bindValue(":file", apiFile)
                query.exec_()
        finally:
            del query
            if self.__aborted:
                db.rollback()
            else:
                db.commit()
    
    def __deleteApiFile(self, apiFile):
        """
        Private method to delete all references to an api file.
        
        @param apiFile filename of the raw API file (string)
        """
        db = QSqlDatabase.database(self.__language)
        db.transaction()
        try:
            query = QSqlQuery(db)
            
            # step 1: get the ID belonging to the api file
            query.prepare(self.file_id_stmt)
            query.bindValue(":file", apiFile)
            query.exec_()
            query.next()
            id = int(query.value(0))
            
            # step 2: delete all api entries belonging to this file
            query.prepare(self.populate_del_api_stmt)
            query.bindValue(":fileId", id)
            query.exec_()
            
            # step 3: delete the file entry
            query.prepare(self.file_delete_id_stmt)
            query.bindValue(":id", id)
            query.exec_()
        finally:
            del query
            db.commit()
    
    def run(self):
        """
        Public method to perform the threads work.
        """
        QCoreApplication.postEvent(self.__proxy, QEvent(QEvent.Type(WorkerStarted)))
        
        db = QSqlDatabase.database(self.__language)
        if db.isValid() and db.isOpen():
            # step 1: remove API files not wanted any longer
            loadedApiFiles = self.__proxy.getApiFiles()
            for apiFile in loadedApiFiles:
                if not self.__aborted and apiFile not in self.__apiFiles:
                    self.__deleteApiFile(apiFile)
            
            # step 2: (re-)load api files
            for apiFile in self.__apiFiles:
                if not self.__aborted:
                    self.__loadApiFileIfNewer(apiFile)
        
        if self.__aborted:
            QCoreApplication.postEvent(self.__proxy, QEvent(QEvent.Type(WorkerAborted)))
        else:
            QCoreApplication.postEvent(self.__proxy, QEvent(QEvent.Type(WorkerFinished)))

class DbAPIs(QObject):
    """
    Class implementing an API storage entity.
    
    @signal apiPreparationFinished() emitted after the API preparation has finished
    @signal apiPreparationStarted() emitted after the API preparation has started
    @signal apiPreparationCancelled() emitted after the API preparation has been 
            cancelled
    """
    DB_VERSION = 3
    
    create_mgmt_stmt = """
        CREATE TABLE mgmt
        (format INTEGER)
    """
    drop_mgmt_stmt = """DROP TABLE IF EXISTS mgmt"""
    
    create_api_stmt = """
        CREATE TABLE api
        (acWord TEXT, 
         context TEXT, 
         fullContext TEXT, 
         signature TEXT,
         fileId INTEGER,
         pictureId INTEGER,
         UNIQUE(acWord, fullContext, signature) ON CONFLICT IGNORE
        )
    """
    drop_api_stmt = """DROP TABLE IF EXISTS api"""
    
    create_file_stmt = """
        CREATE TABLE file
        (id INTEGER PRIMARY KEY AUTOINCREMENT,
         file TEXT UNIQUE ON CONFLICT IGNORE,
         lastRead TIMESTAMP DEFAULT CURRENT_TIMESTAMP
        )
    """
    drop_file_stmt = """DROP TABLE IF EXISTS file"""
    
    create_acWord_idx = """CREATE INDEX acWord_idx on api (acWord)"""
    drop_acWord_idx = """DROP INDEX IF EXISTS acWord_idx"""
    
    create_context_idx = """CREATE INDEX context_idx on api (context)"""
    drop_context_idx = """DROP INDEX IF EXISTS context_idx"""
    
    create_fullContext_idx = """CREATE INDEX fullContext_idx on api (fullContext)"""
    drop_fullContext_idx = """DROP INDEX IF EXISTS fullContext_idx"""
    
    create_file_idx = """CREATE INDEX file_idx on file (file)"""
    drop_file_idx = """DROP INDEX IF EXISTS file_idx"""

    api_files_stmt = """
        SELECT file FROM file WHERE file LIKE '%.api'
    """

    ac_stmt = """
        SELECT DISTINCT acWord, fullContext, pictureId FROM api 
        WHERE acWord GLOB :acWord
        ORDER BY acWord
    """
    ac_context_stmt = """
        SELECT DISTINCT acWord, fullContext, pictureId FROM api 
        WHERE context = :context
        ORDER BY acWord
    """
    ct_stmt = """
        SELECT DISTINCT acWord, signature, fullContext FROM api
        WHERE acWord = :acWord
    """
    ct_context_stmt = """
        SELECT DISTINCT acWord, signature, fullContext FROM api
        WHERE acWord = :acWord
        AND context = :context
    """
    ct_fullContext_stmt = """
        SELECT DISTINCT acWord, signature, fullContext FROM api
        WHERE acWord = :acWord
        AND fullContext = :fullContext
    """
    format_select_stmt = """
        SELECT format FROM mgmt
    """
    mgmt_insert_stmt = """
        INSERT INTO mgmt (format) VALUES ({0:d})
    """.format(DB_VERSION)
    
    def __init__(self, language, parent = None):
        """
        Constructor
        
        @param language language of the APIs object (string)
        @param parent reference to the parent object (QObject)
        """
        QObject.__init__(self, parent)
        self.setObjectName("DbAPIs_{0}".format(language))
        
        self.__inPreparation = False
        self.__worker = None
        self.__workerQueue = []
        
        self.__language = language
        if self.__language == ApisNameProject:
            self.__initAsProject()
        else:
            self.__initAsLanguage()
    
    def __initAsProject(self):
        """
        Private method to initialize as a project API object.
        """
        self.__lexer = None
        
        self.__project = e5App().getObject("Project")
        self.connect(self.__project, SIGNAL("projectOpened"), self.__projectOpened)
        self.connect(self.__project, SIGNAL("newProject"), self.__projectOpened)
        self.connect(self.__project, SIGNAL("projectClosed"), self.__projectClosed)
        if self.__project.isOpen():
            self.__projectOpened()
    
    def __initAsLanguage(self):
        """
        Private method to initialize as a language API object.
        """
        if self.__language in ["Python", "Python3"]:
            self.__discardFirst = "self"
        else:
            self.__discardFirst = ""
        self.__lexer = QScintilla.Lexers.getLexer(self.__language)
        self.__apifiles = Preferences.getEditorAPI(self.__language)
        self.__apifiles.sort()
        if self.__lexer is not None:
            self.__openAPIs()
    
    def _apiDbName(self):
        """
        Protected method to determine the name of the database file.
        
        @return name of the database file (string)
        """
        if self.__language == ApisNameProject:
            return os.path.join(self.__project.getProjectManagementDir(), 
                                "project-apis.db")
        else:
            apiDir = os.path.join(Utilities.getConfigDir(), "APIs")
            if not os.path.exists(apiDir):
                os.makedirs(apiDir)
            return os.path.join(apiDir, "{0}-api.db".format(self.__language))
    
    def close(self):
        """
        Public method to close the database.
        """
        self.__workerQueue = []
        if self.__worker is not None:
            self.__worker.abort()
            if self.__worker is not None:
                self.__worker.wait(5000)
                if self.__worker is not None and \
                   not self.__worker.isFinished():
                    self.__worker.terminate()
                    if self.__worker is not None:
                        self.__worker.wait(5000)
        
        QSqlDatabase.database(self.__language).close()
        QSqlDatabase.removeDatabase(self.__language)
        
    def __openApiDb(self):
        """
        Private method to open the API database.
        """
        db = QSqlDatabase.database(self.__language, False)
        if not db.isValid():
            # the database connection is a new one
            db = QSqlDatabase.addDatabase("QSQLITE", self.__language)
            db.setDatabaseName(self._apiDbName())
            db.open()
    
    def __createApiDB(self):
        """
        Private method to create an API database.
        """
        db = QSqlDatabase.database(self.__language)
        db.transaction()
        try:
            query = QSqlQuery(db)
            # step 1: drop old tables
            query.exec_(self.drop_mgmt_stmt)
            query.exec_(self.drop_api_stmt)
            query.exec_(self.drop_file_stmt)
            # step 2: drop old indices
            query.exec_(self.drop_acWord_idx)
            query.exec_(self.drop_context_idx)
            query.exec_(self.drop_fullContext_idx)
            query.exec_(self.drop_file_idx)
            # step 3: create tables
            query.exec_(self.create_api_stmt)
            query.exec_(self.create_file_stmt)
            query.exec_(self.create_mgmt_stmt)
            query.exec_(self.mgmt_insert_stmt)
            # step 4: create indices
            query.exec_(self.create_acWord_idx)
            query.exec_(self.create_context_idx)
            query.exec_(self.create_fullContext_idx)
            query.exec_(self.create_file_idx)
        finally:
            del query
            db.commit()

    def getApiFiles(self):
        """
        Public method to get a list of API files loaded into the database.
        
        @return list of API filenames (list of strings)
        """
        apiFiles = []
        
        db = QSqlDatabase.database(self.__language)
        db.transaction()
        try:
            query = QSqlQuery(db)
            query.exec_(self.api_files_stmt)
            while query.next():
                apiFiles.append(query.value(0))
        finally:
            del query
            db.commit()
        
        return apiFiles
    
    def __isPrepared(self):
        """
        Private method to check, if the database has been prepared.
        
        @return flag indicating the prepared status (boolean)
        """
        db = QSqlDatabase.database(self.__language)
        prepared = len(db.tables()) > 0
        if prepared:
            db.transaction()
            prepared = False
            try:
                query = QSqlQuery(db)
                ok = query.exec_(self.format_select_stmt)
                if ok:
                    query.next()
                    format = int(query.value(0))
                    if format >= self.DB_VERSION:
                        prepared = True
            finally:
                del query
                db.commit()
        return prepared
    
    def getCompletions(self, start = None, context = None):
        """
        Public method to determine the possible completions.
        
        @keyparam start string giving the start of the word to be 
            completed (string)
        @keyparam context string giving the context (e.g. classname)
            to be completed (string)
        @return list of dictionaries with possible completions (key 'completion'
            contains the completion (string), key 'context'
            contains the context (string) and key 'pictureId'
            contains the ID of the icon to be shown (string))
        """
        completions = []
        
        db = QSqlDatabase.database(self.__language)
        if db.isOpen() and not self.__inPreparation:
            db.transaction()
            try:
                query = None
                
                if start is not None:
                    query = QSqlQuery(db)
                    query.prepare(self.ac_stmt)
                    query.bindValue(":acWord", start + '*')
                elif context is not None:
                    query = QSqlQuery(db)
                    query.prepare(self.ac_context_stmt)
                    query.bindValue(":context", context)
                
                if query is not None:
                    query.exec_()
                    while query.next():
                        completions.append({"completion" : query.value(0), 
                                            "context" : query.value(1), 
                                            "pictureId" : query.value(2)})
                    del query
            finally:
                db.commit()
        
        return completions
    
    def getCalltips(self, acWord, commas, context = None, fullContext = None, 
                    showContext = True):
        """
        Public method to determine the calltips.
        
        @param acWord function to get calltips for (string)
        @param commas minimum number of commas contained in the calltip (integer)
        @param context string giving the context (e.g. classname) (string)
        @param fullContext string giving the full context (string)
        @param showContext flag indicating to show the calltip context (boolean)
        @return list of calltips (list of string)
        """
        calltips = []
        
        db = QSqlDatabase.database(self.__language)
        if db.isOpen() and not self.__inPreparation:
            if self.autoCompletionWordSeparators():
                contextSeparator = self.autoCompletionWordSeparators()[0]
            else:
                contextSeparator = " "
            db.transaction()
            try:
                query = QSqlQuery(db)
                if fullContext:
                    query.prepare(self.ct_fullContext_stmt)
                    query.bindValue(":fullContext", fullContext)
                elif context:
                    query.prepare(self.ct_context_stmt)
                    query.bindValue(":context", context)
                else:
                    query.prepare(self.ct_stmt)
                query.bindValue(":acWord", acWord)
                query.exec_()
                while query.next():
                    word = query.value(0)
                    sig = query.value(1)
                    fullCtx = query.value(2)
                    if sig:
                        if self.__discardFirst:
                            sig = "({0}".format(sig[1:])\
                                    .replace(self.__discardFirst, "", 1)\
                                    .strip(", \t\r\n")
                        if self.__enoughCommas(sig, commas):
                            if showContext:
                                calltips.append("{0}{1}{2}{3}".format(
                                    fullCtx, contextSeparator, word, sig))
                            else:
                                calltips.append("{0}{1}".format(word, sig))
                del query
            finally:
                db.commit()
            
            if context and len(calltips) == 0:
                # nothing found, try without a context
                calltips = self.getCalltips(acWord, commas, showContext = showContext)
        
        return calltips
    
    def __enoughCommas(self, s, commas):
        """
        Private method to determine, if the given string contains enough commas.
        
        @param s string to check (string)
        @param commas number of commas to check for (integer)
        @return flag indicating, that there are enough commas (boolean)
        """
        end = s.find(')')
        
        if end < 0:
            return False
        
        w = s[:end]
        return w.count(',') >= commas
    
    def __openAPIs(self):
        """
        Private method to open the API database.
        """
        self.__openApiDb()
        if not self.__isPrepared():
            self.__createApiDB()
        
        # prepare the database if neccessary
        self.prepareAPIs()
    
    def prepareAPIs(self, rawList = None):
        """
        Public method to prepare the APIs if neccessary.
        
        @keyparam rawList list of raw API files (list of strings)
        """
        if self.__inPreparation:
            return
        
        projectPath = ""
        if rawList:
            apiFiles = rawList[:]
        elif self.__language == ApisNameProject:
            apiFiles = self.__project.getSources()
            projectPath = self.__project.getProjectPath()
        else:
            apiFiles = Preferences.getEditorAPI(self.__language)
        self.__worker = DbAPIsWorker(self, self.__language, apiFiles, projectPath)
        self.__worker.start()
    
    def __processQueue(self):
        """
        Private slot to process the queue of files to load.
        """
        if self.__worker is not None and self.__worker.isFinished():
            self.__worker = None
        
        if self.__worker is None and len(self.__workerQueue) > 0:
            apiFiles = [self.__workerQueue.pop(0)]
            if self.__language == ApisNameProject:
                projectPath = self.__project.getProjectPath()
                apiFiles = [apiFiles[0].replace(projectPath + os.sep, "")]
            else:
                projectPath = ""
            self.__worker = DbAPIsWorker(self, self.__language, apiFiles, projectPath)
            self.__worker.start()
    
    def getLexer(self):
        """
        Public method to return a reference to our lexer object.
        
        @return reference to the lexer object (QScintilla.Lexers.Lexer)
        """
        return self.__lexer
    
    def autoCompletionWordSeparators(self):
        """
        Private method to get the word separator characters.
        
        @return word separator characters (list of strings)
        """
        if self.__lexer:
            return self.__lexer.autoCompletionWordSeparators()
        return None
    
    def event(self, evt):
        """
        Protected method to handle events from the worker thread.
        
        @param evt reference to the event object (QEvent)
        @return flag indicating, if the event was handled (boolean)
        """
        if evt.type() == WorkerStarted:
            self.__inPreparation = True
            self.emit(SIGNAL('apiPreparationStarted()'))
            return True
        
        elif evt.type() == WorkerFinished:
            self.__inPreparation = False
            self.emit(SIGNAL('apiPreparationFinished()'))
            QTimer.singleShot(0, self.__processQueue)
            return True
        
        elif evt.type() == WorkerAborted:
            self.__inPreparation = False
            self.emit(SIGNAL('apiPreparationCancelled()'))
            QTimer.singleShot(0, self.__processQueue)
            return True
        
        else:
            return QObject.event(self,  evt)
    
    ########################################################
    ## project related stuff below
    ########################################################
    
    def __projectOpened(self):
        """
        Private slot to perform actions after a project has been opened.
        """
        if self.__project.getProjectLanguage() in ["Python", "Python3"]:
            self.__discardFirst = "self"
        else:
            self.__discardFirst = ""
        self.__lexer = QScintilla.Lexers.getLexer(self.__project.getProjectLanguage())
        self.__openAPIs()
    
    def __projectClosed(self):
        """
        Private slot to perform actions after a project has been closed.
        """
        self.close()
    
    def editorSaved(self, filename):
        """
        Public slot to handle the editorSaved signal.
        
        @param filename name of the file that was saved (string)
        """
        if self.__project.isProjectSource(filename):
            self.__workerQueue.append(filename)
            self.__processQueue()

class APIsManager(QObject):
    """
    Class implementing the APIsManager class, which is the central store for
    API information used by autocompletion and calltips.
    """
    def __init__(self, parent = None):
        """
        Constructor
        
        @param parent reference to the parent object (QObject)
        """
        QObject.__init__(self, parent)
        self.setObjectName("APIsManager")
        
        # initialize the apis dictionary
        self.__apis = {}
    
    def reloadAPIs(self):
        """
        Public slot to reload the api information.
        """
        for api in list(self.__apis.values()):
            api and api.prepareAPIs()
    
    def getAPIs(self, language):
        """
        Public method to get an apis object for autocompletion/calltips.
        
        This method creates and loads an APIs object dynamically upon request. 
        This saves memory for languages, that might not be needed at the moment.
        
        @param language the language of the requested api object (string)
        @return the apis object (APIs)
        """
        try:
            return self.__apis[language]
        except KeyError:
            if language in QScintilla.Lexers.getSupportedLanguages() or \
               language == ApisNameProject:
                # create the api object
                self.__apis[language] = DbAPIs(language)
                return self.__apis[language]
            else:
                return None
    
    def deactivate(self):
        """
        Public method to perform actions upon deactivation.
        """
        for apiLang in self.__apis:
            self.__apis[apiLang].close()
            self.__apis[apiLang] = None