diff -r 3a4123edc944 -r 89cbc07f4bf0 AssistantEric/APIsManager.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/AssistantEric/APIsManager.py Sun Jan 17 19:22:18 2010 +0000 @@ -0,0 +1,866 @@ +# -*- coding: utf-8 -*- + +# Copyright (c) 2008 - 2010 Detlev Offenbach <detlev@die-offenbachs.de> +# + +""" +Module implementing the APIsManager. +""" + +import os + +from PyQt4.QtCore import * +from PyQt4.QtSql import QSqlDatabase, QSqlQuery + +from E5Gui.E5Application import e5App + +import QScintilla.Lexers + +from DocumentationTools.APIGenerator import APIGenerator +import Utilities.ModuleParser +import Utilities +import Preferences + +WorkerStarted = QEvent.User + 2001 +WorkerFinished = QEvent.User + 2002 +WorkerAborted = QEvent.User + 2003 + +ApisNameProject = "__Project__" + +class DbAPIsWorker(QThread): + """ + Class implementing a worker thread to prepare the API database. + """ + populate_api_stmt = """ + INSERT INTO api (acWord, context, fullContext, signature, fileId, pictureId) + VALUES (:acWord, :context, :fullContext, :signature, :fileId, :pictureId) + """ + populate_del_api_stmt = """ + DELETE FROM api WHERE fileId = :fileId + """ + populate_file_stmt = """ + INSERT INTO file (file) VALUES (:file) + """ + update_file_stmt = """ + UPDATE file SET lastRead = :lastRead WHERE file = :file + """ + + file_loaded_stmt = """ + SELECT lastRead from file WHERE file = :file + """ + file_id_stmt = """ + SELECT id FROM file WHERE file = :file + """ + file_delete_id_stmt = """ + DELETE FROM file WHERE id = :id + """ + + def __init__(self, proxy, language, apiFiles, projectPath = ""): + """ + Constructor + + @param proxy reference to the object that is proxied (DbAPIs) + @param language language of the APIs object (string) + @param apiFiles list of API files to process (list of strings) + @param projectPath path of the project. Only needed, if the APIs + are extracted out of the sources of a project. (string) + """ + QThread.__init__(self) + + self.setTerminationEnabled(True) + + # Get the AC word separators for all of the languages that the editor supports. + # This has to be before we create a new thread, because access to GUI elements + # is not allowed from non-gui threads. + self.__wseps = {} + for lang in QScintilla.Lexers.getSupportedLanguages(): + lexer = QScintilla.Lexers.getLexer(lang) + if lexer is not None: + self.__wseps[lang] = lexer.autoCompletionWordSeparators() + + self.__proxy = proxy + self.__language = language + self.__apiFiles = apiFiles[:] + self.__aborted = False + self.__projectPath = projectPath + + def __autoCompletionWordSeparators(self, language): + """ + Private method to get the word separator characters for a language. + + @param language language of the APIs object (string) + @return word separator characters (list of strings) + """ + return self.__wseps.get(language, None) + + def abort(self): + """ + Public method to ask the thread to stop. + """ + self.__aborted = True + + def __loadApiFileIfNewer(self, apiFile): + """ + Private method to load an API file, if it is newer than the one read + into the database. + + @param apiFile filename of the raw API file (string) + """ + db = QSqlDatabase.database(self.__language) + db.transaction() + try: + query = QSqlQuery(db) + query.prepare(self.file_loaded_stmt) + query.bindValue(":file", apiFile) + query.exec_() + if query.next() and query.isValid(): + loadTime = QDateTime.fromString(query.value(0), Qt.ISODate) + else: + loadTime = QDateTime(1970, 1, 1, 0, 0) + del query + finally: + db.commit() + if self.__projectPath: + modTime = QFileInfo(os.path.join(self.__projectPath, apiFile)).lastModified() + else: + modTime = QFileInfo(apiFile).lastModified() + if loadTime < modTime: + self.__loadApiFile(apiFile) + + def __loadApiFile(self, apiFile): + """ + Private method to read a raw API file into the database. + + @param apiFile filename of the raw API file (string) + """ + if self.__language == ApisNameProject: + try: + module = Utilities.ModuleParser.readModule( + os.path.join(self.__projectPath, apiFile), + basename = self.__projectPath + os.sep, + caching = False) + language = module.getType() + if language: + apiGenerator = APIGenerator(module) + apis = apiGenerator.genAPI(True, "", True) + else: + apis = [] + except (IOError, ImportError): + apis = [] + else: + try: + apis = Utilities.readEncodedFile(apiFile)[0].splitlines(True) + except (IOError, UnicodeError): + apis = [] + language = None + + if len(apis) > 0: + self.__storeApis(apis, apiFile, language) + + def __storeApis(self, apis, apiFile, language): + """ + Private method to put the API entries into the database. + + @param apis list of api entries (list of strings) + @param apiFile filename of the file read to get the APIs (string) + @param language programming language of the file of the APIs (string) + """ + if language: + wseps = self.__autoCompletionWordSeparators(language) + else: + wseps = self.__proxy.autoCompletionWordSeparators() + if wseps is None: + return + + db = QSqlDatabase.database(self.__language) + db.transaction() + try: + query = QSqlQuery(db) + # step 1: create entry in file table and get the ID + query.prepare(self.populate_file_stmt) + query.bindValue(":file", apiFile) + query.exec_() + query.prepare(self.file_id_stmt) + query.bindValue(":file", apiFile) + query.exec_() + query.next() + id = int(query.value(0)) + + # step 2: delete all entries belonging to this file + query.prepare(self.populate_del_api_stmt) + query.bindValue(":fileId", id) + query.exec_() + + # step 3: load the given api file + query.prepare(self.populate_api_stmt) + for api in apis: + if self.__aborted: + break + + api = api.strip() + if len(api) == 0: + continue + + b = api.find('(') + if b == -1: + path = api + sig = "" + else: + path = api[:b] + sig = api[b:] + + while len(path) > 0: + acWord = "" + context = "" + fullContext = "" + pictureId = "" + + # search for word separators + index = len(path) + while index > 0: + index -= 1 + found = False + for wsep in wseps: + if path[:index].endswith(wsep): + found = True + break + if found: + if acWord == "": + # completion found + acWord = path[index:] + path = path[:(index - len(wsep))] + index = len(path) + fullContext = path + context = path + try: + acWord, pictureId = acWord.split("?", 1) + except ValueError: + pass + else: + context = path[index:] + break + # none found? + if acWord == "": + acWord = path + path = "" + + query.bindValue(":acWord", acWord) + query.bindValue(":context", context) + query.bindValue(":fullContext", fullContext) + query.bindValue(":signature", sig) + query.bindValue(":fileId", id) + query.bindValue(":pictureId", pictureId) + query.exec_() + + sig = "" + + if not self.__aborted: + # step 4: update the file entry + query.prepare(self.update_file_stmt) + query.bindValue(":lastRead", QDateTime.currentDateTime()) + query.bindValue(":file", apiFile) + query.exec_() + finally: + del query + if self.__aborted: + db.rollback() + else: + db.commit() + + def __deleteApiFile(self, apiFile): + """ + Private method to delete all references to an api file. + + @param apiFile filename of the raw API file (string) + """ + db = QSqlDatabase.database(self.__language) + db.transaction() + try: + query = QSqlQuery(db) + + # step 1: get the ID belonging to the api file + query.prepare(self.file_id_stmt) + query.bindValue(":file", apiFile) + query.exec_() + query.next() + id = int(query.value(0)) + + # step 2: delete all api entries belonging to this file + query.prepare(self.populate_del_api_stmt) + query.bindValue(":fileId", id) + query.exec_() + + # step 3: delete the file entry + query.prepare(self.file_delete_id_stmt) + query.bindValue(":id", id) + query.exec_() + finally: + del query + db.commit() + + def run(self): + """ + Public method to perform the threads work. + """ + QCoreApplication.postEvent(self.__proxy, QEvent(QEvent.Type(WorkerStarted))) + + db = QSqlDatabase.database(self.__language) + if db.isValid() and db.isOpen(): + # step 1: remove API files not wanted any longer + loadedApiFiles = self.__proxy.getApiFiles() + for apiFile in loadedApiFiles: + if not self.__aborted and apiFile not in self.__apiFiles: + self.__deleteApiFile(apiFile) + + # step 2: (re-)load api files + for apiFile in self.__apiFiles: + if not self.__aborted: + self.__loadApiFileIfNewer(apiFile) + + if self.__aborted: + QCoreApplication.postEvent(self.__proxy, QEvent(QEvent.Type(WorkerAborted))) + else: + QCoreApplication.postEvent(self.__proxy, QEvent(QEvent.Type(WorkerFinished))) + +class DbAPIs(QObject): + """ + Class implementing an API storage entity. + + @signal apiPreparationFinished() emitted after the API preparation has finished + @signal apiPreparationStarted() emitted after the API preparation has started + @signal apiPreparationCancelled() emitted after the API preparation has been + cancelled + """ + DB_VERSION = 3 + + create_mgmt_stmt = """ + CREATE TABLE mgmt + (format INTEGER) + """ + drop_mgmt_stmt = """DROP TABLE IF EXISTS mgmt""" + + create_api_stmt = """ + CREATE TABLE api + (acWord TEXT, + context TEXT, + fullContext TEXT, + signature TEXT, + fileId INTEGER, + pictureId INTEGER, + UNIQUE(acWord, fullContext, signature) ON CONFLICT IGNORE + ) + """ + drop_api_stmt = """DROP TABLE IF EXISTS api""" + + create_file_stmt = """ + CREATE TABLE file + (id INTEGER PRIMARY KEY AUTOINCREMENT, + file TEXT UNIQUE ON CONFLICT IGNORE, + lastRead TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) + """ + drop_file_stmt = """DROP TABLE IF EXISTS file""" + + create_acWord_idx = """CREATE INDEX acWord_idx on api (acWord)""" + drop_acWord_idx = """DROP INDEX IF EXISTS acWord_idx""" + + create_context_idx = """CREATE INDEX context_idx on api (context)""" + drop_context_idx = """DROP INDEX IF EXISTS context_idx""" + + create_fullContext_idx = """CREATE INDEX fullContext_idx on api (fullContext)""" + drop_fullContext_idx = """DROP INDEX IF EXISTS fullContext_idx""" + + create_file_idx = """CREATE INDEX file_idx on file (file)""" + drop_file_idx = """DROP INDEX IF EXISTS file_idx""" + + api_files_stmt = """ + SELECT file FROM file WHERE file LIKE '%.api' + """ + + ac_stmt = """ + SELECT DISTINCT acWord, fullContext, pictureId FROM api + WHERE acWord GLOB :acWord + ORDER BY acWord + """ + ac_context_stmt = """ + SELECT DISTINCT acWord, fullContext, pictureId FROM api + WHERE context = :context + ORDER BY acWord + """ + ct_stmt = """ + SELECT DISTINCT acWord, signature, fullContext FROM api + WHERE acWord = :acWord + """ + ct_context_stmt = """ + SELECT DISTINCT acWord, signature, fullContext FROM api + WHERE acWord = :acWord + AND context = :context + """ + ct_fullContext_stmt = """ + SELECT DISTINCT acWord, signature, fullContext FROM api + WHERE acWord = :acWord + AND fullContext = :fullContext + """ + format_select_stmt = """ + SELECT format FROM mgmt + """ + mgmt_insert_stmt = """ + INSERT INTO mgmt (format) VALUES (%d) + """ % DB_VERSION + + def __init__(self, language, parent = None): + """ + Constructor + + @param language language of the APIs object (string) + @param parent reference to the parent object (QObject) + """ + QObject.__init__(self, parent) + self.setObjectName("DbAPIs_%s" % language) + + self.__inPreparation = False + self.__worker = None + self.__workerQueue = [] + + self.__language = language + if self.__language == ApisNameProject: + self.__initAsProject() + else: + self.__initAsLanguage() + + def __initAsProject(self): + """ + Private method to initialize as a project API object. + """ + self.__lexer = None + + self.__project = e5App().getObject("Project") + self.connect(self.__project, SIGNAL("projectOpened"), self.__projectOpened) + self.connect(self.__project, SIGNAL("newProject"), self.__projectOpened) + self.connect(self.__project, SIGNAL("projectClosed"), self.__projectClosed) + if self.__project.isOpen(): + self.__projectOpened() + + def __initAsLanguage(self): + """ + Private method to initialize as a language API object. + """ + if self.__language in ["Python", "Python3"]: + self.__discardFirst = "self" + else: + self.__discardFirst = "" + self.__lexer = QScintilla.Lexers.getLexer(self.__language) + self.__apifiles = Preferences.getEditorAPI(self.__language) + self.__apifiles.sort() + if self.__lexer is not None: + self.__openAPIs() + + def _apiDbName(self): + """ + Protected method to determine the name of the database file. + + @return name of the database file (string) + """ + if self.__language == ApisNameProject: + return os.path.join(self.__project.getProjectManagementDir(), + "project-apis.db") + else: + return os.path.join(Utilities.getConfigDir(), "%s-api.db" % self.__language) + + def close(self): + """ + Public method to close the database. + """ + self.__workerQueue = [] + if self.__worker is not None: + self.__worker.abort() + if self.__worker is not None: + self.__worker.wait(5000) + if self.__worker is not None and \ + not self.__worker.isFinished(): + self.__worker.terminate() + if self.__worker is not None: + self.__worker.wait(5000) + + QSqlDatabase.database(self.__language).close() + QSqlDatabase.removeDatabase(self.__language) + + def __openApiDb(self): + """ + Private method to open the API database. + """ + db = QSqlDatabase.database(self.__language, False) + if not db.isValid(): + # the database connection is a new one + db = QSqlDatabase.addDatabase("QSQLITE", self.__language) + db.setDatabaseName(self._apiDbName()) + db.open() + + def __createApiDB(self): + """ + Private method to create an API database. + """ + db = QSqlDatabase.database(self.__language) + db.transaction() + try: + query = QSqlQuery(db) + # step 1: drop old tables + query.exec_(self.drop_mgmt_stmt) + query.exec_(self.drop_api_stmt) + query.exec_(self.drop_file_stmt) + # step 2: drop old indices + query.exec_(self.drop_acWord_idx) + query.exec_(self.drop_context_idx) + query.exec_(self.drop_fullContext_idx) + query.exec_(self.drop_file_idx) + # step 3: create tables + query.exec_(self.create_api_stmt) + query.exec_(self.create_file_stmt) + query.exec_(self.create_mgmt_stmt) + query.exec_(self.mgmt_insert_stmt) + # step 4: create indices + query.exec_(self.create_acWord_idx) + query.exec_(self.create_context_idx) + query.exec_(self.create_fullContext_idx) + query.exec_(self.create_file_idx) + finally: + del query + db.commit() + + def getApiFiles(self): + """ + Public method to get a list of API files loaded into the database. + + @return list of API filenames (list of strings) + """ + apiFiles = [] + + db = QSqlDatabase.database(self.__language) + db.transaction() + try: + query = QSqlQuery(db) + query.exec_(self.api_files_stmt) + while query.next(): + apiFiles.append(query.value(0)) + finally: + del query + db.commit() + + return apiFiles + + def __isPrepared(self): + """ + Private method to check, if the database has been prepared. + + @return flag indicating the prepared status (boolean) + """ + db = QSqlDatabase.database(self.__language) + prepared = len(db.tables()) > 0 + if prepared: + db.transaction() + prepared = False + try: + query = QSqlQuery(db) + ok = query.exec_(self.format_select_stmt) + if ok: + query.next() + format = int(query.value(0)) + if format >= self.DB_VERSION: + prepared = True + finally: + del query + db.commit() + return prepared + + def getCompletions(self, start = None, context = None): + """ + Public method to determine the possible completions. + + @keyparam start string giving the start of the word to be + completed (string) + @keyparam context string giving the context (e.g. classname) + to be completed (string) + @return list of dictionaries with possible completions (key 'completion' + contains the completion (string), key 'context' + contains the context (string) and key 'pictureId' + contains the ID of the icon to be shown (string)) + """ + completions = [] + + db = QSqlDatabase.database(self.__language) + if db.isOpen() and not self.__inPreparation: + db.transaction() + try: + query = None + + if start is not None: + query = QSqlQuery(db) + query.prepare(self.ac_stmt) + query.bindValue(":acWord", start + '*') + elif context is not None: + query = QSqlQuery(db) + query.prepare(self.ac_context_stmt) + query.bindValue(":context", context) + + if query is not None: + query.exec_() + while query.next(): + completions.append({"completion" : query.value(0), + "context" : query.value(1), + "pictureId" : query.value(2)}) + del query + finally: + db.commit() + + return completions + + def getCalltips(self, acWord, commas, context = None, fullContext = None, + showContext = True): + """ + Public method to determine the calltips. + + @param acWord function to get calltips for (string) + @param commas minimum number of commas contained in the calltip (integer) + @param context string giving the context (e.g. classname) (string) + @param fullContext string giving the full context (string) + @param showContext flag indicating to show the calltip context (boolean) + @return list of calltips (list of string) + """ + calltips = [] + + db = QSqlDatabase.database(self.__language) + if db.isOpen() and not self.__inPreparation: + if self.autoCompletionWordSeparators(): + contextSeparator = self.autoCompletionWordSeparators()[0] + else: + contextSeparator = " " + db.transaction() + try: + query = QSqlQuery(db) + if fullContext: + query.prepare(self.ct_fullContext_stmt) + query.bindValue(":fullContext", fullContext) + elif context: + query.prepare(self.ct_context_stmt) + query.bindValue(":context", context) + else: + query.prepare(self.ct_stmt) + query.bindValue(":acWord", acWord) + query.exec_() + while query.next(): + word = query.value(0) + sig = query.value(1) + fullCtx = query.value(2) + if sig: + if self.__discardFirst: + sig = "(%s" % sig[1:]\ + .replace(self.__discardFirst, "", 1)\ + .strip(", \t\r\n") + if self.__enoughCommas(sig, commas): + if showContext: + calltips.append("%s%s%s%s" % \ + (fullCtx, contextSeparator, word, sig)) + else: + calltips.append("%s%s" % (word, sig)) + del query + finally: + db.commit() + + if context and len(calltips) == 0: + # nothing found, try without a context + calltips = self.getCalltips(acWord, commas, showContext = showContext) + + return calltips + + def __enoughCommas(self, s, commas): + """ + Private method to determine, if the given string contains enough commas. + + @param s string to check (string) + @param commas number of commas to check for (integer) + @return flag indicating, that there are enough commas (boolean) + """ + end = s.find(')') + + if end < 0: + return False + + w = s[:end] + return w.count(',') >= commas + + def __openAPIs(self): + """ + Private method to open the API database. + """ + self.__openApiDb() + if not self.__isPrepared(): + self.__createApiDB() + + # prepare the database if neccessary + self.prepareAPIs() + + def prepareAPIs(self, rawList = None): + """ + Public method to prepare the APIs if neccessary. + + @keyparam rawList list of raw API files (list of strings) + """ + if self.__inPreparation: + return + + projectPath = "" + if rawList: + apiFiles = rawList[:] + elif self.__language == ApisNameProject: + apiFiles = self.__project.getSources() + projectPath = self.__project.getProjectPath() + else: + apiFiles = Preferences.getEditorAPI(self.__language) + self.__worker = DbAPIsWorker(self, self.__language, apiFiles, projectPath) + self.__worker.start() + + def __processQueue(self): + """ + Private slot to process the queue of files to load. + """ + if self.__worker is not None and self.__worker.isFinished(): + self.__worker = None + + if self.__worker is None and len(self.__workerQueue) > 0: + apiFiles = [self.__workerQueue.pop(0)] + if self.__language == ApisNameProject: + projectPath = self.__project.getProjectPath() + apiFiles = [apiFiles[0].replace(projectPath + os.sep, "")] + else: + projectPath = "" + self.__worker = DbAPIsWorker(self, self.__language, apiFiles, projectPath) + self.__worker.start() + + def getLexer(self): + """ + Public method to return a reference to our lexer object. + + @return reference to the lexer object (QScintilla.Lexers.Lexer) + """ + return self.__lexer + + def autoCompletionWordSeparators(self): + """ + Private method to get the word separator characters. + + @return word separator characters (list of strings) + """ + if self.__lexer: + return self.__lexer.autoCompletionWordSeparators() + return None + + def event(self, evt): + """ + Protected method to handle events from the worker thread. + + @param evt reference to the event object (QEvent) + @return flag indicating, if the event was handled (boolean) + """ + if evt.type() == WorkerStarted: + self.__inPreparation = True + self.emit(SIGNAL('apiPreparationStarted()')) + return True + + elif evt.type() == WorkerFinished: + self.__inPreparation = False + self.emit(SIGNAL('apiPreparationFinished()')) + QTimer.singleShot(0, self.__processQueue) + return True + + elif evt.type() == WorkerAborted: + self.__inPreparation = False + self.emit(SIGNAL('apiPreparationCancelled()')) + QTimer.singleShot(0, self.__processQueue) + return True + + else: + return QObject.event(self, evt) + + ######################################################## + ## project related stuff below + ######################################################## + + def __projectOpened(self): + """ + Private slot to perform actions after a project has been opened. + """ + if self.__project.getProjectLanguage() in ["Python", "Python3"]: + self.__discardFirst = "self" + else: + self.__discardFirst = "" + self.__lexer = QScintilla.Lexers.getLexer(self.__project.getProjectLanguage()) + self.__openAPIs() + + def __projectClosed(self): + """ + Private slot to perform actions after a project has been closed. + """ + self.close() + + def editorSaved(self, filename): + """ + Public slot to handle the editorSaved signal. + + @param filename name of the file that was saved (string) + """ + if self.__project.isProjectSource(filename): + self.__workerQueue.append(filename) + self.__processQueue() + +class APIsManager(QObject): + """ + Class implementing the APIsManager class, which is the central store for + API information used by autocompletion and calltips. + """ + def __init__(self, parent = None): + """ + Constructor + + @param parent reference to the parent object (QObject) + """ + QObject.__init__(self, parent) + self.setObjectName("APIsManager") + + # initialize the apis dictionary + self.__apis = {} + + def reloadAPIs(self): + """ + Public slot to reload the api information. + """ + for api in list(self.__apis.values()): + api and api.prepareAPIs() + + def getAPIs(self, language): + """ + Public method to get an apis object for autocompletion/calltips. + + This method creates and loads an APIs object dynamically upon request. + This saves memory for languages, that might not be needed at the moment. + + @param language the language of the requested api object (string) + @return the apis object (APIs) + """ + try: + return self.__apis[language] + except KeyError: + if language in QScintilla.Lexers.getSupportedLanguages() or \ + language == ApisNameProject: + # create the api object + self.__apis[language] = DbAPIs(language) + return self.__apis[language] + else: + return None + + def deactivate(self): + """ + Public method to perform actions upon deactivation. + """ + for apiLang in self.__apis: + self.__apis[apiLang].close() + self.__apis[apiLang] = None