Implemented the batch check mode for the syntax checker.

Wed, 22 Apr 2015 19:53:58 +0200

author
Detlev Offenbach <detlev@die-offenbachs.de>
date
Wed, 22 Apr 2015 19:53:58 +0200
changeset 4231
0b38613388c9
parent 4230
1117b60c1f9d
child 4232
8349fd3f8802

Implemented the batch check mode for the syntax checker.

Plugins/CheckerPlugins/SyntaxChecker/SyntaxCheck.py file | annotate | diff | comparison | revisions
Plugins/CheckerPlugins/SyntaxChecker/SyntaxCheckService.py file | annotate | diff | comparison | revisions
Plugins/CheckerPlugins/SyntaxChecker/SyntaxCheckerDialog.py file | annotate | diff | comparison | revisions
Plugins/CheckerPlugins/SyntaxChecker/jsCheckSyntax.py file | annotate | diff | comparison | revisions
Plugins/PluginSyntaxChecker.py file | annotate | diff | comparison | revisions
changelog file | annotate | diff | comparison | revisions
--- a/Plugins/CheckerPlugins/SyntaxChecker/SyntaxCheck.py	Tue Apr 21 19:47:31 2015 +0200
+++ b/Plugins/CheckerPlugins/SyntaxChecker/SyntaxCheck.py	Wed Apr 22 19:53:58 2015 +0200
@@ -11,6 +11,7 @@
 import re
 import sys
 import traceback
+import multiprocessing
 
 try:
     from pyflakes.checker import Checker
@@ -32,6 +33,15 @@
     return syntaxAndPyflakesCheck
 
 
+def initBatchService():
+    """
+    Initialize the batch service and return the entry point.
+    
+    @return the entry point for the background client (function)
+    """
+    return syntaxAndPyflakesBatchCheck
+
+
 def normalizeCode(codestring):
     """
     Function to normalize the given code.
@@ -92,6 +102,88 @@
             (file name, line number, column, codestring (only at syntax
             errors), the message, a list with arguments for the message)
     """
+    return __syntaxAndPyflakesCheck(filename, codestring, checkFlakes,
+                                    ignoreStarImportWarnings)
+
+
+def syntaxAndPyflakesBatchCheck(argumentsList, send, fx, cancelled):
+    """
+    Module function to check syntax for a batch of files.
+    
+    @param argumentsList list of arguments tuples as given for 
+        syntaxAndPyflakesCheck
+    @param send reference to send function (function)
+    @param fx registered service name (string)
+    @param cancelled reference to function checking for a cancellation
+        (function)
+    """
+    try:
+        NumberOfProcesses = multiprocessing.cpu_count()
+        if NumberOfProcesses >= 1:
+            NumberOfProcesses -= 1
+    except NotImplementedError:
+        NumberOfProcesses = 1
+
+    # Create queues
+    taskQueue = multiprocessing.Queue()
+    doneQueue = multiprocessing.Queue()
+
+    # Submit tasks (initially two time number of processes
+    initialTasks = 2 * NumberOfProcesses
+    for task in argumentsList[:initialTasks]:
+        taskQueue.put(task)
+
+    # Start worker processes
+    for i in range(NumberOfProcesses):
+        multiprocessing.Process(target=worker, args=(taskQueue, doneQueue))\
+            .start()
+
+    # Get and send results
+    endIndex = len(argumentsList) - initialTasks
+    for i in range(len(argumentsList)):
+        filename, result = doneQueue.get()
+        send(fx, filename, result)
+        if cancelled():
+            # just exit the loop ignoring the results of queued tasks
+            break
+        if i < endIndex:
+            taskQueue.put(argumentsList[i + initialTasks])
+
+    # Tell child processes to stop
+    for i in range(NumberOfProcesses):
+        taskQueue.put('STOP')
+
+
+def worker(input, output):
+    """
+    Module function acting as the parallel worker for the style check.
+    
+    @param input input queue (multiprocessing.Queue)
+    @param output output queue (multiprocessing.Queue)
+    """
+    for filename, args in iter(input.get, 'STOP'):
+        source, checkFlakes, ignoreStarImportWarnings = args
+        result = __syntaxAndPyflakesCheck(filename, source, checkFlakes,
+                                          ignoreStarImportWarnings)
+        output.put((filename, result))
+
+
+def __syntaxAndPyflakesCheck(filename, codestring, checkFlakes=True,
+                             ignoreStarImportWarnings=False):
+    """
+    Function to compile one Python source file to Python bytecode
+    and to perform a pyflakes check.
+    
+    @param filename source filename (string)
+    @param codestring string containing the code to compile (string)
+    @keyparam checkFlakes flag indicating to do a pyflakes check (boolean)
+    @keyparam ignoreStarImportWarnings flag indicating to
+        ignore 'star import' warnings (boolean)
+    @return dictionary with the keys 'error' and 'warnings' which
+            hold a list containing details about the error/ warnings
+            (file name, line number, column, codestring (only at syntax
+            errors), the message, a list with arguments for the message)
+    """
     try:
         import builtins
     except ImportError:
--- a/Plugins/CheckerPlugins/SyntaxChecker/SyntaxCheckService.py	Tue Apr 21 19:47:31 2015 +0200
+++ b/Plugins/CheckerPlugins/SyntaxChecker/SyntaxCheckService.py	Wed Apr 22 19:53:58 2015 +0200
@@ -25,9 +25,12 @@
     and support of an extra checker module on the client side which has to
     connect directly to the background service.
     
-    @signal syntaxChecked(str, dict) emited when the syntax check was done.
+    @signal syntaxChecked(str, dict) emitted when the syntax check was done for
+        one file
+    @signal batchFinished() emitted when a syntax check batch is done
     """
     syntaxChecked = pyqtSignal(str, dict)
+    batchFinished = pyqtSignal()
     
     def __init__(self):
         """
@@ -36,6 +39,9 @@
         super(SyntaxCheckService, self).__init__()
         self.backgroundService = e5App().getObject("BackgroundService")
         self.__supportedLanguages = {}
+        
+        self.queuedBatches = []
+        self.batchesFinished = True
 
     def __determineLanguage(self, filename, source):
         """
@@ -75,7 +81,8 @@
         self.__supportedLanguages[lang] = env, getArgs, getExt
         # Connect to the background service
         self.backgroundService.serviceConnect(
-            '{0}Syntax'.format(lang), env, path, module, callback, onError)
+            '{0}Syntax'.format(lang), env, path, module, callback, onError,
+            onBatchDone=self.batchJobDone)
 
     def getLanguages(self):
         """
@@ -110,8 +117,7 @@
 
     def syntaxCheck(self, lang, filename, source):
         """
-        Public method to prepare to compile one Python source file to Python
-        bytecode and to perform a pyflakes check.
+        Public method to prepare a syntax check of one source file.
         
         @param lang language of the file or None to determine by internal
             algorithm (str or None)
@@ -128,3 +134,121 @@
         data.extend(args())
         self.backgroundService.enqueueRequest(
             '{0}Syntax'.format(lang), env, filename, data)
+    
+    def syntaxBatchCheck(self, argumentsList):
+        """
+        Public method to prepare a syntax check on multiple source files.
+        
+        @param argumentsList list of arguments tuples with each tuple
+            containing filename and source (string, string)
+        """
+        data = {
+        }
+        for lang in self.getLanguages():
+            data[lang] = []
+        
+        for filename, source in argumentsList:
+            lang = self.__determineLanguage(filename, source)
+            if lang not in self.getLanguages():
+                continue
+            else:
+                jobData = [source]
+                # Call the getArgs function to get the required arguments
+                args = self.__supportedLanguages[lang][1]
+                jobData.extend(args())
+                data[lang].append((filename, jobData))
+        
+        self.queuedBatches = []
+        for lang in self.getLanguages():
+            if data[lang]:
+                self.queuedBatches.append(lang)
+                env = self.__supportedLanguages[lang][0]
+                self.backgroundService.enqueueRequest(
+                    'batch_{0}Syntax'.format(lang), env, "", data[lang])
+                self.batchesFinished = False
+    
+    def cancelSyntaxBatchCheck(self):
+        """
+        Public method to cancel all batch jobs.
+        """
+        envs = []
+        for lang in self.getLanguages():
+            env = self.__supportedLanguages[lang][0]
+            if env not in envs:
+                envs.append(env)
+        for lang in envs:
+            self.backgroundService.requestCancel(lang)
+    
+    def __serviceError(self, fn, msg):
+        """
+        Private slot handling service errors.
+        
+        @param fn file name (string)
+        @param msg message text (string)
+        """
+        self.syntaxChecked.emit(fn, {'warnings': [(fn, 1, 0, '', msg)]})
+    
+    def serviceErrorPy2(self, fx, lang, fn, msg):
+        """
+        Public method handling service errors for Python 2.
+        
+        @param fx service name (string)
+        @param lang language (string)
+        @param fn file name (string)
+        @param msg message text (string)
+        """
+        if fx in ['Python2Syntax', 'batch_Python2Syntax']:
+            if fx == 'Python2Syntax':
+                self.__serviceError(fn, msg)
+            else:
+                self.__serviceError(self.tr("Python 2 batch check"), msg)
+                self.batchJobDone(fx, lang)
+    
+    def serviceErrorPy3(self, fx, lang, fn, msg):
+        """
+        Public method handling service errors for Python 2.
+        
+        @param fx service name (string)
+        @param lang language (string)
+        @param fn file name (string)
+        @param msg message text (string)
+        """
+        if fx in ['Python3Syntax', 'batch_Python3Syntax']:
+            if fx == 'Python3Syntax':
+                self.__serviceError(fn, msg)
+            else:
+                self.__serviceError(self.tr("Python 3 batch check"), msg)
+                self.batchJobDone(fx, lang)
+    
+    def serviceErrorJavaScript(self, fx, lang, fn, msg):
+        """
+        Public method handling service errors for JavaScript.
+        
+        @param fx service name (string)
+        @param lang language (string)
+        @param fn file name (string)
+        @param msg message text (string)
+        """
+        if fx in ['JavaScriptSyntax', 'batch_JavaScriptSyntax']:
+            if fx == 'JavaScriptSyntax':
+                self.__serviceError(fn, msg)
+            else:
+                self.__serviceError(self.tr("JavaScript batch check"), msg)
+                self.batchJobDone(fx, lang)
+    
+    def batchJobDone(self, fx, lang):
+        """
+        Public slot handling the completion of a batch job.
+        
+        @param fx service name (string)
+        @param lang language (string)
+        """
+        if fx in ['Python2Syntax', 'batch_Python2Syntax',
+                  'Python3Syntax', 'batch_Python3Syntax',
+                  'JavaScriptSyntax', 'batch_JavaScriptSyntax']:
+            if lang in self.queuedBatches:
+                self.queuedBatches.remove(lang)
+            # prevent sending the signal multiple times
+            if len(self.queuedBatches) == 0 and not self.batchesFinished:
+                self.batchFinished.emit()
+                self.batchesFinished = True
--- a/Plugins/CheckerPlugins/SyntaxChecker/SyntaxCheckerDialog.py	Tue Apr 21 19:47:31 2015 +0200
+++ b/Plugins/CheckerPlugins/SyntaxChecker/SyntaxCheckerDialog.py	Wed Apr 22 19:53:58 2015 +0200
@@ -56,6 +56,7 @@
         self.noResults = True
         self.cancelled = False
         self.__lastFileItem = None
+        self.__finished = True
         
         self.__fileList = []
         self.__project = None
@@ -68,6 +69,7 @@
         try:
             self.syntaxCheckService = e5App().getObject('SyntaxCheckService')
             self.syntaxCheckService.syntaxChecked.connect(self.__processResult)
+            self.syntaxCheckService.batchFinished.connect(self.__batchFinished)
         except KeyError:
             self.syntaxCheckService = None
         self.filename = None
@@ -174,7 +176,13 @@
 
                 # now go through all the files
                 self.progress = 0
-                self.check(codestring)
+                self.files.sort()
+                if codestring or len(self.files) == 1:
+                    self.__batch = False
+                    self.check(codestring)
+                else:
+                    self.__batch = True
+                    self.checkBatch()
     
     def check(self, codestring=''):
         """
@@ -184,6 +192,9 @@
         @keyparam codestring optional sourcestring (str)
         """
         if self.syntaxCheckService is None or not self.files:
+            self.checkProgressLabel.setPath("")
+            self.checkProgress.setMaximum(1)
+            self.checkProgress.setValue(1)
             self.__finish()
             return
         
@@ -215,8 +226,55 @@
                 self.check()
                 return
         
+        self.__finished = False
         self.syntaxCheckService.syntaxCheck(None, self.filename, self.source)
 
+    def checkBatch(self):
+        """
+        Public method to start a style check batch job.
+        
+        The results are reported to the __processResult slot.
+        """
+        self.__lastFileItem = None
+        
+        self.checkProgressLabel.setPath(self.tr("Preparing files..."))
+        progress = 0
+        
+        argumentsList = []
+        for filename in self.files:
+            progress += 1
+            self.checkProgress.setValue(progress)
+            QApplication.processEvents()
+            
+            try:
+                source = Utilities.readEncodedFile(filename)[0]
+                source = Utilities.normalizeCode(source)
+            except (UnicodeError, IOError) as msg:
+                self.noResults = False
+                self.__createResultItem(
+                    self.filename, 1, 0,
+                    self.tr("Error: {0}").format(str(msg))
+                    .rstrip(), "")
+                continue
+            
+            argumentsList.append((filename, source))
+        
+        # reset the progress bar to the checked files
+        self.checkProgress.setValue(self.progress)
+        QApplication.processEvents()
+        
+        self.__finished = False
+        self.syntaxCheckService.syntaxBatchCheck(argumentsList)
+    
+    def __batchFinished(self):
+        """
+        Private slot handling the completion of a batch job.
+        """
+        self.checkProgressLabel.setPath("")
+        self.checkProgress.setMaximum(1)
+        self.checkProgress.setValue(1)
+        self.__finish()
+    
     def __processResult(self, fn, problems):
         """
         Private slot to display the reported messages.
@@ -227,8 +285,12 @@
             (file name, line number, column, codestring (only at syntax
             errors), the message) (dict)
         """
-        # Check if it's the requested file, otherwise ignore signal
-        if fn != self.filename:
+        if self.__finished:
+            return
+        
+        # Check if it's the requested file, otherwise ignore signal if not
+        # in batch mode
+        if not self.__batch and fn != self.filename:
             return
 
         error = problems.get('error')
@@ -247,22 +309,20 @@
 
         self.progress += 1
         self.checkProgress.setValue(self.progress)
+        self.checkProgressLabel.setPath(fn)
         QApplication.processEvents()
         self.__resort()
 
-        if self.files:
+        if not self.__batch:
             self.check()
-        else:
-            self.checkProgressLabel.setPath("")
-            self.checkProgress.setMaximum(1)
-            self.checkProgress.setValue(1)
-            self.__finish()
         
     def __finish(self):
         """
         Private slot called when the syntax check finished or the user
         pressed the button.
         """
+        self.__finished = True
+        
         self.cancelled = True
         self.buttonBox.button(QDialogButtonBox.Close).setEnabled(True)
         self.buttonBox.button(QDialogButtonBox.Cancel).setEnabled(False)
@@ -289,7 +349,10 @@
         if button == self.buttonBox.button(QDialogButtonBox.Close):
             self.close()
         elif button == self.buttonBox.button(QDialogButtonBox.Cancel):
-            self.__finish()
+            if self.__batch:
+                self.syntaxCheckService.cancelSyntaxBatchCheck()
+            else:
+                self.__finish()
         elif button == self.showButton:
             self.on_showButton_clicked()
         
--- a/Plugins/CheckerPlugins/SyntaxChecker/jsCheckSyntax.py	Tue Apr 21 19:47:31 2015 +0200
+++ b/Plugins/CheckerPlugins/SyntaxChecker/jsCheckSyntax.py	Wed Apr 22 19:53:58 2015 +0200
@@ -9,6 +9,7 @@
 """
 import os
 import sys
+import multiprocessing
 
 
 def initService():
@@ -21,7 +22,16 @@
     for i in range(4):
         path = os.path.dirname(path)
     sys.path.insert(2, os.path.join(path, "ThirdParty", "Jasy"))
-    return jsCheckSyntax
+    return jsSyntaxCheck
+
+
+def initBatchService():
+    """
+    Initialize the batch service and return the entry point.
+    
+    @return the entry point for the background client (function)
+    """
+    return jsSyntaxBatchCheck
 
 
 def normalizeCode(codestring):
@@ -46,7 +56,82 @@
     return codestring
 
 
-def jsCheckSyntax(file, codestring):
+def jsSyntaxCheck(file, codestring):
+    """
+    Function to check a Javascript source file for syntax errors.
+    
+    @param file source filename (string)
+    @param codestring string containing the code to check (string)
+    @return dictionary with the keys 'error' and 'warnings' which
+            hold a list containing details about the error/ warnings
+            (file name, line number, column, codestring (only at syntax
+            errors), the message, a list with arguments for the message)
+    """
+    return __jsSyntaxCheck(file, codestring)
+
+
+def jsSyntaxBatchCheck(argumentsList, send, fx, cancelled):
+    """
+    Module function to check syntax for a batch of files.
+    
+    @param argumentsList list of arguments tuples as given for 
+        syntaxAndPyflakesCheck
+    @param send reference to send function (function)
+    @param fx registered service name (string)
+    @param cancelled reference to function checking for a cancellation
+        (function)
+    """
+    try:
+        NumberOfProcesses = multiprocessing.cpu_count()
+        if NumberOfProcesses >= 1:
+            NumberOfProcesses -= 1
+    except NotImplementedError:
+        NumberOfProcesses = 1
+
+    # Create queues
+    taskQueue = multiprocessing.Queue()
+    doneQueue = multiprocessing.Queue()
+
+    # Submit tasks (initially two time number of processes
+    initialTasks = 2 * NumberOfProcesses
+    for task in argumentsList[:initialTasks]:
+        taskQueue.put(task)
+
+    # Start worker processes
+    for i in range(NumberOfProcesses):
+        multiprocessing.Process(target=worker, args=(taskQueue, doneQueue))\
+            .start()
+
+    # Get and send results
+    endIndex = len(argumentsList) - initialTasks
+    for i in range(len(argumentsList)):
+        filename, result = doneQueue.get()
+        send(fx, filename, result)
+        if cancelled():
+            # just exit the loop ignoring the results of queued tasks
+            break
+        if i < endIndex:
+            taskQueue.put(argumentsList[i + initialTasks])
+
+    # Tell child processes to stop
+    for i in range(NumberOfProcesses):
+        taskQueue.put('STOP')
+
+
+def worker(input, output):
+    """
+    Module function acting as the parallel worker for the style check.
+    
+    @param input input queue (multiprocessing.Queue)
+    @param output output queue (multiprocessing.Queue)
+    """
+    for filename, args in iter(input.get, 'STOP'):
+        source = args[0]
+        result = __jsSyntaxCheck(filename, source)
+        output.put((filename, result))
+
+
+def __jsSyntaxCheck(file, codestring):
     """
     Function to check a Javascript source file for syntax errors.
     
--- a/Plugins/PluginSyntaxChecker.py	Tue Apr 21 19:47:31 2015 +0200
+++ b/Plugins/PluginSyntaxChecker.py	Wed Apr 22 19:53:58 2015 +0200
@@ -68,14 +68,14 @@
             self.__getPythonOptions,
             lambda: Preferences.getPython("PythonExtensions"),
             self.__translateSyntaxCheck,
-            self.serviceErrorPy2)
+            self.syntaxCheckService.serviceErrorPy2)
         
         self.syntaxCheckService.addLanguage(
             'Python3', 'Python3', path, 'SyntaxCheck',
             self.__getPythonOptions,
             lambda: Preferences.getPython("Python3Extensions"),
             self.__translateSyntaxCheck,
-            self.serviceErrorPy3)
+            self.syntaxCheckService.serviceErrorPy3)
         
         # Jasy isn't yet compatible to Python2
         self.syntaxCheckService.addLanguage(
@@ -85,53 +85,7 @@
             lambda: ['.js'],
             lambda fn, problems:
                 self.syntaxCheckService.syntaxChecked.emit(fn, problems),  # __IGNORE_WARNING__
-            self.serviceErrorJavaScript)
-    
-    def __serviceError(self, fn, msg):
-        """
-        Private slot handling service errors.
-        
-        @param fn file name (string)
-        @param msg message text (string)
-        """
-        self.syntaxCheckService.syntaxChecked.emit(
-            fn, {'warnings': [(fn, 1, 0, '', msg)]})
-    
-    def serviceErrorPy2(self, fx, lang, fn, msg):
-        """
-        Public method handling service errors for Python 2.
-        
-        @param fx service name (string)
-        @param lang language (string)
-        @param fn file name (string)
-        @param msg message text (string)
-        """
-        if fx == 'Python2Syntax':
-            self.__serviceError(fn, msg)
-    
-    def serviceErrorPy3(self, fx, lang, fn, msg):
-        """
-        Public method handling service errors for Python 2.
-        
-        @param fx service name (string)
-        @param lang language (string)
-        @param fn file name (string)
-        @param msg message text (string)
-        """
-        if fx == 'Python3Syntax':
-            self.__serviceError(fn, msg)
-    
-    def serviceErrorJavaScript(self, fx, lang, fn, msg):
-        """
-        Public method handling service errors for JavaScript.
-        
-        @param fx service name (string)
-        @param lang language (string)
-        @param fn file name (string)
-        @param msg message text (string)
-        """
-        if fx == 'JavaScriptSyntax':
-            self.__serviceError(fn, msg)
+            self.syntaxCheckService.serviceErrorJavaScript)
 
     def __initialize(self):
         """
--- a/changelog	Tue Apr 21 19:47:31 2015 +0200
+++ b/changelog	Wed Apr 22 19:53:58 2015 +0200
@@ -8,6 +8,8 @@
 - Checkers
   -- added a batch mode to the code style checker to make use of
      multiple CPUs/CPU-Cores
+  -- added a batch mode to the syntax checker to make use of
+     multiple CPUs/CPU-Cores
 - Editor
   -- added capability to configure the indentation guides colors
   -- Lexers

eric ide

mercurial