Mercurial Repositories > eric / changeset
changeset
Did some optimizations in the multiprocessing code.
Tue, 16 Aug 2022 19:07:16 +0200
- author
- Detlev Offenbach <detlev@die-offenbachs.de>
- date
- Tue, 16 Aug 2022 19:07:16 +0200
- branch
- eric7
- changeset 9289
- ba49c41e8f63
- parent 9288
- ce20d7c936b1
- child 9290
- fc0cbfb71714
Did some optimizations in the multiprocessing code.
--- a/src/eric7/CodeFormatting/BlackFormattingDialog.py Tue Aug 16 13:14:42 2022 +0200 +++ b/src/eric7/CodeFormatting/BlackFormattingDialog.py Tue Aug 16 19:07:16 2022 +0200 @@ -272,8 +272,11 @@ taskQueue = multiprocessing.Queue() doneQueue = multiprocessing.Queue() - # Submit tasks - for file in files: + # Submit tasks (initially two times the number of processes) + tasks = len(files) + initialTasks = 2 * NumberOfProcesses + for _ in range(initialTasks): + file = files.pop(0) relSrc = self.__project.getRelativePath(str(file)) if self.__project else "" taskQueue.put((file, relSrc)) @@ -289,7 +292,7 @@ worker.start() # Get the results from the worker tasks - for _ in range(len(files)): + for _ in range(tasks): result = doneQueue.get() self.__handleBlackFormattingResult( result.status, result.filename, result.data @@ -298,13 +301,16 @@ if self.__cancelled: break - if self.__cancelled: - for worker in workers: - worker.terminate() - else: - # Tell child processes to stop - for _ in range(NumberOfProcesses): - taskQueue.put("STOP") + if files: + file = files.pop(0) + relSrc = ( + self.__project.getRelativePath(str(file)) if self.__project else "" + ) + taskQueue.put((file, relSrc)) + + # Tell child processes to stop + for _ in range(NumberOfProcesses): + taskQueue.put("STOP") for worker in workers: worker.join()
--- a/src/eric7/Plugins/CheckerPlugins/CodeStyleChecker/CodeStyleChecker.py Tue Aug 16 13:14:42 2022 +0200 +++ b/src/eric7/Plugins/CheckerPlugins/CodeStyleChecker/CodeStyleChecker.py Tue Aug 16 19:07:16 2022 +0200 @@ -224,10 +224,11 @@ taskQueue = multiprocessing.Queue() doneQueue = multiprocessing.Queue() - # Submit tasks (initially two time number of processes + # Submit tasks (initially two times the number of processes) + tasks = len(argumentsList) initialTasks = 2 * NumberOfProcesses - for task in argumentsList[:initialTasks]: - taskQueue.put(task) + for _ in range(initialTasks): + taskQueue.put(argumentsList.pop(0)) # Start worker processes workers = [ @@ -238,8 +239,7 @@ worker.start() # Get and send results - endIndex = len(argumentsList) - initialTasks - for i in range(len(argumentsList)): + for _ in range(tasks): resultSent = False wasCancelled = False @@ -259,8 +259,8 @@ # just exit the loop ignoring the results of queued tasks break - if i < endIndex: - taskQueue.put(argumentsList[i + initialTasks]) + if argumentsList: + taskQueue.put(argumentsList.pop(0)) # Tell child processes to stop for _ in range(NumberOfProcesses):
--- a/src/eric7/Plugins/CheckerPlugins/SyntaxChecker/SyntaxCheck.py Tue Aug 16 13:14:42 2022 +0200 +++ b/src/eric7/Plugins/CheckerPlugins/SyntaxChecker/SyntaxCheck.py Tue Aug 16 19:07:16 2022 +0200 @@ -145,10 +145,11 @@ taskQueue = multiprocessing.Queue() doneQueue = multiprocessing.Queue() - # Submit tasks (initially two time number of processes + # Submit tasks (initially two times the number of processes) + tasks = len(argumentsList) initialTasks = 2 * NumberOfProcesses - for task in argumentsList[:initialTasks]: - taskQueue.put(task) + for _ in range(initialTasks): + taskQueue.put(argumentsList.pop(0)) # Start worker processes workers = [ @@ -159,8 +160,7 @@ worker.start() # Get and send results - endIndex = len(argumentsList) - initialTasks - for i in range(len(argumentsList)): + for _ in range(tasks): resultSent = False wasCancelled = False @@ -180,8 +180,8 @@ # just exit the loop ignoring the results of queued tasks break - if i < endIndex: - taskQueue.put(argumentsList[i + initialTasks]) + if argumentsList: + taskQueue.put(argumentsList.pop(0)) # Tell child processes to stop for _ in range(NumberOfProcesses):
--- a/src/eric7/Plugins/CheckerPlugins/SyntaxChecker/jsCheckSyntax.py Tue Aug 16 13:14:42 2022 +0200 +++ b/src/eric7/Plugins/CheckerPlugins/SyntaxChecker/jsCheckSyntax.py Tue Aug 16 19:07:16 2022 +0200 @@ -94,10 +94,11 @@ taskQueue = multiprocessing.Queue() doneQueue = multiprocessing.Queue() - # Submit tasks (initially two time number of processes + # Submit tasks (initially two times the number of processes) + tasks = len(argumentsList) initialTasks = 2 * NumberOfProcesses - for task in argumentsList[:initialTasks]: - taskQueue.put(task) + for _ in range(initialTasks): + taskQueue.put(argumentsList.pop(0)) # Start worker processes workers = [ @@ -108,8 +109,7 @@ worker.start() # Get and send results - endIndex = len(argumentsList) - initialTasks - for i in range(len(argumentsList)): + for _ in range(tasks): resultSent = False wasCancelled = False @@ -129,8 +129,8 @@ # just exit the loop ignoring the results of queued tasks break - if i < endIndex: - taskQueue.put(argumentsList[i + initialTasks]) + if argumentsList: + taskQueue.put(argumentsList.pop(0)) # Tell child processes to stop for _ in range(NumberOfProcesses):
--- a/src/eric7/Plugins/CheckerPlugins/SyntaxChecker/jsonCheckSyntax.py Tue Aug 16 13:14:42 2022 +0200 +++ b/src/eric7/Plugins/CheckerPlugins/SyntaxChecker/jsonCheckSyntax.py Tue Aug 16 19:07:16 2022 +0200 @@ -95,10 +95,11 @@ taskQueue = multiprocessing.Queue() doneQueue = multiprocessing.Queue() - # Submit tasks (initially two time number of processes + # Submit tasks (initially two times the number of processes) + tasks = len(argumentsList) initialTasks = 2 * NumberOfProcesses - for task in argumentsList[:initialTasks]: - taskQueue.put(task) + for _ in range(initialTasks): + taskQueue.put(argumentsList.pop(0)) # Start worker processes workers = [ @@ -109,8 +110,7 @@ worker.start() # Get and send results - endIndex = len(argumentsList) - initialTasks - for i in range(len(argumentsList)): + for _ in range(tasks): resultSent = False wasCancelled = False @@ -130,8 +130,8 @@ # just exit the loop ignoring the results of queued tasks break - if i < endIndex: - taskQueue.put(argumentsList[i + initialTasks]) + if argumentsList: + taskQueue.put(argumentsList.pop(0)) # Tell child processes to stop for _ in range(NumberOfProcesses):
--- a/src/eric7/Plugins/CheckerPlugins/SyntaxChecker/tomlCheckSyntax.py Tue Aug 16 13:14:42 2022 +0200 +++ b/src/eric7/Plugins/CheckerPlugins/SyntaxChecker/tomlCheckSyntax.py Tue Aug 16 19:07:16 2022 +0200 @@ -95,10 +95,11 @@ taskQueue = multiprocessing.Queue() doneQueue = multiprocessing.Queue() - # Submit tasks (initially two time number of processes + # Submit tasks (initially two times the number of processes) + tasks = len(argumentsList) initialTasks = 2 * NumberOfProcesses - for task in argumentsList[:initialTasks]: - taskQueue.put(task) + for _ in range(initialTasks): + taskQueue.put(argumentsList.pop(0)) # Start worker processes workers = [ @@ -109,8 +110,7 @@ worker.start() # Get and send results - endIndex = len(argumentsList) - initialTasks - for i in range(len(argumentsList)): + for _ in range(tasks): resultSent = False wasCancelled = False @@ -130,8 +130,8 @@ # just exit the loop ignoring the results of queued tasks break - if i < endIndex: - taskQueue.put(argumentsList[i + initialTasks]) + if argumentsList: + taskQueue.put(argumentsList.pop(0)) # Tell child processes to stop for _ in range(NumberOfProcesses):
--- a/src/eric7/Plugins/CheckerPlugins/SyntaxChecker/yamlCheckSyntax.py Tue Aug 16 13:14:42 2022 +0200 +++ b/src/eric7/Plugins/CheckerPlugins/SyntaxChecker/yamlCheckSyntax.py Tue Aug 16 19:07:16 2022 +0200 @@ -95,10 +95,11 @@ taskQueue = multiprocessing.Queue() doneQueue = multiprocessing.Queue() - # Submit tasks (initially two time number of processes + # Submit tasks (initially two times the number of processes) + tasks = len(argumentsList) initialTasks = 2 * NumberOfProcesses - for task in argumentsList[:initialTasks]: - taskQueue.put(task) + for _ in range(initialTasks): + taskQueue.put(argumentsList.pop(0)) # Start worker processes workers = [ @@ -109,8 +110,7 @@ worker.start() # Get and send results - endIndex = len(argumentsList) - initialTasks - for i in range(len(argumentsList)): + for _ in range(tasks): resultSent = False wasCancelled = False @@ -130,8 +130,8 @@ # just exit the loop ignoring the results of queued tasks break - if i < endIndex: - taskQueue.put(argumentsList[i + initialTasks]) + if argumentsList: + taskQueue.put(argumentsList.pop(0)) # Tell child processes to stop for _ in range(NumberOfProcesses):
