--- a/src/eric7/CodeFormatting/BlackFormattingDialog.py Sun Aug 07 15:47:54 2022 +0200
+++ b/src/eric7/CodeFormatting/BlackFormattingDialog.py Sun Aug 07 17:31:26 2022 +0200
@@ -9,6 +9,7 @@
import copy
import datetime
+import multiprocessing
import pathlib
from dataclasses import dataclass
@@ -33,6 +34,7 @@
from .BlackFormattingAction import BlackFormattingAction
import Utilities
+import Preferences
class BlackFormattingDialog(QDialog, Ui_BlackFormattingDialog):
@@ -80,12 +82,8 @@
self.__statistics = BlackStatistics()
- self.__report = BlackReport(self)
- self.__report.check = action is BlackFormattingAction.Check
- self.__report.diff = action is BlackFormattingAction.Diff
- self.__report.result.connect(self.__handleBlackFormattingResult)
-
self.__config = copy.deepcopy(configuration)
+ self.__config["__action__"] = action # needed by the workers
self.__project = project
self.__action = action
@@ -101,8 +99,11 @@
self.show()
QCoreApplication.processEvents()
- self.__files = self.__filterFiles(filesList)
- self.__formatFiles()
+ files = self.__filterFiles(filesList)
+ if len(files) > 1:
+ self.__formatManyFiles(files)
+ elif len(files) == 1:
+ self.__formatOneFile(files[0])
def __filterFiles(self, filesList):
"""
@@ -128,7 +129,7 @@
for filterRegExp in filterRegExps:
filterMatch = filterRegExp.search(file)
if filterMatch and filterMatch.group(0):
- self.__report.path_ignored(file)
+ self.__handleBlackFormattingResult("ignored", file, "")
break
else:
files.append(file)
@@ -190,18 +191,19 @@
make them visible.
"""
self.reformattedLabel.setText(
- self.tr("reformatted")
+ self.tr("Reformatted:")
if self.__action is BlackFormattingAction.Format
- else self.tr("would reformat")
+ else self.tr("Would Reformat:")
)
total = self.progressBar.maximum()
- processed = total - self.__statistics.ignoreCount
self.totalCountLabel.setText("{0:n}".format(total))
self.excludedCountLabel.setText("{0:n}".format(self.__statistics.ignoreCount))
self.failuresCountLabel.setText("{0:n}".format(self.__statistics.failureCount))
- self.processedCountLabel.setText("{0:n}".format(processed))
+ self.processedCountLabel.setText(
+ "{0:n}".format(self.__statistics.processedCount)
+ )
self.reformattedCountLabel.setText(
"{0:n}".format(self.__statistics.changeCount)
)
@@ -246,13 +248,141 @@
self.__diffDialog = BlackDiffWidget()
self.__diffDialog.showDiff(item.data(0, BlackFormattingDialog.DataRole))
- def __formatFiles(self):
+ def __formatManyFiles(self, files):
+ """
+ Private method to format the list of files according the configuration using
+ multiple processes in parallel.
+
+ @param files list of files to be processed
+ @type list of str
+ """
+ maxProcesses = Preferences.getUI("BackgroundServiceProcesses")
+ if maxProcesses == 0:
+ # determine based on CPU count
+ try:
+ NumberOfProcesses = multiprocessing.cpu_count()
+ if NumberOfProcesses >= 1:
+ NumberOfProcesses -= 1
+ except NotImplementedError:
+ NumberOfProcesses = 1
+ else:
+ NumberOfProcesses = maxProcesses
+
+ # Create queues
+ taskQueue = multiprocessing.Queue()
+ doneQueue = multiprocessing.Queue()
+
+ # Submit tasks
+ for file in files:
+ relSrc = self.__project.getRelativePath(str(file)) if self.__project else ""
+ taskQueue.put((file, relSrc))
+
+ # Start worker processes
+ workers = [
+ multiprocessing.Process(
+ target=self.formattingWorkerTask,
+ args=(taskQueue, doneQueue, self.__config),
+ )
+ for _ in range(NumberOfProcesses)
+ ]
+ for worker in workers:
+ worker.start()
+
+ # Get the results from the worker tasks
+ for _ in range(len(files)):
+ result = doneQueue.get()
+ self.__handleBlackFormattingResult(
+ result.status, result.filename, result.data
+ )
+
+ if self.__cancelled:
+ break
+
+ if self.__cancelled:
+ for worker in workers:
+ worker.terminate()
+ else:
+ # Tell child processes to stop
+ for _ in range(NumberOfProcesses):
+ taskQueue.put("STOP")
+
+ for worker in workers:
+ worker.join()
+ worker.close()
+
+ taskQueue.close()
+ doneQueue.close()
+
+ self.__finish()
+
+ @staticmethod
+ def formattingWorkerTask(inputQueue, outputQueue, config):
+ """
+ Static method acting as the parallel worker for the formatting task.
+
+ @param inputQueue input queue
+ @type multiprocessing.Queue
+ @param outputQueue output queue
+ @type multiprocessing.Queue
+ @param config dictionary containing the configuration parameters
+ @type dict
+ """
+ report = BlackMultiprocessingReport(outputQueue)
+ report.check = config["__action__"] is BlackFormattingAction.Check
+ report.diff = config["__action__"] is BlackFormattingAction.Diff
+
+ versions = (
+ {black.TargetVersion[target.upper()] for target in config["target-version"]}
+ if config["target-version"]
+ else set()
+ )
+
+ mode = black.Mode(
+ target_versions=versions,
+ line_length=int(config["line-length"]),
+ string_normalization=not config["skip-string-normalization"],
+ magic_trailing_comma=not config["skip-magic-trailing-comma"],
+ )
+
+ if config["__action__"] is BlackFormattingAction.Diff:
+ for file, relSrc in iter(inputQueue.get, "STOP"):
+ BlackFormattingDialog.__diffFormatFile(
+ pathlib.Path(file),
+ fast=False,
+ mode=mode,
+ report=report,
+ relSrc=relSrc,
+ )
+ else:
+ writeBack = black.WriteBack.from_configuration(
+ check=config["__action__"] is BlackFormattingAction.Check,
+ diff=config["__action__"] is BlackFormattingAction.Diff,
+ )
+
+ for file, _relSrc in iter(inputQueue.get, "STOP"):
+ black.reformat_one(
+ pathlib.Path(file),
+ fast=False,
+ write_back=writeBack,
+ mode=mode,
+ report=report,
+ )
+
+ def __formatOneFile(self, file):
"""
Private method to format the list of files according the configuration.
+
+ @param file name of the file to be processed
+ @type str
"""
+ report = BlackReport(self)
+ report.check = self.__config["__action__"] is BlackFormattingAction.Check
+ report.diff = self.__config["__action__"] is BlackFormattingAction.Diff
+ report.result.connect(self.__handleBlackFormattingResult)
+
writeBack = black.WriteBack.from_configuration(
- check=self.__action is BlackFormattingAction.Check,
- diff=self.__action is BlackFormattingAction.Diff,
+ check=self.__config["__action__"] is BlackFormattingAction.Check,
+ diff=self.__config["__action__"] is BlackFormattingAction.Diff,
)
versions = (
@@ -271,28 +401,26 @@
magic_trailing_comma=not self.__config["skip-magic-trailing-comma"],
)
- for file in self.__files:
- if self.__action is BlackFormattingAction.Diff:
- self.__diffFormatFile(
- pathlib.Path(file), fast=False, mode=mode, report=self.__report
- )
- else:
- black.reformat_one(
- pathlib.Path(file),
- fast=False,
- write_back=writeBack,
- mode=mode,
- report=self.__report,
- )
-
- if self.__cancelled:
- break
+ if self.__action is BlackFormattingAction.Diff:
+ relSrc = self.__project.getRelativePath(str(file)) if self.__project else ""
+ self.__diffFormatFile(
+ pathlib.Path(file), fast=False, mode=mode, report=report, relSrc=relSrc
+ )
+ else:
+ black.reformat_one(
+ pathlib.Path(file),
+ fast=False,
+ write_back=writeBack,
+ mode=mode,
+ report=report,
+ )
self.__finish()
- def __diffFormatFile(self, src, fast, mode, report):
+ @staticmethod
+ def __diffFormatFile(src, fast, mode, report, relSrc=""):
"""
- Private method to check, if the given files need to be reformatted, and generate
+ Static method to check, if the given files need to be reformatted, and generate
a unified diff.
@param src path of file to be checked
@@ -303,6 +431,8 @@
@type black.Mode
@param report reference to the report object
@type BlackReport
+ @param relSrc name of the file relative to the project (defaults to "")
+ @type str (optional)
"""
then = datetime.datetime.utcfromtimestamp(src.stat().st_mtime)
with open(src, "rb") as buf:
@@ -313,9 +443,7 @@
report.done(src, black.Changed.NO)
return
- fileName = str(src)
- if self.__project:
- fileName = self.__project.getRelativePath(fileName)
+ fileName = relSrc if bool(relSrc) else str(src)
now = datetime.datetime.utcnow()
srcName = f"{fileName}\t{then} +0000"
@@ -395,6 +523,9 @@
self.__statistics.failureCount += 1
isError = True
+ if status != "ignored":
+ self.__statistics.processedCount += 1
+
if self.__project:
filename = self.__project.getRelativePath(filename)
@@ -420,6 +551,7 @@
changeCount: int = 0
sameCount: int = 0
failureCount: int = 0
+ processedCount: int = 0
class BlackReport(QObject, black.Report):
@@ -433,20 +565,16 @@
result = pyqtSignal(str, str, str)
- def __init__(self, dialog):
+ def __init__(self, parent=None):
"""
Constructor
- @param dialog reference to the result dialog
- @type QDialog
+ @param parent reference to the parent object (defaults to None
+ @type QObject (optional)
"""
- QObject.__init__(self, dialog)
+ QObject.__init__(self, parent)
black.Report.__init__(self)
- self.ignored_count = 0
-
- self.__dialog = dialog
-
def done(self, src, changed, diff=""):
"""
Public method to handle the end of a reformat.
@@ -490,3 +618,83 @@
@type str (optional)
"""
self.result.emit("ignored", str(src), "")
+
+
+@dataclass
+class BlackMultiprocessingResult:
+ """
+ Class containing the reformatting result data.
+
+ This class is used when reformatting multiple files in parallel using processes.
+ """
+
+ status: str = ""
+ filename: str = ""
+ data: str = ""
+
+
+class BlackMultiprocessingReport(black.Report):
+ """
+ Class extending the black Report to work with multiprocessing.
+ """
+
+ def __init__(self, resultQueue):
+ """
+ Constructor
+
+ @param resultQueue reference to the queue to put the results into
+ @type multiprocessing.Queue
+ """
+ super().__init__()
+
+ self.__queue = resultQueue
+
+ def done(self, src, changed, diff=""):
+ """
+ Public method to handle the end of a reformat.
+
+ @param src name of the processed file
+ @type pathlib.Path
+ @param changed change status
+ @type black.Changed
+ @param diff unified diff of potential changes (defaults to "")
+ @type str
+ """
+ if changed is black.Changed.YES:
+ status = "changed"
+
+ elif changed is black.Changed.NO:
+ status = "unchanged"
+
+ elif changed is black.Changed.CACHED:
+ status = "unmodified"
+
+ self.__queue.put(
+ BlackMultiprocessingResult(status=status, filename=str(src), data=diff)
+ )
+
+ def failed(self, src, message):
+ """
+ Public method to handle a reformat failure.
+
+ @param src name of the processed file
+ @type pathlib.Path
+ @param message error message
+ @type str
+ """
+ self.__queue.put(
+ BlackMultiprocessingResult(status="failed", filename=str(src), data=message)
+ )
+
+ def path_ignored(self, src, message=""):
+ """
+ Public method handling an ignored path.
+
+ @param src name of the processed file
+ @type pathlib.Path or str
+ @param message ignore message (default to "")
+ @type str (optional)
+ """
+ self.__queue.put(
+ BlackMultiprocessingResult(status="ignored", filename=str(src), data="")
+ )
